abstract type SplittingCriterion  end
struct DifferenceSplit      <: SplittingCriterion end

abstract type SortBy end
struct SortByMean 	<: SortBy end

mutable struct ModelSettings
	model_type::String # 1
	minWeight::Float64 # 2
	randomw::Float64 # 3
	crit::SplittingCriterion # fn version of 4
	maxSplittingPoints::Int # 5
	iterations::Int # 6
	learningRate::Float64 # 7 moderationfactor
	nScores::Int # 8
	adaptiveLearningRate::Float64 # 9
	prem_buffer::Int # 11
	startAtMean::Bool # 12
	writeTree::Bool # 13
	number_of_num_features::Int # 14
	spawnsmaller::Bool # 26
    boolRankOptimization::Bool # 35
	boolRandomizeOnlySplitAtTopNode::Bool # 37
	subsampling_prop::Float64 # 38 #NOTE Friedman suggests to sample without replacement see Friedman 2002 Stochastic Gradient Boosting,” Computational Statistics and Data Analysis 38(4):367-378
	subsampling_features_prop::Float64 # 39
	version::String # 40
	preppedJLDFileExists::Bool # 41
	catSortByThreshold::Int # 42
	catSortBy::SortBy # 43
	scorebandsstartingpoints::Array{Int,1} # 44
	showTimeUsedByEachIteration::Bool # 45
	smoothEstimates::String # 46
	deriveFitPerScoreFromObservedRatios::Bool # deriveFitPerScoreFromObservedRatios::Bool (if this is true then the fit per score will be based on the observed ratio per score (instead of the fitted ratio per score)) . Note that this does not influence the structure of the tree, but only the fitted ratio per score (and scoreband)
    roptForcedPremIncr::Bool # 47
    writeSasCode::Bool # 49
	writeIterationMatrix::Bool # 50
	writeResult::Bool # 51
	writeStatistics::Bool # 52
	boolCreateZipFile::Bool # 53
	writeCsharpCode::Bool
	writeVbaCode::Bool
	nDepthToStartParallelization::Int
	baggingWeightTreesError::String
	cBB_niterBoosting::Int
	cBB_niterBagging::Int
	fixedinds::Array{Int,1}
	boolNumeratorStats::Bool
	statsByVariables::Array{Int,1}
	statsRandomByVariable::Int
	saveJLDFile::Bool # if this is set to false no *.jld2 file is saved #todo we should rename this to ensure it is clear that this refers to the prepped data only!
	saveResultAsJLDFile::Bool # this refers to the model output/result
	print_details::Bool # whether julia should print details to the log or not (note to developers: warnings/errors and certain important infos should always be printed!)
	seed::Int # optional entropy
	graphvizexecutable::String # e.g. C:\Program Files (x86)\Graphviz2.38\ if this
	showProgressBar_time::Bool # bar form iterators package (or some other package)
	prroduceEstAndLeafMatrices::Bool
	write_dot_graph::Bool
	calculateGini::Bool # whether or not to calculate the gini (which requires sorting of the data (which takes time!))
	calculatePoissonError::Bool
    performanceMeasure::String
	fitForStatsAndCharts::String
	ignoreZeroDenominatorValues::Bool

	# the following are treated specially
	df_name_vector::Array{String,1}
    number_of_char_features::Int # this is calculated by Julia from number_of_num_features and ncolsdfIndata (which is defined by the data)

	chosen_apply_tree_fn::String # defined below
	moderationvector::Array{Float64,1} # 50 onwards

    function ModelSettings()
	# initialize default settings
	model_type = "build_tree" # 1
	minWeight = -0.1 # 2
	randomw = 0.0 # 3
	crit = DifferenceSplit()	 # 4
	maxSplittingPoints = 250 # 5 #arbitrary choice. Preliminary testing has shown no performance difference for different values (say 10 or 200) for this setting
	iterations = 2 # 6
	learningRate = 0.1 # 7
	nScores = 1000 # 8
	adaptiveLearningRate = 1.0 # 9
	prem_buffer = 0 # 11
	startAtMean = true # 12
	writeTree = true # 13
	number_of_num_features = -1 # 14
	spawnsmaller = true # 26
	boolRankOptimization = false # 35
	boolRandomizeOnlySplitAtTopNode = true # 37
	subsampling_prop = 1.0 # 38
	subsampling_features_prop = 1.0 # 39
	version = "not_initialized" # 40
	preppedJLDFileExists = false # 41
	catSortByThreshold = 7 # 42
	catSortBy = SortByMean() # 43
	scorebandsstartingpoints = [parse(Int, x) for x in split("1,100,200,300,400,500,600,700,800,900", ',')] # scorebandsstartingpoints=parse(Int,split("1,100,200,300,400,500,600,700,800,900",',')) #44
	showTimeUsedByEachIteration = false # true #45
	smoothEstimates = "1" # 46 this is a string for now (as there could be different smoothing methods specified by this string)
	deriveFitPerScoreFromObservedRatios = true
	roptForcedPremIncr = false # 47
        writeSasCode = false # 49
	writeIterationMatrix = false# 50
	writeResult = false # true #51
	writeStatistics = true # 52
	boolCreateZipFile = false # true #53
	writeCsharpCode = false # true
	writeVbaCode = false
	nDepthToStartParallelization = -1
	baggingWeightTreesError = "mae" # ::String
	# combined bagging&boosting model
	cBB_niterBoosting = 0
	cBB_niterBagging = 0
	fixedinds = Array{Int}(undef, 0)
	boolNumeratorStats = false
	statsByVariables = Int[]
	statsRandomByVariable = 5
	saveJLDFile = true
	saveResultAsJLDFile = false
	print_details = true
	seed = 90210
	graphvizexecutable = "" # C:\\Program Files (x86)\\Graphviz2.38\\bin\\dot.exe""
	showProgressBar_time = true
	prroduceEstAndLeafMatrices = false
	write_dot_graph = false
	calculateGini = false
	calculatePoissonError = false
        performanceMeasure = "Lift Val"
	fitForStatsAndCharts = "rawRelativities" # rawRelativities,unsmoothedPerScore,smoothedPerScore
	ignoreZeroDenominatorValues = false

	# the following are treated specially
	df_name_vector = Array{String}(undef, 0) #
        number_of_char_features = -1 #
	chosen_apply_tree_fn = "apply_tree_by_leaf" # #apply_tree_by_row does not seem to work for (certain?) boosting models
	moderationvector = [0.1] #

	return new(model_type,minWeight,randomw,crit,maxSplittingPoints,iterations,learningRate,nScores,adaptiveLearningRate,prem_buffer,startAtMean,writeTree,number_of_num_features,spawnsmaller,boolRankOptimization,boolRandomizeOnlySplitAtTopNode,subsampling_prop,subsampling_features_prop,version,preppedJLDFileExists,catSortByThreshold,catSortBy,scorebandsstartingpoints,showTimeUsedByEachIteration,smoothEstimates,deriveFitPerScoreFromObservedRatios,roptForcedPremIncr,writeSasCode,writeIterationMatrix,writeResult,writeStatistics,boolCreateZipFile,writeCsharpCode,writeVbaCode,nDepthToStartParallelization,baggingWeightTreesError,cBB_niterBoosting,cBB_niterBagging,fixedinds,boolNumeratorStats,statsByVariables,statsRandomByVariable,saveJLDFile,saveResultAsJLDFile,print_details,seed,graphvizexecutable,showProgressBar_time,prroduceEstAndLeafMatrices,write_dot_graph,calculateGini,calculatePoissonError,performanceMeasure,fitForStatsAndCharts,ignoreZeroDenominatorValues
	 ,df_name_vector,number_of_char_features,chosen_apply_tree_fn,moderationvector)
    end  # ModelSettings()

end # type definition

function writeAllFieldsToArray(s)
	nn = fieldnames(typeof(s))
	res = Array{AbstractString}(undef, length(nn), 2)
	i = 0
	for x in nn
		i += 1
        @show x
        tmp = AbstractString[string(x) string(getfield(s, x))][:]
        @show tmp
        @show size(tmp)
        @show typeof(tmp)
		res[i,:] = AbstractString[string(x) string(getfield(s, x))][:]
	end
	return res
end

mi = ModelSettings()
foo = writeAllFieldsToArray(mi)


using DataFrames, Dash, DashHtmlComponents, DashCoreComponents, UrlDownload, PlotlyJS, JSON3
using Statistics
using LinearAlgebra
using CSV
using RCall

app = dash()

#calling R libraries

    R"""
    source("RScripts/preamble.R")
    """

#dataframes for R analysis
genon = DataFrame(CSV.File("RScripts/GO_final.txt"))
cellNew = DataFrame(CSV.File("RScripts/cell_new_final.txt"))

Yte=Matrix(CSV.read("PCA_matrix_log2.txt",DataFrame))
PCM = DataFrame(CSV.File("data/Kidney_Q3Norm_TargetCountMatrix.txt"))
print(PCM[2,2])
foreach(n -> PCM[!, n] = log2.(PCM[!, n]), names(PCM)[2:end])
print(PCM[2,2])
features = DataFrame(CSV.File("data/Kidney_Sample_Annotations.txt"))
structuresDict = Dict("abnormal"=>"Glom (Abnormal)","healthy"=>"Glom (Healthy)"," PanCK" => "Tub. Distal", " neg" => "Tub. Proximal")
comprehensiveStates = [structuresDict[ismissing(row.pathology) ? split(row["SegmentDisplayName"],"|")[3] : row.pathology] for row in eachrow(features)]
genes = PCM.TargetName
diseaseStatusMarkers = replace(features[:,"disease_status"], "DKD" => "triangle-up", "normal" => "circle")
# diseaseStatusMarkers .= relpace.(diseaseStatusMarkers, "DKD", "square")
insertcols!(features,"states"=>comprehensiveStates)
insertcols!(features, "barID" =>["$(row[1]) | $(row[2])" for row in eachrow( features[!,["ROILabel","states"]]) ])

CellTypes = DataFrame(CSV.File("data/Cell_Types_for_Spatial_Decon.txt"))
GO = DataFrame(CSV.File("data/Kidney_ssGSEA.txt"))
rename!(GO, names(GO)[2:end] .=> features.Sample_ID)
#group immune together
CellTypes[end-15:end,9].="Immune"
CDC = DataFrame(CSV.File("data/Kidney_Spatial_Decon.txt"))
rename!(CDC, names(CDC)[2:end] .=> features.Sample_ID)
insertcols!(CDC,  "cellGroup" => [CellTypes[ CellTypes.Alias .== row[1],9][1] for row in eachrow(CDC)])
to_group = names(CDC[!, Not(["Alias", "cellGroup"])])
newCDC = combine(groupby(CDC,"cellGroup"), to_group .=> sum .=> to_group)
cells = CDC.Alias
cellGroups = newCDC.cellGroup

diseaseOptions = [
    Dict("label" => "Normal", "value" => "normal"),
    Dict("label" => "DKD", "value" => "DKD"),
]

function generate_table(dataframe, max_rows = 10)
    html_table([
        html_thead(html_tr([html_th(col) for col in names(dataframe)])),
        html_tbody([
            html_tr([html_td(dataframe[r, c]) for c in names(dataframe)]) for r = 1:min(nrow(dataframe), max_rows)
        ]),
    ])
end

app = dash()

#available_indicators = ["disease_status","region","pathology"]]

patients = unique(features.SlideName)
patientsNormal = patients[ occursin.("normal", patients) ]
patientsDKD = patients[ occursin.("disease", patients) ]

app.layout = html_div() do
    html_div(
        children = [

    	    html_h1(
				"Data Visualization App",
        		style = Dict("color" => "#0066cc", "textAlign" => "left"),
    		),
		    dcc_markdown("## Data selection"),
            html_div("Select data to plot:"),
        	html_div(
        		children = [
        		html_div(
		            dcc_dropdown(
		                id = "crossfilter-roi-type",
		                options = [
		                    (label = i, value = i)
		                    for i in values(structuresDict)
		                ],
		                multi = true,
		                # value = first(values(structuresDict)),
		                placeholder = "Select type of structure",
		                style = (width = "70%", display = "block", marginBottom = "5px")
		            ),
	            	# style = (width = "100%")
	            ),

	            ##Sync to patients in crossfilter-patient
	            html_div(
		            dcc_dropdown(
		                id = "crossfilter-patient",
		                options = [
		                    (label = i, value = i)
		                    for i in patients
		                ],
		                multi = true,
		                # value = [patients[1]],
		                placeholder = "Select patient",
		                style = (width = "70%", display = "block", marginBottom = "5px")
		            ),
	            ),

	            ##Add checkbox for healthy vs DKD
	            html_button(id = "buttonAll", children = "All", n_clicks = 0),
	            html_button(id = "buttonDKD", children = "▲ - DKD", n_clicks = 0),
	            html_button(id = "buttonNormal", children = "● - Normal", n_clicks = 0),
	            html_button(id = "buttonClear", children = "Clear", n_clicks = 0),
	            ],
	            style = (borderBottom = "thin lightgrey solid", backgroundColor = "rgb(250, 250, 250)", padding = "10px 5px",),
            ),

        	html_div(
        		children = [
		            dcc_graph(
		                id = "graph-1",
		            ),
	            ],
                style = ( width = "49%", display = "inline-block"),
            ),

        	html_div(
        		children = [
		            dcc_graph(
		                id = "graph-2",
		            ),
	            ],
                style = ( width = "49%", display = "inline-block"),
            ),

            dcc_markdown("## Group selection"),
            dcc_markdown("Select a region of interest from any of the plots above and store the information in a group."),
            dcc_dropdown(
                id = "stage",
                options = [
                    (label = i, value = i)
                    for i in unique(features.SegmentDisplayName)
                ],
                multi = true,
                value =features[1,"SegmentDisplayName"],
            ),
            html_button(id = "g1", children = "Store in group 1", n_clicks = 0),
            html_button(id = "g2", children = "Store in group 2", n_clicks = 0),
            html_div(
                children = [
                    dcc_markdown("
                    **Group 1**
                    "),
                    html_pre(id = "g1-header"),
                ],
            ),
            dcc_dropdown(
                id = "group1",
                options = [
                    (label = i, value = i)
                    for i in unique(features.SegmentDisplayName)
                ],
                multi = true,
                value =features[1:2,"SegmentDisplayName"],
            ),
            html_div(
                children = [
                    dcc_markdown("
                    **Group 2**
                    "),
                    html_pre(id = "g2-header"),
                ],
            ),
            dcc_dropdown(
                id = "group2",
                options = [
                    (label = i, value = i)
                    for i in unique(features.SegmentDisplayName)
                ],
                multi = true,
                value =features[3:4,"SegmentDisplayName"],
            ),
            html_div(
                children = [
                    dcc_markdown("
                    **R - Analysis**
                    Use the data from the selected groups (Gene expression, Cell Deconvolution, GESA)
                    "),
                    html_pre(id = "aaaa"),
                ],
            ),
            html_div(id = "dummy", children = "e"),

            html_button(id = "RRun", children = "Execute R Analysis", n_clicks = 0),

            #Group analysis
            html_div(
                children = [
                    dcc_markdown("
                    **BubblePlot**
                    "),
                    html_pre(id = "bubble-header"),
                ],
            ),
            #restrict to patients present in group1 intersection group2
            dcc_dropdown(
                id = "patientsBubble",
                options = [
                    (label = i, value = i)
                    for i in patients
                ],
                value =patients[1],
            ),
            dcc_dropdown(
                id = "geneBubble",
                options = [
                    (label = i, value = i)
                    for i in genes
                ],
                value = genes[1],
            ),

            dcc_graph(
                id = "bubblePlot1",
            ),
            dcc_dropdown(
                id = "cellBubble",
                options = [
                    (label = i, value = i)
                    for i in cellGroups
                ],
                value = cellGroups[1],
            ),

            dcc_graph(
                id = "bubblePlot2",
            ),

        	html_div(
        		children = [
                    dcc_graph(
                        id = "bar1",
                    ),
	            ],
                style = ( width = "49%", display = "inline-block"),
            ),

        	html_div(
        		children = [
                    dcc_graph(
                        id = "bar2",
                    ),
	            ],
                style = ( width = "49%", display = "inline-block"),
            ),
        ],
        style = (
            borderBottom = "thin lightgrey solid",
            backgroundColor = "rgb(250, 250, 250)",
            padding = "10px 5px",
        ),
    )
end

callback!(
    app,
    Output("crossfilter-patient", "value"),
    Input("buttonNormal", "n_clicks"),
    Input("buttonDKD", "n_clicks"),
    Input("buttonAll", "n_clicks"),
    Input("buttonClear", "n_clicks"),
    State("crossfilter-patient", "value"),
) do clickNormal, clickDKD, clickAll, clickClear, patientList

    ctx = callback_context()
    if isempty(ctx.triggered)
    	return patientList
    end

    trigger = ctx.triggered[1]
    button_id = split(trigger.prop_id, ".")[1]

    if button_id == "buttonAll"
    	return patients
    end
    if button_id == "buttonClear"
    	return []
    end
    if button_id == "buttonDKD"
    	#patientList = union(patientList,patientsDKD)
    	 patientList = patientsDKD
    end
    if button_id == "buttonNormal"
    	#patientList = union(patientList,patientsNormal)
    	 patientList = patientsNormal
    end

    return patientList

end

callback!(
    app,
    Output("graph-1", "figure"),
    Input("crossfilter-roi-type", "value"),
    # Input("crossfilter-disease-status", "value"),
    Input("crossfilter-patient", "value"),
) do roiList, patientList

	# if no selection return empty plot
	if roiList == nothing
        return (data = [], layout = (title = "By Structure", xaxis_title = "", yaxis_title = "", clickmode = "event+select",))
    end

    # diseaseFilter = true
    # if !isempty(diseaseStatus)
    # 	for ds in diseaseStatus
    # 		diseaseFilter = (features[!,"disease_status"] .== ds) .| diseaseFilter
    # 	end
    # end

    patientFilter = false
    if patientList != nothing
    	patientFilter = [slide in patientList for slide in features.SlideName]
    end

    accumFilter = patientFilter # .& diseaseFilter

    plotData = [    ( x = Yte[1, (comprehensiveStates .== status) .& accumFilter], y = Yte[2, (comprehensiveStates .== status) .& accumFilter],  type = "scatter", name = status, mode = "markers", marker = (size = 8, symbol = diseaseStatusMarkers[ (comprehensiveStates .== status) .& accumFilter ], ), text = features[(comprehensiveStates .== status) .& accumFilter,"SlideName" ], customdata = features[(comprehensiveStates .== status) .& accumFilter,"SegmentDisplayName" ]) for status in roiList]

    return (
        data = plotData,
        layout = (
            title = "By Structure",
            xaxis = (title = "pca_1",),
            yaxis = (title = "pca_2",),
            clickmode = "event+select",
        ),
    )

end

callback!(
    app,
    Output("graph-2", "figure"),
    Input("crossfilter-patient", "value"),
    Input("stage", "value"),
    Input("crossfilter-roi-type", "value"),
) do patientList, selected_data, roiList

	# if no selection return empty plot
	if patientList == nothing
        return ( data = [], layout =  "title" => "By Patient"  )
    end

    selectedFilter = true
    # if selected_data != nothing
    # 	selectedFilter = [x in selected_data for x in features[!,"SegmentDisplayName"]]
    # end

	roiFilter = false
	if roiList != nothing
    	roiFilter = [x in roiList for x in comprehensiveStates]
    end

    accumFilter = selectedFilter .& roiFilter
    # accumFilter = true

    selector = "SlideName"

    plotData = [    ( x = Yte[1, (features[!,selector] .== status ) .& accumFilter ], y = Yte[2, (features[!,selector] .== status) .& accumFilter],  type = "scatter", name = status, mode = "markers", text = comprehensiveStates[(features[!,selector] .== status) .& accumFilter]	) for status in patientList]

    return (
        data = plotData,
        layout = (
            title = "By Patient",
            xaxis = (title = "pca_1",),
            yaxis = (title = "pca_2",),
        ),
    )
end

callback!(
    app,
    Output("stage", "value"),
    Input("graph-1", "selectedData"),
) do selected_data
    selectedpoints = 1:2
    if selected_data != nothing
        selectedpoints = [p[:customdata] for p in selected_data.points]
        #print(selectedpoints)
    end
    #print(features[selectedpoints, "SegmentDisplayName"])
    #return features[selectedpoints, "SegmentDisplayName"]
    return selectedpoints
end

#buttons
callback!(
    app,
    Output("group1", "value"),
    Input("g1", "n_clicks"),
    State("stage", "value"),
) do clicks, input_1
    return input_1
end

callback!(
    app,
    Output("group2", "value"),
    Input("g2", "n_clicks"),
    State("stage", "value"),
) do clicks, input_1
    return input_1
end

callback!(
    app,
    Output("dummy", "children"),
    Input("RRun", "n_clicks"),
    State("group1", "value"),
    State("group2", "value"),
) do clicks, input_1, input_2
    clicks == 0 && return " "
    CSV.write("matrix_group2_group1.txt",rename(PCM,names(PCM)[1] => "gene")[!,vcat(["gene"], input_1, input_2)], delim='\t')
    n1 = length(input_1)
    n2 = length(input_2)
    @rput n1
    @rput n2
    CSV.write("matrix_Design_final.txt", DataFrame("sample" => vcat(input_1, input_2), "group1" => convert(Vector{UInt8},vcat(trues(n1),falses(n2))), "group2" => convert(Vector{UInt8},vcat(falses(n1),trues(n2)))), delim = '\t')
groupListsInPatients= [
    [ any(x .== input_1) for x in features[!,"SegmentDisplayName"]],
    [ any(x .== input_2) for x in features[!,"SegmentDisplayName"]]
    ]

    cnf1 = DataFrame((newCDC.cellGroup .=> [Vector(row) for row in eachrow( newCDC[!,features[groupListsInPatients[1],"Sample_ID"]])])... , "Group" => "Group1")
    cnf2 = DataFrame((newCDC.cellGroup .=> [Vector(row) for row in eachrow( newCDC[!,features[groupListsInPatients[2],"Sample_ID"]])])... , "Group" => "Group2")
    CSV.write("cell_new_final.txt", vcat(cnf1,cnf2), delim = '\t')

    gf1 = DataFrame((GO.Column1 .=> [Vector(row) for row in eachrow( GO[!,features[groupListsInPatients[1],"Sample_ID"]])])... , "Group" => "Group1")
    gf2 = DataFrame((GO.Column1 .=> [Vector(row) for row in eachrow( GO[!,features[groupListsInPatients[2],"Sample_ID"]])])... , "Group" => "Group2")
    CSV.write("GO_final.txt", vcat(gf1,gf2), delim = '\t')

    R"""
    source("RScripts/de.R")
    """

    return "DE and all"
end

callback!(
    app,
    Output("bubblePlot1", "figure"),
    Input("patientsBubble", "value"),
    Input("geneBubble", "value"),
    Input("group1", "value"),
    Input("group2", "value"),
) do patient, selectedGene, group1,group2
patientRegions= features[!,"SlideName"].==patient
points = features[patientRegions,["SlideName", "SegmentDisplayName", "RoiReportX", "RoiReportY", "states"]]
groupListsInPatients= [
    [ any(x .== group1) for x in points[!,"SegmentDisplayName"]],
    [ any(x .== group2) for x in points[!,"SegmentDisplayName"]]
    ]
#print(any(any.(groupListsInPatients)))
normalizedCounts =Any[ ]

maxCounts = mean(Vector(PCM[PCM[!,"TargetName" ] .== selectedGene,points[!,:].SegmentDisplayName][1,:]))
for group in groupListsInPatients
    if any(group)
        tmp = Vector(PCM[PCM[!,"TargetName" ] .== selectedGene,points[group,:].SegmentDisplayName][1,:])
        #print(tmp)
        push!(normalizedCounts,50 .* tmp./maxCounts)
    end
end
#Plots.scatter(points.ROICoordinateX,points.ROICoordinateY,color=1,label="Tubules")
#Plots.title!(patient)
#Plots.scatter!(points[gloms,:].ROICoordinateX,points[gloms,:].ROICoordinateY,color=2,label="Gloms")
#print(normalizedCounts)

####PENDING#####
#text = structuresDict
    plotData = [    ( x =points[group,:].RoiReportX, y = points[group,:].RoiReportY,  type = "scatter", name = string(i), mode = "markers", marker = (size=normalizedCounts[i], symbol = "circle", ), text = points[group,:].states, customdata = points[group,:].SegmentDisplayName )  for(i,  group) in enumerate( groupListsInPatients) if any(group)]

    return ( data = plotData, layout = ( title = "Gene Expression in Slide", xaxis_title = "x", yaxis_title = "y",),)

end

callback!(
    app,
    Output("bubblePlot2", "figure"),
    Input("patientsBubble", "value"),
    Input("cellBubble", "value"),
    Input("group1", "value"),
    Input("group2", "value"),
) do patient, selectedCell, group1,group2
patientRegions= features[!,"SlideName"].==patient
points = features[patientRegions,["SlideName", "SegmentDisplayName", "RoiReportX", "RoiReportY", "states","Sample_ID"]]
groupListsInPatients= [
    [ any(x .== group1) for x in points[!,"SegmentDisplayName"]],
    [ any(x .== group2) for x in points[!,"SegmentDisplayName"]]
    ]
#print(any(any.(groupListsInPatients)))
normalizedCounts =Any[ ]

maxCounts = maximum(Vector(newCDC[newCDC[!,"cellGroup" ] .== selectedCell,points[!,:].Sample_ID][1,:]))
for group in groupListsInPatients
    if any(group)
        tmp = Vector(newCDC[newCDC[!,"cellGroup" ] .== selectedCell,points[group,:].Sample_ID][1,:])
        #print(tmp)
        push!(normalizedCounts,50 .* tmp./maxCounts)
    end
end

####PENDING#####
#text = structuresDict
    plotData = [    ( x =points[group,:].RoiReportX, y = points[group,:].RoiReportY,  type = "scatter", name = string(i), mode = "markers", marker = (size=normalizedCounts[i], symbol = "circle", ), text = points[group,:].states, customdata = points[group,:].SegmentDisplayName )  for(i,  group) in enumerate( groupListsInPatients) if any(group)]

    return ( data = plotData, layout = ( title = "Cell Fraction in Slide", xaxis_title = "x", yaxis_title = "y",),)

end

callback!(
    app,
    Output("bar1", "figure"),
    Input("patientsBubble", "value"),
    #Input("cellBubble", "value"),
    Input("group1", "value"),
    #Input("group2", "value"),
) do patient,  group1
#patientRegions= features[!,"SlideName"].==patient
points = features[!,["SlideName", "SegmentDisplayName", "RoiReportX", "RoiReportY", "states","barID","Sample_ID"]]
group=  [ any(x .== group1) for x in points[!,"SegmentDisplayName"]]

#print(groupListsInPatients)

    plotData = [    ( x =points[group,:].SegmentDisplayName, y = Vector(newCDC[newCDC[!,"cellGroup" ] .== cell,points[group,:].Sample_ID][1,:]),  type = "bar", name = cell, text = cell, customdata = cell ) for cell in cellGroups  ]

    return ( data = plotData, layout = ( title = "Group 1 Cell Composition",barmode = "stack",
            #xaxis = (title = "ROIs",),
            yaxis = (title = "CD Cell Fraction",),
    )
    )

end

callback!(
    app,
    Output("bar2", "figure"),
    Input("patientsBubble", "value"),
    #Input("cellBubble", "value"),
    Input("group2", "value"),
) do patient,  group1
#patientRegions= features[!,"SlideName"].==patient
points = features[!,["SlideName", "SegmentDisplayName", "RoiReportX", "RoiReportY", "states","barID","Sample_ID"]]
group=  [ any(x .== group1) for x in points[!,"SegmentDisplayName"]]

#print(groupListsInPatients)

    plotData = [    ( x =points[group,:].SegmentDisplayName, y = Vector(newCDC[newCDC[!,"cellGroup" ] .== cell,points[group,:].Sample_ID][1,:]),  type = "bar", name = cell, text = cell, customdata = cell ) for cell in cellGroups  ]

    return ( data = plotData, layout = ( title = "Group 2 Cell Composition",barmode = "stack",
            #xaxis = (title = "ROIs",),
            yaxis = (title = "CD Cell Fraction",),
    showlegend = false),)

end

run_server(app, "0.0.0.0", debug=true)


module Aurora

using Reexport

using LaTeXStrings
using LinearAlgebra
using MultivariateStats
using NearestNeighbors
using RecipesBase

using Statistics
@reexport using StatsBase

abstract type AbstractAurora end
abstract type AbstractFittedAurora end
StatsBase.predict(fitted_aurora::AbstractFittedAurora) = fitted_aurora.μs

include("replicated_sample.jl")
include("auroral.jl")
include("auroraknn.jl")

export ReplicatedSample,
       Auroral,
       CoeyCunningham,
       AuroraKNN

end


    formats = (
        "+##(#)##########",
        "+##(#)##########",
        "0##########",
        "0##########",
        "###-###-####",
        "(###)###-####",
        "1-###-###-####",
        "###.###.####",
        "###-###-####",
        "(###)###-####",
        "1-###-###-####",
        "###.###.####",
        "###-###-####x###",
        "(###)###-####x###",
        "1-###-###-####x###",
        "###.###.####x###",
        "###-###-####x####",
        "(###)###-####x####",
        "1-###-###-####x####",
        "###.###.####x####",
        "###-###-####x#####",
        "(###)###-####x#####",
        "1-###-###-####x#####",
        "###.###.####x#####"
    )

 function phone_number()

    t= formats[rand(1:length(formats))]

    for i=t
     t= replace(t,"#","$(rand(0:9))",1)
    end

    return t
end


# old goldstandard
const goldstandard_file = "C:\\Users\\Daniel\\Documents\\Bitbucket\\musical-grammars\\data\\annotations\\Goldstandard.txt"
const chroma = Dict("C"=>0,"D"=>2,"E"=>4,"F"=>5,"G"=>7,"A"=>9,"B"=>11)

function pitch_class(note_name::AbstractString)
    m = match(r"([A-G])(b*|#*)", note_name)
    if contains(m[2], "#")
        chroma[m[1]] + length(m[2])
    else
        chroma[m[1]] - length(m[2])
    end
end

function translate_goldstandard_notation1(chord::AbstractString)
    m = match(r"([A-G]b*#*)(.*)", chord)
    string(pitch_class(m[1]), m[2])
end

function translate_goldstandard_notation1(sequence::AbstractVector)
    map(translate_goldstandard_notation1, sequence)
end

const chord_form_dict = Dict(""=>"maj", "-"=>"min", "%"=>"hdim", "o"=>"dim")

function translate_goldstandard_notation2(chord::AbstractString)
    # println(chord)
    m = match(r"([A-G]b*#*)(.*)", chord)
    # println(m)
    string(pitch_class(m[1]), chord_form_dict[m[2]])
end

function translate_goldstandard_notation2(sequence::AbstractVector)
    map(translate_goldstandard_notation2, sequence)
end

function produce_goldstandard_sequences()
    lines = map(strip, readlines(goldstandard_file))
    for line in lines
        if !isempty(line) && line[1] == '\$' && line[end] == '\$'
            if contains(line[2:end-1], "maj")
                produce(translate_goldstandard_notation1(split(line[2:end-1])))
            else
                produce(translate_goldstandard_notation2(split(line[2:end-1])))
            end
        end
    end
end

function read_goldstandard()
    collect(@task(produce_goldstandard_sequences()))
end

#new treebank
const treebank_chord_form_dict = Dict(""=>"maj", "7"=>"maj", "^7"=>"maj", "6"=>"maj",
                                      "m"=>"min", "m7"=>"min", "m6"=>"min",
                                      "%7"=>"hdim", "o"=>"dim", "o7"=>"dim", "+"=>"aug")

function translate_treebank_notation(chord::AbstractString)
    m = match(r"([A-G]b*#*)(.*)", chord)
    string(pitch_class(m[1]), treebank_chord_form_dict[m[2]])
end

function translate_treebank_notation(chords::AbstractVector)
    map(translate_treebank_notation, chords)
end

function right_branching_tree(num_leafs::Int)
    if num_leafs == 1
        TreeNode("x")
    else
        n = TreeNode("x")
        insert_child!(n, right_branching_tree(num_leafs - 1))
        insert_child!(n, TreeNode("x"))
        n
    end
end

function read_tree(treebank_file::AbstractString)
    chomp(readlines(treebank_file)[4])[3:end]
end

function read_treebank_trees()
    dir = joinpath(dirname(@__FILE__), "..", "..", "trees")
    files = ["afternoon_in_paris.tsv", "all_of_me.tsv", "alone_together.tsv", "anthropology.tsv", "a_night_in_tunisia.tsv"]
    # files = ["afternoon_in_paris.tsv", "all_of_me.tsv"]
    trees = TreeNode{String}[]
    for file in files
        t = tree(read_tree(joinpath(dir, file)))
        for n in t
            n.data = translate_treebank_notation(n.data)
        end
        push!(trees, t)
    end
    trees
end

function read_goldstandard_trees_1()
    trees = map(tree,
    ["[Cmaj[Cmaj[Cmaj[Cmaj[Cmaj][Cmaj[Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Dmin][Gmaj]]][Cmaj]]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Dmin][Gmaj]]][Cmaj]]]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Dmin][Gmaj]]][Cmaj[Cmaj][Cmaj]]]]][Cmaj[Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Gmaj[Dmin[Amaj][Dmin]][Gmaj]][Cmaj]]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Dmin][Gmaj]]][Cmaj]]]]"
    "[Cmaj[Cmaj[Cmaj][Gmaj[Dmin[Amaj[Emaj][Amaj]][Dmin]][Gmaj[Dmaj[Amin[Emaj[Bmin][Emaj]][Amin][Dmaj]][Gmaj[Dmin][Gmaj]]]]]][Cmaj[Cmaj[Cmaj][Cmaj[Gmaj[Dmin[Amaj[Emaj][Amaj]][Dmin]][Gmaj[Fmaj][Gmaj[Fmin][Gmaj[Dmin[Amaj[Emin][Amaj]][Dmin]][Gmaj]]]]][Cmaj]]][Gmaj[Dmin[Ebdim][Dmin]][Gmaj]]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Gmaj[Gmaj[Dmin[Amin[Emaj[Bhdim[Fmaj][Bhdim]][Emaj]][Amin]][Dmin[Ebmaj][Dmin]]][Gmaj]][Gmaj[Dmin[Amin[Emin][Amin]][Dmin]][Gmaj]]]][Cmaj[Cmaj[Gmaj[Dmaj[Amaj[Emin][Amaj]][Dmaj[Amin][Dmaj]]][Gmaj[Dmin][Gmaj]]][Cmaj]][Cmaj[Gmaj[Gmaj[Dmin[Amin[Emaj[Bhdim[Fmaj][Bhdim]][Emaj]][Amin]][Dmin[Ebmaj][Dmin]]][Gmaj]][Gmaj[Dmin[Amin[Emin][Amin]][Dmin]][Gmaj]]][Cmaj]]]][Cmaj[Gmaj[Amin[Gmaj[Dhdim[Amin][Dhdim]][Gmaj]][Amin[Emin][Amin]]][Gmaj[Fmaj[Cmaj[Gmaj[Dhdim][Gmaj]][Cmaj]][Fmaj]][Gmaj[Dmin[Dmin[Amaj[Emin[Bmaj[F#min][Bmaj]][Emin]][Amaj]][Dmin]][Dmin[Dmin[Amaj][Dmin]][Dmin[Amaj][Dmin]]]][Gmaj[Abmaj][Gmaj]]]]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Gmaj[Gmaj[Dmin[Amin[Emaj[Bhdim[Fmaj][Bhdim]][Emaj]][Amin]][Dmin[Ebmaj][Dmin]]][Gmaj]][Gmaj[Dmin[Amin[Emin][Amin]][Dmin]][Gmaj]]][Cmaj]]]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj[Cmaj[Cmaj][Cmaj[Fmaj][Cmaj]]][Gmaj[Dmaj[Amin][Dmaj]][Gmaj[Gmaj][Gmaj]]]][Cmaj[Cmaj[Cmaj][Cmaj[Fmaj][Cmaj]]][Cmaj[Gmaj][Cmaj]]]][Cmaj[Cmaj[Cmaj[Cmaj][Cmaj[Fmaj][Cmaj]]][Gmaj[Dmaj[Amin][Dmaj]][Gmaj[Gmaj][Gmaj]]]][Cmaj[Cmaj[Cmaj][Cmaj[Fmaj][Cmaj]]][Gmaj[Gmaj][Gmaj[Abmaj][Gmaj]]]]]][Cmaj[Cmaj[Cmaj][Cmaj[Fmaj][Cmaj]]][Cmaj[Gmaj[Dmaj[Amin][Dmaj]][Gmaj]][Cmaj]]]]"
    "[Cmin[Cmin[Cmin[Cmin][Cmin[Gmaj][Cmin]]][Gmaj[Abmaj][Gmaj]]][Cmin[Cmin[Cmin][Cmin[Gmaj][Cmin]]][Cmin[Gmaj[Abmaj][Gmaj]][Cmin]]]]"
    "[Cmaj[Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Fmaj][Cmaj]]]][Cmaj[Cmaj[Fmaj][Cmaj]][Cmaj[Gmaj[Dmin[Amaj[Emin][Amaj]][Dmin]][Gmaj]][Cmaj]]]]"
    "[Cmaj[Cmaj[Cmaj][Cmaj[Bbmaj][Cmaj]]][Cmaj[Dbmaj[Dmaj[Ebmaj[Bbmaj][Ebmaj]][Dmaj]][Dbmaj]][Cmaj]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj][Cmaj[Gmaj[Dmaj][Gmaj[Dmin][Gmaj]]][Cmaj]]][Cmaj[Cmaj][Cmaj[Gmaj[Dmaj][Gmaj[Dmin][Gmaj]]][Cmaj]]]][Cmaj[Cmaj[Gmaj[Fmaj][Gmaj[Dmaj][Gmaj[Dmin][Gmaj]]]][Cmaj[Dbmaj][Cmaj]]][Cmaj[Gmaj[Dmaj][Gmaj[Dmin][Gmaj]]][Cmaj[Cmaj][Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]]]]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj][Amin[Emaj[Bhdim][Emaj]][Amin]]][Cmaj[Cmaj[Bbmaj[Fmaj[Cmaj[Gmin][Cmaj]][Fmaj]][Bbmaj]][Cmaj]][Gmaj[Dmaj[Amin][Dmaj]][Gmaj[Dmin][Gmaj]]]]][Cmaj[Cmaj[Cmaj[Cmaj][Amin[Emaj[Bhdim][Emaj]][Amin]]][Cmaj[Bbmaj[Fmaj[Cmaj[Gmin][Cmaj]][Fmaj]][Bbmaj]][Cmaj]]][Cmaj[Gmaj[Dmaj][Gmaj[Dmin[Amaj[Emin[Emin[D#dim][Emin]][Emin[Fmaj][Emin]]][Amaj]][Dmin]][Gmaj]]][Cmaj[Cmaj][Cmaj[Gmaj][Cmaj]]]]]]"
    "[Cmaj[Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Fmin][Gmaj[Dmin][Gmaj[Abmaj[Ebmaj][Abmaj]][Gmaj]]]]]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Cmaj[Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Fmin][Gmaj[Dmin][Gmaj]]]][Cmaj]]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj[Dbmaj][Cmaj[Gaug][Cmaj[Cmaj][Cmaj]]]][Cmaj[Galt[Fmin[Cmaj[Ghdim[Dmaj[Cmaj][Dmaj]][Ghdim]][Cmaj]][Fmin[Fmin][Fmin]]][Galt[Dhdim][Galt]]][Cmaj[Cmaj][Cmaj]]]][Cmaj[Galt[Fmin[Cmaj[Ghdim[Dmaj[Cmaj][Dmaj]][Ghdim]][Cmaj]][Fmin[Fmin][Fmin]]][Galt[Dhdim][Galt]]][Cmaj[Cmaj][Cmaj[Cmaj][Cmaj]]]]][Cmaj[Gmaj[Bbmaj[Fmaj[Cmin[Cmin][Cmin]][Fmaj]][Bbmaj[Bbmaj][Bbmaj]]][Gmaj[Abmaj[Abmaj][Abmaj]][Gmaj[Dmin][Gmaj]]]][Cmaj[Gaug[Fmin[Cmaj[Ghdim][Cmaj]][Fmin[Fmin][Fmin]]][Gaug[Dbmaj][Gaug]]][Cmaj[Cmaj[Cmaj][Cmaj[Cmaj][Cmaj]]][Cmaj[Dmaj][Cmaj]]]]]]"
    "[Cmaj[Cmaj[Cmaj][Cmaj[Cmin[Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Dmin][Gmaj]]][Cmin]][Cmaj[Gmaj[Dhdim][Gmaj]][Cmaj]]]][Cmaj[Cmaj[Gmaj[Gmaj[Dmin][Gmaj]][Gmaj[Dmin[Ebmaj[Emin][Ebmaj]][Dmin]][Gmaj]]][Cmaj]][Cmaj[Cmaj[Gmaj[Abmaj[Ebmaj[Bbmaj[Fmaj[Cmin][Fmaj]][Bbmaj]][Ebmaj[Bbmin][Ebmaj]]][Abmaj]][Gmaj[Dmin][Gmaj]]][Cmaj]][Cmaj[Cmaj[Gmaj[Gmaj[Dmin][Gmaj]][Gmaj[Dmin[Ebmaj[Emin][Ebmaj]][Dmin]][Gmaj]]][Cmaj]][Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]]]]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj][Cmaj[Fmaj][Cmaj]]][Cmaj[Fmaj][Cmaj]]][Cmaj[Gmaj][Cmaj[Fmaj][Cmaj]]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj][Amin[Emaj[Bhdim][Emaj]][Amin]]][Gmaj[Dmaj[Amaj[Bbmaj[Fmaj[Cmaj[Gmin][Cmaj]][Fmaj]][Bbmaj]][Amaj[Emin][Amaj]]][Dmaj]][Gmaj[Dmin][Gmaj]]]][Cmaj[Cmaj[Cmaj][Amin[Emaj[Bhdim][Emaj]][Amin]]][Gmaj[Abmaj[Ebmin[Bbmaj[Fmaj[Cmaj[Gmin][Cmaj]][Fmaj]][Bbmaj]][Ebmin[Emin][Ebmin]]][Abmaj]][Gmaj[Dmin][Gmaj]]]]]"
    "[Cmaj[Cmaj[Cmaj][Cmaj[Bbmaj[Fmin][Bbmaj]][Cmaj]]][Cmaj[Cmaj[Gmaj[Abmaj[Ebmaj[Bbmin][Ebmaj]][Abmaj]][Gmaj[Dmaj[Amin][Dmaj]][Gmaj[Dmin][Gmaj]]]][Cmaj]][Dbmaj[Abmaj[Ebmaj][Abmaj]][Dbmaj]]]]"
    "[Cmaj[Cmaj[Cmaj[Cmaj[Cmaj][Cmaj[Gmaj[Dmin[Amin][Dmin]][Gmaj]][Cmaj]]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Gmaj[Gmaj[Dmin][Gmaj]][Gmaj[Dmin[Amaj[Emin][Amaj]][Dmin]][Gmaj]]]]][Gmaj[Gmaj[Dmaj[Ebmaj[Ebmaj][Ebmaj[Bbmaj[Fmin][Bbmaj]][Ebmaj[Ebmaj][Ebmaj]]]][Dmaj]][Gmaj]][Gmaj[Dmin[Ebmaj[Emin][Ebmaj]][Dmin]][Gmaj]]]][Cmaj[Cmaj[Cmaj[Cmaj][Cmaj[Gmaj[Dmin[Amin][Dmin]][Gmaj]][Cmaj]]][Cmaj[Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]][Gmaj[Gmaj[Dmin][Gmaj]][Gmaj[Dmin[Amaj[Emin][Amaj]][Dmin]][Gmaj]]]]][Cmaj[Cmaj[Dmin[Fmaj[Cmaj[Gmin][Cmaj]][Fmaj]][Dmin]][Cmaj]][Cmaj[Cmaj[Gmaj[Dmin[Amin][Dmin]][Gmaj]][Cmaj]][Cmaj[Gmaj[Dmin][Gmaj]][Cmaj]]]]]]"])
    for i in 1:length(trees)
        trees[i] = map(translate_goldstandard_notation1, trees[i])
    end
    trees
end

function read_parsable_goldstandard_trees_1()
    map(tree, ["[0maj[0maj[0maj[7maj[2min][7maj]][0maj]][0maj[7maj[2min[9maj][2min]][7maj]][0maj]]][0maj[0maj[7maj[2min][7maj]][0maj]][0maj[7maj[8maj[3maj[10maj[5maj[0min][5maj]][10maj]][3maj[10min][3maj]]][8maj]][7maj[2min][7maj]]][0maj]]]]",
    "[0min[0min[0min[0min][0min[7maj][0min]]][7maj[8maj][7maj]]][0min[0min[0min][0min[7maj][0min]]][0min[7maj[8maj][7maj]][0min]]]]",
    "[0maj[0maj[0maj[7maj[2min][7maj]][0maj]][0maj[0maj[7maj[2min][7maj]][0maj]][0maj[5maj][0maj]]]][0maj[0maj[5maj][0maj]][0maj[7maj[2min[9maj[4min][9maj]][2min]][7maj]][0maj]]]]",
    "[0maj[0maj[0maj][0maj[10maj][0maj]]][0maj[1maj[2maj[3maj[10maj][3maj]][2maj]][1maj]][0maj]]]",
    "[0maj[0maj[0maj[0maj][0maj[7maj[2maj][7maj[2min][7maj]]][0maj]]][0maj[0maj][0maj[7maj[2maj][7maj[2min][7maj]]][0maj]]]][0maj[0maj[7maj[5maj][7maj[2maj][7maj[2min][7maj]]]][0maj[1maj][0maj]]][0maj[7maj[2maj][7maj[2min][7maj]]][0maj[0maj][0maj[7maj[2min][7maj]][0maj]]]]]]",
    "[0maj[0maj[0maj][0maj[0min[7maj[8maj[3maj[10maj[5maj[0min][5maj]][10maj]][3maj[10min][3maj]]][8maj]][7maj[2min][7maj]]][0min]][0maj[7maj[2hdim][7maj]][0maj]]]][0maj[0maj[7maj[7maj[2min][7maj]][7maj[2min[3maj[4min][3maj]][2min]][7maj]]][0maj]][0maj[0maj[7maj[8maj[3maj[10maj[5maj[0min][5maj]][10maj]][3maj[10min][3maj]]][8maj]][7maj[2min][7maj]]][0maj]][0maj[0maj[7maj[7maj[2min][7maj]][7maj[2min[3maj[4min][3maj]][2min]][7maj]]][0maj]][0maj[7maj[2min][7maj]][0maj]]]]]]",
    "[0maj[0maj[0maj[0maj][0maj[5maj][0maj]]][0maj[5maj][0maj]]][0maj[7maj][0maj[5maj][0maj]]]]"])
end

function read_goldstandard_trees_2()
    trees = map(tree,
    # ["[C[C[C[C[C[C][A-]][C[Db[Ab][Db]]C]][C[Db[D[Eb][D]]Db][C[C][A-]]]][C[Db[Ab][Db]]C]][C[C[G[F[F[C[G[B][G[Ab][G]]][C]][F[F][D-]]][F[Gb[Db][Gb]][F]]][G[Ab][G]]][C[Db][C[C][A-]]]][C[C[Db[Ab][Db]][C]][C[Db][C[G[Ab][G]][C]]]]]]",
    ["[C-[C-[C-[C-[G[D%][G]][C-]][Eb[Bb[F-[C][F-]][Bb]][Eb]]][C-[C-[G[D%][G]][C-]][C-[G[F-][G[Ab][G]]][C-]]]][C-[G[D%[A][D%]][G]][C-[C-[C-[C-[G[D%][G]][C-]][Eb[Bb[F-[C][F-]][Bb]][Eb]]][C-[C-[G[D%][G]][C-]][C-[G[F-][G[Ab][G]]][C-]]]] [C-[G[Ab[A][Ab]][G]][C-]]]]]",
    "[C-[C-[C-[Eb[Bb[F-][Bb]][Eb]][C-[G[D%[Ab][D%]][G]][C-]]][C-[Eb[Bb[F-[C][F-]][Bb]][Eb]][C-[G[D%[Ab][D%]][G]][C-]]]][C-[C-[C-[G[D%][G]][C-]][Eb[Bb[F-][Bb]][Eb]]][C-[C-[G[D%][G]][C-]][C-[G[D%[Ab[Bbb[Bb-[Cb][Bb-]][Bbb]][Ab]][D%]][G]][C-]]]]]",
    "[C-[C-[C-[C-[C-[C-][C-[G[D%][G]][C-]]][C-[C-[G[D%][G]][C-]][Eb[Bb[F-][Bb]][Eb]]]][C-[Eb[Bb[F-[Eo][F-[C][F-]]][Bb]][Eb]][C-[G[D%[Ab][D%]][G]][C-]]]][C-[C-[C-[G[D%][G]][C-]][C-[G[D%][G]][C-]]][C-[G[F-[C[G[D%][G]][C[G%][C]]][F-]][G[D%[Ab[C-[G[F-][G[D%][G]]][C-]][Ab]][D%]][G]]][C-]]]][C-[C-[C-[F-][C-]][C-[F-][C-]]][C-[G-[F-][G-]][C-]]]]",
    "[C-[C-[C-[C-][C-[G[F-][G[D%][G]]][C-]]][C-[Db[Ab[Eb-][Ab]][Db]][C-[G[D%][G]][C-]]]][C-[G[D%][G]][C-]]]",
    "[C[C[C][C[C[G[F[C[G-[D[A-[E[B%][E]][A-]][D]][G-]][C]][F]][G[Ab[A[Bb[F-][Bb]][A[E-][A]]][Ab[Eb-][Ab]]][G[D-][G]]]][C]][A-]]][C[G[D-][G]][C]]]",
    # "[C[C[C[C][C[Bb[D-[A[Bb][A]][D-]][Bb[F-][Bb]]][C[G[G[D-[A-[E-][A-]][D-]][G]][G[D[A-[E[B%][E]][A-]]D]][G[D-][G]]][C]]]][C[Bb[D-[A[Bb][A]][D-]][Bb[F-][Bb]]][C[G[D-[A-[A-[E-][A-]][A-[E-[F[F#%][F]][E-]][A-]]][D-]][G]][C[C][C[F][C]]]]]][C[G[G[G[D-[A-[E-][A-]][D-]][G]][G[D-[A-[E-][A-]][D-]][G]]][G[D-[A-[E-][A-]][D-]][G]]][C[C][C[F][C]]]]]",
    "[C-[C-[C-][C-[F-][C-]]][C-[Db[D][Db]][C-]]]",
    "[C[C[C[C[G[D-[A-][D-]][G]][C]][A-[E[B%[F][B%]][E]][A-]]][C[C[G[D-[A][D-]][G]][C]][C[G[D-][G]][C]]]][C[C[C[G[D-[A-[E[B-][E]][A-]][D-]][G]][C]][A-[E[B%[F][B%]][E]][A-]]][C[C[G[D-[A][D-]][G]][C]][C[G[F[C][F]][G]][C]]]]]",])
    for i in 1:length(trees)
        # println(i)
        trees[i] = map(translate_goldstandard_notation2, trees[i])
    end
    trees
end

function read_parsable_goldstandard_trees_2()
    map(tree, [
    "[0min[0min[0min[0min[7maj[2hdim][7maj]][0min]][3maj[10maj[5min[0maj][5min]][10maj]][3maj]]][0min[0min[7maj[2hdim][7maj]][0min]][0min[7maj[5min][7maj[8maj][7maj]]][0min]]]][0min[7maj[2hdim[9maj][2hdim]][7maj]][0min[0min[0min[0min[7maj[2hdim][7maj]][0min]][3maj[10maj[5min[0maj][5min]][10maj]][3maj]]][0min[0min[7maj[2hdim][7maj]][0min]][0min[7maj[5min][7maj[8maj][7maj]]][0min]]]][0min[7maj[8maj[9maj][8maj]][7maj]][0min]]]]]",
    "[0min[0min[0min[0min][0min[7maj[5min][7maj[2hdim][7maj]]][0min]]][0min[1maj[8maj[3min][8maj]][1maj]][0min[7maj[2hdim][7maj]][0min]]]][0min[7maj[2hdim][7maj]][0min]]]",
    "[0maj[0maj[0maj][0maj[0maj[7maj[5maj[0maj[7min[2maj[9min[4maj[11hdim][4maj]][9min]][2maj]][7min]][0maj]][5maj]][7maj[8maj[9maj[10maj[5min][10maj]][9maj[4min][9maj]]][8maj[3min][8maj]]][7maj[2min][7maj]]]][0maj]][9min]]][0maj[7maj[2min][7maj]][0maj]]]",
    "[0min[0min[0min][0min[5min][0min]]][0min[1maj[2maj][1maj]][0min]]]",
    "[0maj[0maj[0maj[0maj[7maj[2min[9min][2min]][7maj]][0maj]][9min[4maj[11hdim[5maj][11hdim]][4maj]][9min]]][0maj[0maj[7maj[2min[9maj][2min]][7maj]][0maj]][0maj[7maj[2min][7maj]][0maj]]]][0maj[0maj[0maj[7maj[2min[9min[4maj[11min][4maj]][9min]][2min]][7maj]][0maj]][9min[4maj[11hdim[5maj][11hdim]][4maj]][9min]]][0maj[0maj[7maj[2min[9maj][2min]][7maj]][0maj]][0maj[7maj[5maj[0maj][5maj]][7maj]][0maj]]]]]",
    ])
end

# read_goldstandard_trees_2()

function train!(grammar, trees)
    for tree in trees
        forest = run_chartparser(tree, grammar)
        add_obs!(grammar, best_tree(forest))
    end
    grammar
end

function single_fold_cross_validation(trees, make_grammar=make_jazz_grammar, with_training=true)
    precision_vals = zeros(Float64, length(trees))
    for (i,test_tree) in enumerate(trees)
        grammar = make_grammar()
        if with_training
            train!(grammar, [trees[1:i-1];trees[i+1:end]])
        end
        @time forest  = run_chartparser(leaf_data(test_tree), grammar)
        precision_vals[i] = precision(best_tree(forest), test_tree)
    end
    precision_vals
end

function single_fold_cross_validation_baseline(trees)
    precision_vals = zeros(Float64, length(trees))
    for (i,test_tree) in enumerate(trees)
        baseline_tree = right_branching_tree(length(leafs(test_tree)))
        precision_vals[i] = precision(baseline_tree, test_tree)
    end
    precision_vals
end

# function single_fold_cross_validation(trees)
#     precision_vals = Float64[]
#     recall_vals = Float64[]
#     precision_base = Float64[]
#     recall_base = Float64[]
#
#     for (i,test_tree) in enumerate(trees)
#         # grammar = make_jazz_grammar()
#         grammar = make_plain_PCFG_jazz_grammar()
#
#         # training
#         for tree in [trees[1:i-1];trees[i+1:end]]
#             @time forest = run_chartparser(tree, grammar)
#             add_obs!(grammar, best_tree(forest))
#         end
#
#         # evaluating
#         @time forest  = run_chartparser(leaf_data(test_tree), grammar)
#         parsed_tree   = best_tree(forest)
#         baseline_tree = right_branching_tree(length(leafs(test_tree)))
#         push!(precision_vals, precision(parsed_tree,   test_tree))
#         push!(recall_vals,       recall(parsed_tree,   test_tree))
#         push!(precision_base, precision(baseline_tree, test_tree))
#         push!(recall_base,       recall(baseline_tree, test_tree))
#     end
#     precision_vals, recall_vals, precision_base, recall_base
# end

function parse_leaf_data(trees)
    grammar = make_jazz_grammar(Int)
    # grammar = make_plain_PCFG_jazz_grammar(BigInt)
    for k in 1:2
        println("id\tlength\ttime_in_seconds\t number_of_parses")
        for (i,t) in enumerate(trees)
            chords = leaf_data(t)
            tic()
            forest = run_chartparser(chords, grammar)
            time = toq()
            @assert is_complete(forest)
            println(i, "\t", length(chords), "\t", time, "\t", score(forest))
        end
    end
end

function make_parsable_trees()
    parsable_trees = [
        read_treebank_trees();
        read_parsable_goldstandard_trees_1();
        read_parsable_goldstandard_trees_2()
    ]
    [parsable_trees[1:6]; parsable_trees[8:end]]
end

# parse_leaf_data(parsable_trees)

# p,r,pb,rb = single_fold_cross_validation(parsable_trees)

# parse_trees(parsable_trees)


module olcamat

struct MatrixHeader
    version::Int32
    format::Int32
    dataType::Int32
    typeLength::Int32
    rows::Int32
    columns::Int32
end

struct TechIndexEntry
    process_id::String
    process_name::String
    process_type::String
    process_location::String
    flow_id::String
    flow_name::String
    flow_type::String
    flow_location::String
    flow_property_id::String
    flow_property_name::String
    unit_id::String
    unit_name::String
end

struct EnviIndexEntry
    flow_id::String
    flow_name::String
    flow_type::String
    flow_location::String
    flow_property_id::String
    flow_property_name::String
    unit_id::String
    unit_name::String
end

function read_tech_index(path::String)::Vector{TechIndexEntry}
    stream = open(path)
    records = readdlm(stream, ',', String)
    rows, columns = size(records)
    entries = Vector{TechIndexEntry}(rows - 1)
    for row = 2:rows
        record = records[row, :]
        i = parse(Int, record[1]) + 1
        entries[i] = TechIndexEntry(
            record[2],
            record[3],
            record[4],
            record[5],
            record[6],
            record[7],
            record[8],
            record[9],
            record[10],
            record[11],
            record[12],
            record[13])
    end
    close(stream)
    return entries
end

function read_envi_index(path::String)::Vector{EnviIndexEntry}
    stream = open(path)
    records = readdlm(stream, ',', String)
    rows, columns = size(records)
    entries = Vector{EnviIndexEntry}(rows - 1)
    for row = 2:rows
        record = records[row, :]
        i = parse(Int, record[1]) + 1
        entries[i] = EnviIndexEntry(
            record[2],
            record[3],
            record[4],
            record[5],
            record[6],
            record[7],
            record[8],
            record[9])
    end
    close(stream)
    return entries
end

function read_matrix(path::String)::Array{Float64,2}
    stream = open(path)
    h = read_matrix_header(stream)
    M = Array{Float64,2}(h.rows, h.columns)
    for col = 1:h.columns
        M[:, col] = read(stream, Float64, h.rows)
    end
    close(stream)
    return M
end

function read_matrix_header(stream::IOStream)::MatrixHeader
    h = MatrixHeader(
        read(stream, Int32),
        read(stream, Int32),
        read(stream, Int32),
        read(stream, Int32),
        read(stream, Int32),
        read(stream, Int32)
    )
    return h
end

function read_column(path::String, column::Int64)::Array{Float64,1}
    stream = open(path)
    h = read_matrix_header(stream)
    offset = (column - 1) * h.rows * 8
    skip(stream, offset)
    v = read(stream, Float64, h.rows)
    close(stream)
    return v
end

function write_matrix(M::Array{Float64,2}, path::String)
    stream = open(path, "w")
    # version
    write(stream, Int32(1))
    # format
    write(stream, Int32(0))
    # data type
    write(stream, Int32(0))
    # type length
    write(stream, Int32(8))
    rows, cols = size(M)
    write(stream, Int32(rows))
    write(stream, Int32(cols))
    for col = 1:cols
        for row = 1:rows
            write(stream, Float64(M[row, col]))
        end
    end
    close(stream)
end

end # module


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct VirtualInertia <: ActivePowerControl
        Ta::Float64
        kd::Float64
        kω::Float64
        P_ref::Float64
        ext::Dict{String, Any}
        states::Vector{Symbol}
        n_states::Int64
    end

Parameters of a Virtual Inertia with SRF using VSM for active power controller

# Arguments
- `Ta::Float64`: VSM inertia constant, validation range: (0, nothing)
- `kd::Float64`: VSM damping constant, validation range: (0, nothing)
- `kω::Float64`: frequency droop gain, validation range: (0, nothing)
- `P_ref::Float64`: Reference Power Set-point, validation range: (0, nothing)
- `ext::Dict{String, Any}`
- `states::Vector{Symbol}`
- `n_states::Int64`
"""
mutable struct VirtualInertia <: ActivePowerControl
    "VSM inertia constant"
    Ta::Float64
    "VSM damping constant"
    kd::Float64
    "frequency droop gain"
    kω::Float64
    "Reference Power Set-point"
    P_ref::Float64
    ext::Dict{String, Any}
    states::Vector{Symbol}
    n_states::Int64
end

function VirtualInertia(Ta, kd, kω, P_ref=1.0, ext=Dict{String, Any}(), )
    VirtualInertia(Ta, kd, kω, P_ref, ext, [:ω_oc, :θ_oc], 2, )
end

function VirtualInertia(; Ta, kd, kω, P_ref=1.0, ext=Dict{String, Any}(), )
    VirtualInertia(Ta, kd, kω, P_ref, ext, )
end

# Constructor for demo purposes; non-functional.
function VirtualInertia(::Nothing)
    VirtualInertia(;
        Ta=0,
        kd=0,
        kω=0,
        P_ref=0,
        ext=Dict{String, Any}(),
    )
end

"""Get VirtualInertia Ta."""
get_Ta(value::VirtualInertia) = value.Ta
"""Get VirtualInertia kd."""
get_kd(value::VirtualInertia) = value.kd
"""Get VirtualInertia kω."""
get_kω(value::VirtualInertia) = value.kω
"""Get VirtualInertia P_ref."""
get_P_ref(value::VirtualInertia) = value.P_ref
"""Get VirtualInertia ext."""
get_ext(value::VirtualInertia) = value.ext
"""Get VirtualInertia states."""
get_states(value::VirtualInertia) = value.states
"""Get VirtualInertia n_states."""
get_n_states(value::VirtualInertia) = value.n_states

"""Set VirtualInertia Ta."""
set_Ta!(value::VirtualInertia, val::Float64) = value.Ta = val
"""Set VirtualInertia kd."""
set_kd!(value::VirtualInertia, val::Float64) = value.kd = val
"""Set VirtualInertia kω."""
set_kω!(value::VirtualInertia, val::Float64) = value.kω = val
"""Set VirtualInertia P_ref."""
set_P_ref!(value::VirtualInertia, val::Float64) = value.P_ref = val
"""Set VirtualInertia ext."""
set_ext!(value::VirtualInertia, val::Dict{String, Any}) = value.ext = val
"""Set VirtualInertia states."""
set_states!(value::VirtualInertia, val::Vector{Symbol}) = value.states = val
"""Set VirtualInertia n_states."""
set_n_states!(value::VirtualInertia, val::Int64) = value.n_states = val


# path to the GISSMO Julia storage folder.
#cs_config_path = "/home/roy/MEGAsync/inputs/NMR/configs/reduced_cs_config.txt"

function calibratesolute(project_name, molecule_names, w;
    max_iters = 50000,
    projects_dir = "/home/roy/MEGAsync/outputs/NMR/calibrate/final",
    base_path_JLD = "/home/roy/Documents/repo/NMRData//src/input/molecules",
    cs_config_path = "/home/roy/Documents/repo/NMRData/src/input/reduced_cs_config.txt")

    println("Now on $(project_name)")

    PyPlot.close("all")
    fig_num = 1

    Random.seed!(25)
    PyPlot.matplotlib["rcParams"][:update](["font.size" => 22, "font.family" => "serif"])

    ### user inputs.
    # 0.1% DSS is 0.0046 M = 4.6 mM.
    save_folder_path = joinpath(projects_dir, project_name)
    #@assert 1==2

    #### TODO: load from project name, molecule_names, w from BSON files.
    # find a story to write about.

    # ## reboot. #######
    # project_name = "D-(+)-Glucose-700"
    # molecule_names = ["D-(+)-Glucose"; "DSS"]
    # w = [20.0/4.6; 1.0] # BMRB: DSS is 0.1 % => 4.6 mM

    # project_name = "D-(+)-Glucose-NRC-600"
    # molecule_names = ["D-(+)-Glucose";]
    # w = [1.0; ]

    # project_name = "L-Phenylalanine-700"
    # molecule_names = ["L-Phenylalanine"; "DSS"]
    # w = [20/0.5; 1.0] # BMRB-700 phenylalanine: DSS is 500 micro M.

    # project_name = "L-Glutamine-700"
    # molecule_names = ["L-Glutamine"; "DSS"]
    # w = [20.0/0.5; 1.0] # BMRB: DSS is 500 uM => 0.5 mM

    # project_name = "L-Glutamic acid-700"
    # molecule_names = ["L-Glutamic acid"; "DSS"]
    # w = [20.0/4.6; 1.0] # BMRB: DSS is 0.1% => 4.6 mM

    # project_name = "L-Histidine-700"
    # molecule_names = ["L-Histidine"; "DSS"]
    # w = [20.0/46; 1.0] # BMRB: DSS is 1 % => 46 mM
    #
    # project_name = "L-Isoleucine-700"
    # molecule_names = ["L-Isoleucine"; "DSS"]
    # w = [20.0/46; 1.0] # BMRB: DSS is 1 % => 46 mM

    # project_name = "L-Serine-700"
    # molecule_names = ["L-Serine"; "DSS"]
    # w = [20.0/46; 1.0] # BMRB: DSS is 1 % => 46 mM
    #
    #
    # project_name = "L-Alanine-700"
    # molecule_names = ["L-Alanine"; "DSS"]
    # w = [20.0/0.5; 1.0] # BMRB: DSS is 500uM => 0.5 mM
    #
    # project_name = "L-Threonine-700"
    # molecule_names = ["L-Threonine"; "DSS"]
    # w = [20.0/0.5; 1.0] # BMRB: DSS is 500uM => 0.5 mM
    #
    # project_name = "L-Tryptophan-700"
    # molecule_names = ["L-Tryptophan"; "DSS"]
    # w = [20.0/0.5; 1.0] # BMRB: DSS is 500uM => 0.5 mM
    #
    # project_name = "L-Valine-700"
    # molecule_names = ["L-Valine"; "DSS"]
    # w = [20.0/0.5; 1.0] # BMRB: DSS is 500uM => 0.5 mM

    w = w ./ norm(w) # since the fit data, y, is normalized.

    # proxy-related.
    tol_coherence = 1e-2
    α_relative_threshold = 0.05
    #Δc_partition_radius = 1e-1 # 17 element partition for glucose spin group 2.
    Δc_partition_radius_candidates = [1e-1; 0.3; 0.5; 0.7; 0.8; 0.9]
    #Δc_partition_radius_candidates = [0.9;]
    λ0 = 3.4
    Δcs_max = 0.2 # for proxy.
    κ_λ_lb = 0.5
    κ_λ_ub = 2.5

    # if a Δc_partition_radius candidate gives max partition sizes less than of
    #   equal to this value, then use that candidate.
    early_exit_part_size = 7

    ### end inputs.

    ## load data.
    load_folder_path = joinpath(projects_dir, project_name)
    load_path = joinpath(load_folder_path, "$(project_name).bson")
    dic = BSON.load(load_path)
    s_t = dic[:s_t]
    fs = dic[:fs]
    SW = dic[:SW]
    ν_0ppm = dic[:ν_0ppm]

    # normalize.
    s_t = s_t

    hz2ppmfunc = uu->(uu - ν_0ppm)*SW/fs
    ppm2hzfunc = pp->(ν_0ppm + pp*fs/SW)

    offset_Hz = ν_0ppm - (ppm2hzfunc(0.3)-ppm2hzfunc(0.0))

    N = length(s_t)
    DFT_s = fft(s_t)
    U_DFT, U_y, U_inds = NMRDataSetup.getwraparoundDFTfreqs(N, fs, offset_Hz)

    Z = maximum(abs.(DFT_s))
    y = DFT_s[U_inds] ./ Z

    #
    S_U = DFT_s[U_inds]
    P_y = hz2ppmfunc.(U_y)

    PyPlot.figure(fig_num)
    fig_num += 1

    PyPlot.plot(P_y, real.(S_U), label = "data spectrum")

    PyPlot.legend()
    PyPlot.xlabel("ppm")
    PyPlot.ylabel("real")
    PyPlot.title("data spectra")

    ####### mixture proxy.

    Δcs_max_mixture = collect( Δcs_max for i = 1:length(molecule_names))

    println("Timing: trydiffΔcradius()")
    @time mixture_params, Δc_partition_radius = NMRCalibrate.trydiffΔcradius(Δc_partition_radius_candidates,
        molecule_names, base_path_JLD, Δcs_max_mixture, hz2ppmfunc, ppm2hzfunc,
        fs, SW, λ0, ν_0ppm, early_exit_part_size, Δcs_max, tol_coherence, α_relative_threshold)
    As = mixture_params

    ΩS_ppm = NMRSpectraSimulator.getPsnospininfo(As, hz2ppmfunc)
    ΩS_ppm_sorted = sort(NMRSpectraSimulator.combinevectors(ΩS_ppm))

    println("$(project_name): Partition sizes:")
    display(NMRCalibrate.displaypartitionsizes(As[1]))
    println("Δc_partition_radius = ", Δc_partition_radius)
    println()

    #@assert 1==2

    # u_min = ppm2hzfunc(-0.5)
    # u_max = ppm2hzfunc(3.0)

    u_offset = 0.5
    u_min = ppm2hzfunc(ΩS_ppm_sorted[1] - u_offset)
    u_max = ppm2hzfunc(ΩS_ppm_sorted[end] + u_offset)

    println("Timing: fitproxies!()")
    @time NMRSpectraSimulator.fitproxies!(As;
    #NMRSpectraSimulator.fitproxiessimple!(As;
    κ_λ_lb = κ_λ_lb,
    κ_λ_ub = κ_λ_ub,
    u_min = u_min,
    u_max = u_max,
    Δr = 1.0,
    Δκ_λ = 0.05)

    #
    # BSON.bson("/home/roy/MEGAsync/inputs/NMR/debug/test_As.bson",
    # As = As)
    # @assert 1==33

    ### cost func.
    combinevectors = NMRSpectraSimulator.combinevectors

    Δsys_cs, y_cost_all, U_cost_all, P_cost_all, exp_info, cost_inds,
    cost_inds_set = NMRCalibrate.prepareoptim(cs_config_path, molecule_names, hz2ppmfunc,
    U_y, y, As; region_min_dist = 0.1)

    # visualize cost regions.
    # sort(P_cost_set[1]) # etc..
    P_cost_set = collect( P_y[cost_inds_set[r]] for r = 1:length(cost_inds_set) )

    ### optim all regions. # 900 secs.
    y_cost = y_cost_all
    U_cost = U_cost_all
    P_cost = P_cost_all

    PyPlot.figure(fig_num)
    fig_num += 1

    PyPlot.plot(P_y, real.(y), label = "data spectrum")
    PyPlot.plot(P_cost, real.(y_cost), "^", label = "positions")

    PyPlot.legend()
    PyPlot.xlabel("ppm")
    PyPlot.ylabel("real")
    PyPlot.title("positions against data spectrum, real part")

    #@assert 1==2

    Δ_shifts = NMRSpectraSimulator.combinevectors(Δsys_cs)

    ##### set up updates.

    P = LinRange(hz2ppmfunc(u_min), hz2ppmfunc(u_max), 50000)
    U = ppm2hzfunc.(P)
    #ΩS_ppm = collect( hz2ppmfunc.( NMRSpectraSimulator.combinevectors(A.Ωs) ./ (2*π) ) for A in mixture_params )

    ## parameters that affect qs.
    # A.ss_params.d, A.ss_params.κs_λ, A.ss_params.κs_β
    # A.d_singlets, A.αs_singlets, A.Ωs_singlets, A.β_singlets, A.λ0, A.κs_λ_singlets
    # purposely perturb κ.

    Es = collect( NMRSpectraSimulator.κCompoundFIDType(As[i]) for i = 1:length(As) )

    κ_lb_default = 0.2
    κ_ub_default = 50.0

    #@assert 1==2

    ## fit model.
    println("Timing: calibrateregions()")
    @time cost_inds_set, p_star_set, κ_BLS_set, d_star_set, β_star_set, λ_star_set,
    proxies_set = calibrateregions(y, U_y, P_y, cost_inds_set,
    Δ_shifts, As, fs, SW, w;
    max_iters = max_iters,
    xtol_rel = 1e-7,
    ftol_rel = 1e-12,
    κ_lb_default = κ_lb_default,
    κ_ub_default = κ_ub_default,
    λ_each_lb = 0.9,
    λ_each_ub = 1.1)

    ### save block.

    save_path = joinpath(save_folder_path, "results_full.bson")
    BSON.bson(save_path,
    p_star_set = p_star_set,
    κ_lb_default = κ_lb_default,
    κ_ub_default = κ_ub_default,
    κ_star_set = κ_BLS_set,
    d_star_set = d_star_set,
    β_star_set = β_star_set,
    λ_star_set = λ_star_set,
    w = w,
    # proxy setup-related below.
    Δc_partition_radius = Δc_partition_radius,
    tol_coherence = tol_coherence,
    α_relative_threshold = α_relative_threshold,
    λ0 = λ0,
    Δcs_max = Δcs_max,
    κ_λ_lb = κ_λ_lb,
    κ_λ_ub = κ_λ_ub,
    # experiement/cost-related below.
    cost_inds_set = cost_inds_set,
    Δsys_cs = Δsys_cs,
    y_cost_all = y_cost_all,
    U_cost_all = U_cost_all,
    P_cost_all = P_cost_all,
    #exp_info = exp_info,
    cost_inds = cost_inds,
    As = As,
    y = y,
    U_y = U_y,
    fs = fs,
    SW = SW,
    ν_0ppm = ν_0ppm)
    ## end save block.

    q_U_set = collect( proxies_set[r].(U) for r = 1:length(proxies_set) )

end


const ERFA_DPI = 3.141592653589793
const ERFA_D2PI = 6.283185307179586
const ERFA_DR2D = 57.29577951308232
const ERFA_DD2R = 0.017453292519943295
const ERFA_DR2AS = 206264.80624709636
const ERFA_DAS2R = 4.84813681109536e-6
const ERFA_DS2R = 7.27220521664304e-5
const ERFA_TURNAS = 1.296e6
const ERFA_DMAS2R = ERFA_DAS2R / 1000.0
const ERFA_DTY = 365.242198781
const ERFA_DAYSEC = 86400.0
const ERFA_DJY = 365.25
const ERFA_DJC = 36525.0
const ERFA_DJM = 365250.0
const ERFA_DJ00 = 2.451545e6
const ERFA_DJM0 = 2.4000005e6
const ERFA_DJM00 = 51544.5
const ERFA_DJM77 = 43144.0
const ERFA_TTMTAI = 32.184
const ERFA_DAU = 1.4959787e11
const ERFA_CMPS = 2.99792458e8
const ERFA_AULT = 499.004782
const ERFA_DC = ERFA_DAYSEC / ERFA_AULT
const ERFA_ELG = 6.969290134e-10
const ERFA_ELB = 1.550519768e-8
const ERFA_TDB0 = -6.55e-5
const ERFA_SRS = 1.97412574336e-8

# Skipping MacroDefinition: ERFA_DINT ( A ) ( ( A ) < 0.0 ? ceil ( A ) : floor ( A ) )
# Skipping MacroDefinition: ERFA_DNINT ( A ) ( ( A ) < 0.0 ? ceil ( ( A ) - 0.5 ) : floor ( ( A ) + 0.5 ) )
# Skipping MacroDefinition: ERFA_DSIGN ( A , B ) ( ( B ) < 0.0 ? - fabs ( A ) : fabs ( A ) )
# Skipping MacroDefinition: ERFA_GMAX ( A , B ) ( ( ( A ) > ( B ) ) ? ( A ) : ( B ) )
# Skipping MacroDefinition: ERFA_GMIN ( A , B ) ( ( ( A ) < ( B ) ) ? ( A ) : ( B ) )

const ERFA_WGS84 = 1
const ERFA_GRS80 = 2
const ERFA_WGS72 = 3

immutable Array_3_Cdouble
    d1::Cdouble
    d2::Cdouble
    d3::Cdouble
end

zero(::Type{Array_3_Cdouble}) = Array_3_Cdouble(fill(zero(Cdouble),3)...)

immutable Array_3_Array_3_Cdouble
    d1::Array_3_Cdouble
    d2::Array_3_Cdouble
    d3::Array_3_Cdouble
end

zero(::Type{Array_3_Array_3_Cdouble}) = Array_3_Array_3_Cdouble(fill(zero(Array_3_Cdouble),3)...)

type eraASTROM
    pmt::Cdouble
    eb::Array_3_Cdouble
    eh::Array_3_Cdouble
    em::Cdouble
    v::Array_3_Cdouble
    bm1::Cdouble
    bpn::Array_3_Array_3_Cdouble
    along::Cdouble
    phi::Cdouble
    xpl::Cdouble
    ypl::Cdouble
    sphi::Cdouble
    cphi::Cdouble
    diurab::Cdouble
    eral::Cdouble
    refa::Cdouble
    refb::Cdouble
end

immutable Array_2_Array_3_Cdouble
    d1::Array_3_Cdouble
    d2::Array_3_Cdouble
end

zero(::Type{Array_2_Array_3_Cdouble}) = Array_2_Array_3_Cdouble(fill(zero(Array_3_Cdouble),2)...)

immutable eraLDBODY
    bm::Cdouble
    dl::Cdouble
    pv::Array_2_Array_3_Cdouble
end

immutable Array_4_Cint
    d1::Cint
    d2::Cint
    d3::Cint
    d4::Cint
end

zero(::Type{Array_4_Cint}) = Array_4_Cint(fill(zero(Cint),4)...)

immutable Array_2_Cdouble
    d1::Cdouble
    d2::Cdouble
end

zero(::Type{Array_2_Cdouble}) = Array_2_Cdouble(fill(zero(Cdouble),2)...)


module TestRegInfo

using MLStyle
using YaoLocations
using YaoCompiler
using YaoCompiler.Intrinsics
using YaoTargetQASM
using CompilerPluginTools
using Test

@device function test_basic(theta, phi)
    # syntax sugar
    1 => X
    @gate 2 => Z
    @ctrl 1 4 => Rx(theta)
    @ctrl 2 4 => Ry(phi)
    a = @measure 3
    # direct intrinsic
    apply(Y, 3)
    apply(X, 1, 4)
    c = measure((2, 3))
    return (a = a, b = c)
end

@device function test_pure_quantum()
    ret = @gate 1:4 => test_basic(1.0, 2.0)
    @ctrl 2 1 => Rx(2.2)
    return ret
end

@testset "RegInfo" begin
    op = test_pure_quantum()
    interp = YaoInterpreter()
    ci, type = code_typed(Intrinsics.apply, (AnyReg, typeof(op)); interp)[1]
    ci = CompilerPluginTools.rm_code_coverage_effect(ci)
    info = RegInfo(OpenQASMTarget(), ci)

    @test info.creg_ssa_map[:a] == 5
    @test info.creg_ssa_map[:b] == 8
    @test info.ssa_creg_map[5] === :a
    @test info.ssa_creg_map[8] === :b
    @test info.creg_size[:a] == 1
    @test info.creg_size[:b] == 2
    @test info.qreg_to_locs[1] == [1, 4]
    @test info.qreg_to_locs[2] == [2, 3]
    @test info.locs_to_qreg[1] == 1
    @test info.locs_to_qreg[2] == 2
    @test info.locs_to_qreg[3] == 2
    @test info.locs_to_qreg[4] == 1
    @test info.locs_to_addr[1] == 0
    @test info.locs_to_addr[2] == 0
    @test info.locs_to_addr[3] == 1
    @test info.locs_to_addr[4] == 1

    buf = IOBuffer()
    show(buf, MIME"text/plain"(), info)
    s = String(take!(buf))
    @test occursin("RegInfo\n", s)
    @test occursin("creg:\n", s)
    @test occursin("  a[1]\n", s)
    @test occursin("  b[2]\n", s)
    @test occursin("qreg:\n", s)
    @test occursin("  %2[2] => [2, 3]", s)
    @test occursin("  %1[2] => [1, 4]", s)
end

end


using Parameters, LinearAlgebra

# export System, Reaction, Reactions, Compound, Compounds, IDs

@with_kw struct Reaction
    id::String
    left::Vector{String}
    right::Vector{String}
    metadata::Dict{String}=Dict{String,Any}()
end

@with_kw struct Compound
    entry_id::String
    name::String
    formula::String
    exact_mass::Float64
    mol_weight::Float64
    names::Vector{String}=Vector{String}()
    elements::Vector{String}=Vector{String}()
    reactions::Vector{String}=Vector{String}()
    comment::String=""
    enzymes::Vector{String}=Vector{String}()
    rpairs::Vector{String}=Vector{String}()
    dblinks::Dict{String,Vector{String}}=Dict{String,Vector{String}}()
    pathways::Dict{String,String}=Dict{String,String}()
    kcf::String=""
    remark::String=""
    glycans::Vector{String}=Vector{String}()
end

#consts basically will just check type, but wont actually keep constant
const IDs = Vector{String} #either "C00000" or "R00000"

const Reactions = Dict{String,Reaction}

const Compounds = Dict{String,Compound}

# This is just cosmetic
const TArray{T, N} = Transpose{T, Array{T, N}}

@with_kw struct System
    rstructs::Reactions
    rids::IDs
    sids::IDs
    tids::IDs = IDs()
    allowed_forward::Union{Vector{Bool},Nothing} = nothing
    allowed_backward::Union{Vector{Bool},Nothing} = nothing
    # function System(args...)

    #     new(rstructs, rids, sids, nothing, nothing, nothing)
    # end
end

struct Expansion
    x::Vector{Int}
    t::Vector{Int}
    compounds::IDs
    reactions::IDs
    X::Vector{Vector{Int}}
    Y::Vector{Vector{Int}}
end


using Flux, RoME, RoMEPlotting
using ImageMagick, Images, ImageView, ImageDraw
using Colors
using JSON2

using FreeTypeAbstraction, AprilTags

function imdrawline!(img, col; color=RGB(1.0,0.0,0.0) )
  for i in 1:size(img, 1)
    img[i, col] = color
  end
end

cd("/media/dehann/logs3/caesar/2020-08-19T11:45:38.074")

fg = loadDFG("fg_final_resolve.tar.gz")

# wire up the blob store
getSolverParams(fg).logpath = pwd()
storeDir = joinLogPath(fg,"data")
mkpath(storeDir)
datastore = FolderStore{Vector{UInt8}}(:default_folder_store, storeDir)
addBlobStore!(fg, datastore)

# Retrieve camera calibration
listDataEntries(fg, :x0)
rD = getData(fg, :x0, :camCalib)
camCalib = JSON2.read(IOBuffer(rD[2]))
K = reshape(camCalib[:vecK], camCalib[:size]...) .|> Float64

# get image from x1
imgE ,imgD = getData(fg, :x1, :KEYFRAME_IMG)
img = ImageMagick.readblob(imgD);
# check that we have the image
imshow(img)

# detect and draw tags
detector = AprilTagDetector()
tags = detector(img)

img_ = drawTags(img,K, tags)
imshow(img_)

ls(fg, :x1)

x1l2f1 = getFactorType(fg, :x1l2f1)

x1l2f1.z.μ

function calcBearingRange(dfg::AbstractDFG, from::Symbol, to::Symbol)
  getSofttype(dfg, from)
end

measimgcol = round(Int,camc-camf*tan(getfnctype(ve).bearing.μ)) # measurement center
imdrawline!(img, measimgcol)

# deptdist = Base.mean(depthcloud[260:270,measimgcol,3])
# push!(DISTS[fct], deptdist)


using Random
import SharedArrays
import Base.Threads.@threads

function execute!(model::GebModel)
    # log births of the initial organisms in the first execution step
    if model.time == 0
        for organism in model.organisms
            log_birth(model.logger, model, organism)
        end
    end

    shuffle(model.organisms)
    organisms_in_batch = [(i, o) for (i, o) in enumerate(model.organisms)]

    activations_in_batch = SharedArrays.SharedArray{Float64}(length(organisms_in_batch), 4*MAX_NEURONS)
    activation_size_in_batch = SharedArrays.SharedArray{UInt8}(length(organisms_in_batch))

    for i in 1:length(organisms_in_batch)
        for j in 1:MAX_NEURONS
            activations_in_batch[i, j] = 0.0
        end
        activation_size_in_batch[i] = 0
    end

    for (i, organism) in organisms_in_batch
        develop_nodes!(organism.network, organism.rules)
    end

    for (i, organism) in organisms_in_batch
        activations = determine_input_activations(model, organism)
        num_outputs = length(activations)

        activations_in_batch[i, 1:num_outputs] = activations
        activation_size_in_batch[i] = num_outputs
    end

    for (i, organism) in organisms_in_batch
        fire!(organism.network, activations_in_batch[i, 1:activation_size_in_batch[i]])
    end

    for (_, organism) in organisms_in_batch
        if !(organism in model.organisms) continue end

        perform!(model, organism)
        organism.age += 1
    end

    model.time += 1
    log_step(model.logger, model)
end


# This file is a part of SimilaritySearch.jl

export allknn

"""
    allknn(g::AbstractSearchContext, k::Integer; parallel=false, pools=getpools(g)) -> knns, dists

Computes all the k nearest neighbors (all vs all) using the index `g`. It removes self references.

Parameters:

- `g`: the index
- `k`: the number of neighbors to retrieve
- `parallel`: If true, the construction will use all threads available threads
- `pools`: A pools object, dependent of `g`

Returns:

- `knns` a (k, n) matrix of identifiers; the i-th column corresponds to the i-th object in the dataset.
    Zeros can happen to the end of each column meaning that the retrieval was less than the desired `k`
- `dists` a (k, n) matrix of distances; the i-th column corresponds to the i-th object in the dataset.
    Zero values in `knns` should be ignored in `dists`

"""
function allknn(g::AbstractSearchContext, k::Integer; parallel=false, pools=getpools(g))
    n = length(g)
    knns = zeros(Int32, k, n)
    dists = Matrix{Float32}(undef, k, n)
    allknn(g, knns, dists; parallel, pools)
end

function _allknn_loop(g::SearchGraph, i, knns, dists, pools)
    k = size(knns, 1) + 1
    res = getknnresult(k, pools)
    vstate = getvstate(length(g), pools)
    # the loop helps to overcome when the current nn is in a small clique (smaller the the desired k)
    for h in g.links[i] # hints
        visited(vstate, h) && continue
        @inbounds search(g.search_algo, g, g[i], res, h, pools; vstate)
        length(res) == k && break
    end

    # again for the same issue
    if length(res) < k
        for j in 1:2k
            h = rand(1:length(g))
            visited(vstate, h) && continue
            @inbounds search(g.search_algo, g, g[i], res, h, pools; vstate)
            length(res) == k && break
        end
    end

    _allknn_inner_loop(res, i, knns, dists)
end

function _allknn_loop(g, i, knns, dists, pools)
    k = size(knns, 1) + 1
    res = getknnresult(k, pools)
    @inbounds search(g, g[i], res)
    _allknn_inner_loop(res, i, knns, dists)
end

@inline function _allknn_inner_loop(res, i,  knns, dists)
    j = 0
    @inbounds for (id, dist) in res
        i == id && continue
        j += 1
        knns[j, i] = id
        dists[j, i] = dist
    end
end

"""
allknn(g, knns, dists; parallel=false, pools=getpools(g)) -> knns, dists

Computes all the k nearest neighbors (all vs all) using the index `g`. It removes self references.

Arguments:

- `g`: the index
- `knns`: an uninitialized integer matrix of (k, n) size for storing the `k` nearest neighbors of the `n` elements
- `dists`: an uninitialized floating point matrix of (k, n) size for storing the `k` nearest distances of the `n` elements
- `parallel`: If true, the construction will use all threads available threads
- `pools`: A pools object, dependent of `g`

Results:

- `knns` and `dists` are returned. Note that the index can retrieve less than `k` objects, and these are represented as zeros at the end of each column (can happen)
"""
function allknn(g::AbstractSearchContext, knns::AbstractMatrix{Int32}, dists::AbstractMatrix{Float32}; parallel=false, pools=getpools(g))
    n = length(g)
    @assert n > 0

    if parallel
        Threads.@threads for i in 1:n
            _allknn_loop(g, i, knns, dists, pools)
        end
    else
        for i in 1:n
            _allknn_loop(g, i, knns, dists, pools)
        end
    end

    knns, dists
end


using Distributed
using Dates
# addprocs(6) # Call instead with: julia -O3 -p18
using RoME
using RoMEPlotting
using Gadfly
@everywhere using RoME, RoMEPlotting, Gadfly

# Parse the arguments.
initial_offset = parse(Int, ARGS[1])
final_timestep = parse(Int, ARGS[2])
qfl_length = parse(Int, ARGS[3])

# Let's load the Manhattan scenario using the g2o file.
file = (normpath(Base.find_package("RoME"), "../..", "examples", "manhattan_incremental.g2o"))
global instructions = importG2o(file)

# Make sure plots look a bit nicer.
latex_fonts = Theme(major_label_font="CMU Serif", major_label_font_size=16pt,
                    minor_label_font="CMU Serif", minor_label_font_size=14pt,
                    key_title_font="CMU Serif", key_title_font_size=12pt,
                    key_label_font="CMU Serif", key_label_font_size=10pt)
Gadfly.push_theme(latex_fonts)

function go_fixedlag(initial_offset::Integer, final_timestep::Integer, qfl_length_arg::Integer)
    # Choose where to save the step's data.
    qfl_length = qfl_length_arg # Fixed lag window size.
    data_logpath = "/media/data2/tonio_results/manhattan-margONLY-qfl$(qfl_length)-$(now())"

    # Create initial factor graph with specified logging path.
    fg = LightDFG{SolverParams}(solverParams=SolverParams(logpath=data_logpath))
    tree = emptyBayesTree()

    # Set up the fixed lag smoothing.
    getSolverParams(fg).isfixedlag = true
    getSolverParams(fg).qfl = qfl_length
    getSolverParams(fg).limitfixeddown = true
    getSolverParams(fg).dbg = true

    # Add initial variable with a prior measurement to anchor the graph.
    addVariable!(fg, :x0, Pose2)
    initial_pose = MvNormal([0.0; 0.0; 0.0], Matrix(Diagonal([0.1;0.1;0.05].^2)))
    addFactor!(fg, [:x0], PriorPose2(initial_pose))

    # Add the next---or initial offset of---measurements to the graph.
    padded_step = lpad(1, 4, "0")
    if initial_offset == 1
        parseG2oInstruction!(fg, instructions[1])
    else
        for j in 1:initial_offset
            parseG2oInstruction!(fg, instructions[j])
        end
        padded_step = lpad(initial_offset, 4, "0")
    end

    # Solve the graph, and save a copy of the tree.
    saveDFG(fg, "$(getLogPath(fg))/fg-before-solve$(padded_step)")
    getSolverParams(fg).maxincidence = 1000
    tree, smt, hist = solveTree!(fg)
    saveDFG(fg, "$(getLogPath(fg))/fg-after-solve$(padded_step)")
    saveTree(tree, "$(getLogPath(fg))/tree$(padded_step).jld2")
    drawTree(tree, show=false, filepath="$(getLogPath(fg))/bt$(padded_step).pdf")

    # Analyze clique counts.
    fid = open("$(getLogPath(fg))/clique-counts.txt", "w")
    nMarg, nReused = calcCliquesRecycled(tree)
    nCliqs = length(tree.cliques)
    println(fid, "$(padded_step), $(nCliqs), $(nMarg), $(nReused)")

    # Just store some quick plots.
    pl1 = drawPoses(fg, spscale=0.6)
    Gadfly.draw(PDF("$(getLogPath(fg))/poses$(padded_step).pdf", 20cm, 10cm), pl1)

    # Solver stride.
    solveStride = 0
    # Run the loop for the remaining time steps.
    for i in (initial_offset + 1):final_timestep
        # Add the next measurement to the graph.
        parseG2oInstruction!(fg, instructions[i])
        padded_step = lpad(i, 4, "0")

        # Store each graph.
        saveDFG(fg, "$(getLogPath(fg))/fg-before-solve$(padded_step)")

        # Just store some quick plots, on another process
        remotecall((fgl, padded_stepl) -> begin
          @info "drawPoses, $(padded_stepl), for fg num variables=$(length(ls(fgl)))."
          pl1 = plotSLAM2DPoses(fgl, spscale=0.6, lbls=false)
          pl1 |> PDF("$(getLogPath(fgl))/poses$(padded_stepl).pdf", 20cm, 10cm)
        end, rand(Categorical(nprocs()-1))+1, fg, padded_step)

        # Only solve every 10th instruction.
        solveStride += 1
        if solveStride % 10 != 0
            @info "solveStride=$solveStride"
            continue
        end
        @info "Going for solve"

        # Solve the graph, and save a copy of the tree.
        getSolverParams(fg).maxincidence = 1000
        tree, smt, hist = solveTree!(fg)
        saveDFG(fg, "$(getLogPath(fg))/fg-after-solve$(padded_step)")
        saveTree(tree, "$(getLogPath(fg))/tree$(padded_step).jld2")
        drawTree(tree, show=false, filepath="$(getLogPath(fg))/bt$(padded_step).pdf")

        # Analyze clique number.
        nMarg, nReused = calcCliquesRecycled(tree)
        nCliqs = length(tree.cliques)
        println(fid, "$(padded_step), $(nCliqs), $(nMarg), $(nReused)")
        flush(fid)

        # Run the garbage collector.
        GC.gc()
    end
    # Final plot.
    padded_step = lpad(final_timestep+1, 4, "0")
    pl1 = drawPoses(fg, spscale=0.6)
    pl1 |> PDF("$(getLogPath(fg))/poses$(padded_step).pdf", 20cm, 10cm)

    close(fid)
end

# Run within a function to avoid undefined variable errors.
go_fixedlag(initial_offset, final_timestep, qfl_length)


# some convenient type-piracy for scalars
AD.gradient(ad::AD.AbstractBackend, f, x::Real) = AD.derivative(ad, f, x)
AD.hessian(ad::AD.AbstractBackend, f, x::Real) = first.(AD.hessian(ad, f∘first, [x]))

# ForwardDiffBackend is built-in ot AutomaticDifferentiation
@init @require ForwardDiff="f6369f11-7733-5829-9624-2563aa707210" begin
    @eval const ForwardDiffBackend = AD.ForwardDiffBackend
end

# Zygote not yet in AutomaticDifferentiation, but remove this once it is
@init @require Zygote="e88e6eb3-aa80-5325-afca-941959d7151f" begin
    struct ZygoteBackend <: AD.AbstractBackend end
    AD.gradient(ad::ZygoteBackend, f, x) = Zygote.gradient(f, x)
    function AD.value_and_gradient(::ZygoteBackend, f, x)
        y, back = Zygote.pullback(f, x)
        y, back(Zygote.sensitivity(y))
    end
end


# This file is a part of ForwardDiffPullbacks.jl, licensed under the MIT License (MIT).

__precompile__(true)

"""
    ForwardDiffPullbacks

ChainRulesCore compatible pullbacks using ForwardDiff.
"""
module ForwardDiffPullbacks

using LinearAlgebra

import ChainRulesCore
import ForwardDiff
import StaticArrays

using ChainRulesCore: AbstractTangent, Tangent, NoTangent, ZeroTangent, AbstractThunk, unthunk

# using Requires

include("dual_numbers.jl")
include("chainrules_types.jl")
include("fwd_back.jl")
include("broadcasting.jl")
include("with_forwarddiff.jl")
include("rrules.jl")

function __init__()
    # Possible extensions:
    # @require Nabla = "49c96f43-aa6d-5a04-a506-44c7070ebe78" include("nabla_support.jl")
    # @require ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" include("reversediff_support.jl")
    # @require Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" include("tracker_support.jl")
end

end # module


"""
    SearchMetrics

In order to have insights about what is happening during the search, this structure stores a lot
of information that describe it. Those informations can eather be used to understand what's happening
but also to design the rewards that will be given to the agent.

Do not hesitate to open github issues to ask for other metrics that could be useful for your current research !

Significations:

total_steps::Int64 -> number of removing, assignment & backtrack !
total_states::Int64 -> number of differents states we've been in (removing & assignment only)
total_decisions::Int64
last_backtrack::Int64 -> steps since last backtrack
last_unfeasible::Int64 -> steps since last unfeasible state
last_foundsolution::Int64 -> steps since last found solution
last_feasible::Int64 -> step since a feasible state was reached
backtrack_length::Int64 -> number of steps of the current backtrack or 0
tree_depth::Int64 -> depth in the current research tree
variable_domain_size
new_variable_domain_size
total_boundvariables
new_total_boundvariables
domains_product
new_domains_product
nb_solutions
current_best::Union{Nothing, Int64} -> best solution found so far
true_best::Union{Nothing, Int64} -> eventually true best of the problem
"""

mutable struct SearchMetrics
    total_steps::Int64
    total_states::Int64
    total_decisions::Int64
    total_backtrack::Int64
    last_backtrack::Int64
    last_unfeasible::Int64
    last_foundsolution::Int64
    last_feasible::Int64
    backtrack_length::Int64
    #tree_depth::Int64
    variable_domain_size::Union{Nothing, Int64}
    new_variable_domain_size::Union{Nothing, Int64}
    total_boundvariables::Union{Nothing, Int64}
    new_total_boundvariables::Union{Nothing, Int64}
    domains_product::Union{Nothing, Int64}
    new_domains_product::Union{Nothing, Int64}
    nb_solutions::Int64
    current_best::Union{Nothing, Int64}
    true_best::Union{Nothing, Int64}
end

SearchMetrics() = SearchMetrics(0, 1, 0, 0, 0, 0, 0, 0, 0, nothing, nothing, 0, nothing, 0, nothing, 0, nothing, nothing)

"""
    SearchMetrics(model::CPModel)

Create a SearchMetrics instance initialized thanks to a CPModel.
"""
function SearchMetrics(model::CPModel)
    total_boundvariables = nb_boundvariables(model)
    domains_product = domains_cartesian_product(model)
    SearchMetrics(0, 1, 0, 0, 0, 0, 0, 0, 0, nothing, nothing, total_boundvariables, nothing, domains_product, nothing, 0, nothing, nothing)
end

"""
    set_metrics!(::StepPhase, search_metrics::SearchMetrics, model::CPModel, symbol::Union{Nothing, Symbol}, x::Union{Nothing, AbstractIntVar})

Set the search metrics thanks to informations from the CPModel and the current status during the StepPhase.
"""
function set_metrics!(::Type{StepPhase}, search_metrics::SearchMetrics, model::CPModel, symbol::Union{Nothing, Symbol} )
    search_metrics.total_steps += 1
    search_metrics.total_states += 1
    search_metrics.last_backtrack += 1
    search_metrics.last_unfeasible += 1
    search_metrics.last_foundsolution += 1
    search_metrics.last_feasible += 1

    if symbol == :Infeasible
        search_metrics.last_unfeasible = 1
        search_metrics.backtrack_length = 0
        #search_metrics.tree_depth += 1
    elseif symbol == :FoundSolution
        search_metrics.total_steps -= 1
        search_metrics.total_states -= 1
        search_metrics.last_backtrack -= 1
        search_metrics.last_unfeasible -= 1
        search_metrics.last_foundsolution = 1
        search_metrics.last_feasible = 1
        search_metrics.backtrack_length = 0
        search_metrics.nb_solutions += 1
    elseif symbol == :Feasible
        search_metrics.last_feasible = 1
        search_metrics.backtrack_length = 0
        #search_metrics.tree_depth += 1
    elseif symbol == :BackTracking
        search_metrics.total_states -= 1
        search_metrics.total_backtrack += 1
        search_metrics.last_backtrack = 1
        search_metrics.backtrack_length += 1
        #search_metrics.tree_depth -= 1
    end

    if !isnothing(model.objectiveBound)
        search_metrics.current_best = model.objectiveBound + 1
    end

    search_metrics.total_boundvariables = nb_boundvariables(model)
    # couldn't it be improved with CPModification ?
    search_metrics.domains_product = domains_cartesian_product(model)

    nothing
end

"""
    set_metrics!(::DecisionPhase, search_metrics::SearchMetrics, model::CPModel, symbol::Union{Nothing, Symbol}, x::Union{Nothing, AbstractIntVar})

Set the search metrics thanks to informations from the CPModel and the current status during the DecisionPhase
"""
function set_metrics!(::Type{DecisionPhase}, search_metrics::SearchMetrics, model::CPModel, x::Union{Nothing, AbstractIntVar})
    search_metrics.total_decisions += 1
    search_metrics.domains_product = search_metrics.new_domains_product
    search_metrics.new_domains_product = domains_cartesian_product(model)
    search_metrics.total_boundvariables = search_metrics.new_total_boundvariables
    search_metrics.new_total_boundvariables = nb_boundvariables(model)
    search_metrics.variable_domain_size = search_metrics.new_variable_domain_size
    search_metrics.new_variable_domain_size = length(x.domain)
end


function reshop_set_mathprgm_modeltype(m::ReSHOPMathProgBaseModel, idx)
    error("to implement")
#    discrete = any((m.vartypes .== :Int) + (m.vartypes .== :Bin) .> 0)
#    if discrete
#        if m.model_type == qcp
#            m.model_type = miqcp
#        elseif m.model_type == nlp
#             m.model_type = minlp
#        end
#    end
end

function reshop_declare_mathprgm(mp, mdl::Ptr{reshop_model}, NLoffset::Int)

    reshop_mp = emp_mp_alloc(mdl)

    CONFIG[:debug] && println("DEBUG: reshop_declare_mathprgm: tackling MP sense = $(mp.sense) objequ = $(mp.objequ) equs = $(mp.equs) vars = $(mp.vars)")

    (objequ, isNL) = mp.objequ
    if objequ > 0 || mp.objvar > 0
        typ = 0

        emp_mp_start(reshop_mp, typ)
        emp_mp_objdir(reshop_mp, mp.sense)
        if (objequ <= 0)
            error("wrong objective equation value $(mp.objequ)")
        end

        if isNL
            emp_mp_objequ(reshop_mp, objequ-1 + NLoffset)
        else
            emp_mp_objequ(reshop_mp, objequ-1)
        end

        emp_mp_objvar(reshop_mp, mp.objvar-1)

        for (eidx, isnl) in mp.equs
            if (eidx, isnl) != mp.objequ
                if isnl
                    emp_mp_constraint(reshop_mp, eidx-1+NLoffset)
                else
                    emp_mp_constraint(reshop_mp, eidx-1)
                end
            end
        end

        for vidx in mp.vars
            emp_mp_var(reshop_mp, vidx-1)
        end
    else
        typ = 2

        emp_mp_start(reshop_mp, typ)
        VIvars = keys(mp.matching)
        # TODO this should be an array of booleans ... --xhub
        equ_seen = fill((-1,false), length(mp.equs))
        sidx = 1
        for vidx in mp.vars
            if vidx in VIvars
                (eidx, isnl) = mp.matching[vidx]
                if isnl
                    emp_mp_vipair(reshop_mp, eidx-1+NLoffset, vidx-1)
                else
                    emp_mp_vipair(reshop_mp, eidx-1, vidx-1)
                end
                equ_seen[sidx] = (eidx, isnl)
                sidx += 1
                CONFIG[:debug] && println("DEBUG: eqn $eidx perp x[$vidx]")
            else
                # QVI case
                emp_mp_var(reshop_mp, vidx-1)
            end
        end

        for (eidx, isnl) in mp.equs
            if !((eidx, isnl) in equ_seen)
                CONFIG[:debug] && println("DEBUG: eqn $eidx is a constraint")
                if isnL
                    emp_mp_constraint(reshop_mp, eidx-1+NLoffset)
                else
                    emp_mp_constraint(reshop_mp, eidx-1)
                end
            end
        end
    end

    return reshop_mp
end