struct HighsModel
   colcost
   collower
   colupper
   rowlower
   rowupper
   astart
   aindex
   avalue
end

struct HighsSolution
   colvalue
   coldual
   rowvalue
   rowdual
end

struct HighsBasis
   colbasisstatus
   rowbasisstatus
end

function Highs_call(model)
   n_col = convert(Cint, size(model.colcost, 1))
   n_row = convert(Cint, size(model.rowlower, 1))
   n_nz = convert(Cint, size(model.aindex, 1))

   colcost = convert(Array{Cdouble}, model.colcost)
   collower = convert(Array{Cdouble}, model.collower)
   colupper = convert(Array{Cdouble}, model.colupper)

   rowlower = convert(Array{Cdouble}, model.rowlower)
   rowupper = convert(Array{Cdouble}, model.rowupper)
   matstart = convert(Array{Cint}, model.astart)
   matindex = convert(Array{Cint}, model.aindex)
   matvalue = convert(Array{Cdouble}, model.avalue)

   solution = HighsSolution(Array{Cdouble, 1}(undef, n_col), Array{Cdouble, 1}(undef, n_col), Array{Cdouble, 1}(undef, n_row),  Array{Cdouble, 1}(undef, n_row))
   basis = HighsBasis(Array{Cint, 1}(undef, n_col), Array{Cint, 1}(undef, n_row))

   modelstatus = convert(Cint, 0)

   status = ccall((:Highs_call, "libhighs.so"), Cint, (Cint, Cint, Cint, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cint}, Ptr{Cint}, Ref{Cint}),
   n_col, n_row, n_nz, colcost, collower, colupper, rowlower, rowupper, matstart, matindex, matvalue, solution.colvalue, solution.coldual, solution.rowvalue, solution.rowdual, basis.colbasisstatus, basis.rowbasisstatus, modelstatus)

   return status, solution, basis, modelstatus
end

function Highs_create()
   return ccall((:Highs_create, "libhighs.so"), Ptr{Cvoid}, () )
end

function Highs_destroy(highs)
   ccall((:Highs_destroy, "libhighs.so"), Cvoid, (Ptr{Cvoid},), highs)
end

function Highs_run(highs)
   return ccall((:Highs_run, "libhighs.so"), Cint, (Ptr{Cvoid},), highs)
end

function Highs_readModel(highs, filename)
   name = convert(Cstring, pointer(filename))
   return ccall((:Highs_readModel, "libhighs.so"), Cint, (Ptr{Cvoid}, Cstring), highs, name)
end

function Highs_writeModel(highs, filename)
   name = convert(Cstring, pointer(filename))
   return ccall((:Highs_writeModel, "libhighs.so"), Cint, (Ptr{Cvoid}, Cstring), highs, name)
end

function Highs_passLp(highs, model)
   n_col = convert(Cint, size(model.colcost, 1))
   n_row = convert(Cint, size(model.rowlower, 1))
   n_nz = convert(Cint, size(model.aindex, 1))

   colcost = convert(Array{Cdouble}, model.colcost)
   collower = convert(Array{Cdouble}, model.collower)
   colupper = convert(Array{Cdouble}, model.colupper)

   rowlower = convert(Array{Cdouble}, model.rowlower)
   rowupper = convert(Array{Cdouble}, model.rowupper)
   matstart = convert(Array{Cint}, model.astart)
   matindex = convert(Array{Cint}, model.aindex)
   matvalue = convert(Array{Cdouble}, model.avalue)

   return ccall((:Highs_passLp, "libhighs.so"), Cint, (Ptr{Cvoid},Cint, Cint, Cint, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cint},Ptr{Cint},Ptr{Cdouble}),
   highs, n_col, n_row, n_nz, colcost, collower, colupper, rowlower, rowupper, matstart, matindex, matvalue)
end

function Highs_setOptionValue(highs, option, value)
   opt = convert(Cstring, pointer(option))
   val = convert(Cstring, pointer(val))
   return ccall((:Highs_setOptionValue, "libhighs.so"), Cint, (Ptr{Cvoid}, Cstring, Cstring), highs, opt, val)
end

function Highs_getNumCols(highs)
   return ccall((:Highs_getNumCols, "libhighs.so"), Cint, ())
end

function Highs_getNumRows(highs)
   return ccall((:Highs_getNumRows, "libhighs.so"), Cint, ())
end

function Highs_getNumNz(highs)
   return ccall((:Highs_getNumNz, "libhighs.so"), Cint, ())
end

function Highs_getSolution(highs)
   numcol = Highs_getNumCols(highs)
   numrow = Highs_getNumRows(highs)

   solution = HighsSolution(Array{Cdouble, 1}(undef, numcol), Array{Cdouble, 1}(undef, numcol), Array{Cdouble, 1}(undef, numrow),  Array{Cdouble, 1}(undef, numrow))

   ccall((:Highs_getSolution, "libhighs.so"), Cvoid, (Ptr{Cvoid}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}), highs, solution.colvalue, solution.coldual, solution.rowvalue, solution.rowdual)

   return solution
end

function Highs_getBasis(highs)
   numcol = Highs_getNumCols(highs)
   numrow = Highs_getNumRows(highs)

   basis = HighsBasis(Array{Cint, 1}(undef, numcol), Array{Cint, 1}(undef, numrow))

   ccall((:Highs_getBasis, "libhighs.so"), Cvoid, (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cint}), highs, basis.colbasisstatus, basis.rowbasisstatus)

   return basis
end

function Highs_getObjectiveValue(highs)
   return ccall((:Highs_getObjectiveValue, "libhighs.so"), Cdouble, (Ptr{Cvoid},), highs)
end

function Highs_getIterationCount(highs)
   return ccall((:Highs_getIterationCount, "libhighs.so"), Cint, (Ptr{Cvoid},), highs)
end

function Highs_addRow(highs, lower, upper, indices, values)
   n_new_nz = convert(Cint, size(indices, 1))

   lo = convert(Cdouble, lower)
   hi = convert(Cdouble, upper)

   idx = convert(Array{Cint}, indices)
   val = convert(Array{Cdouble}, values)

   return ccall((:Highs_addRow, "libhighs.so"), Cint, (Ptr{Cvoid}, Cdouble, Cdouble, Cint, Ptr{Cint}, Ptr{Cdouble}), highs, lo, hi, n_new_nz, idx, val)
end

function Highs_addRows(highs, lower, upper, starts, indices, values)
   n_new_rows = convert(Cint, size(lower, 1))
   n_new_nz = convert(Cint, size(indices, 1))

   lo = convert(Array{Cdouble}, lower)
   hi = convert(Array{Cdouble}, upper)

   st = convert(Array{Cint}, starts)
   idx = convert(Array{Cint}, indices)
   val = convert(Array{Cdouble}, values)

   return ccall((:Highs_addRows, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Ptr{Cdouble}, Ptr{Cdouble}, Cint, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble}),
   highs, n_new_rows, lo, hi, n_new_nz, st, idx, val)
end

function Highs_addCol(highs, cost, lower, upper, indices, values)
   n_new_nz = convert(Cint, size(indices, 1))

   cc = convert(Cdouble, cost)
   lo = convert(Cdouble, lower)
   hi = convert(Cdouble, upper)

   idx = convert(Array{Cint}, indices)
   val = convert(Array{Cdouble}, values)

   return ccall((:Highs_addCol, "libhighs.so"), Cint, (Ptr{Cvoid}, Cdouble, Cdouble, Cdouble, Cint, Ptr{Cint}, Ptr{Cdouble}),
   highs, cc, lo, hi, n_new_nz, idx, val)
end

function Highs_addCols(highs, costs, lower, upper, starts, indices, values)
   n_new_cols = convert(Cint, size(lower, 1))
   n_new_nz = convert(Cint, size(indices, 1))

   co = convert(Array{Cdouble}, costs)
   lo = convert(Array{Cdouble}, lower)
   hi = convert(Array{Cdouble}, upper)

   st = convert(Array{Cint}, starts)
   idx = convert(Array{Cint}, indices)
   val = convert(Array{Cdouble}, values)

   return ccall((:Highs_addCols, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Cint, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble}),
   highs, n_new_cols, co, lo, hi, n_new_nz, st, idx, val)
end

function Highs_changeObjectiveSense(highs, sense)
   sns = convert(Cint, sense)

   return ccall((:Highs_changeObjectiveSense, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint), highs, sns)
end

function Highs_changeColCost(highs, colidx, cost)
   col = convert(Cint, colidx)
   cst = convert(Cdouble, cost)

   return ccall((:Highs_changeColCost, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Cdouble), highs, colidx, cost)
end

function Highs_changeColsCostBySet(highs, set, cost)
   num_set_entries = convert(Cint, size(set, 1))

   st = convert(Array{Cint}, set)
   cst = convert(Array{Cdouble}, cost)

   return ccall((:Highs_changeColsCostBySet, "libhighs.so"), Cint, (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cdouble}), highs, st, cst)
end

function Highs_changeColsCostByMask(highs, mask, cost)
   msk = convert(Array{Cint}, mask)
   cst = convert(Array{Cdouble}, cost)

   return ccall((:Highs_changeColsCostByMask, "libhighs.so"), Cint, (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cdouble}), highs, msk, cst)
end

function Highs_changeColBounds(highs, col, lower, upper)
   colidx = convert(Cint, col)

   lo = convert(Cdouble, lower)
   hi = convert(Cdouble, upper)

   return ccall((:Highs_changeColBounds, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Cdouble, Cdouble), highs, colidx, lo, hi)
end

function Highs_changeColsBoundsByRange(highs, from, to, lower, upper)
   f = convert(Cint, from)
   t = convert(Int33, to)

   lo = convert(Array{Cdouble}, lower)
   hi = convert(Array{Cdouble}, upper)

   return ccall((:Highs_changeColsBoundsByRange, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Cint, Ptr{Cdouble}, Ptr{Cdouble}),
   highs, f, t, lo, hi)
end

function Highs_changeColsBoundsBySet(highs, set, lower, upper)
   nset = convert(Cint, size(set, 1))

   st = convert(Array{Cint}, set)

   lo = convert(Array{Cdouble}, lower)
   hi = convert(Array{Cdouble}, upper)

   return ccall((:Highs_changeColsBoundsBySet, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Ptr{Cint}, Ptr{Cdouble}, Ptr{Cdouble}),
   highs, nset, st, lo, hi)
end

function Highs_changeColsBoundsByMask(highs, mask, lower, upper)
   msk = convert(Array{Cint}, mask)

   lo = convert(Array{Cdouble}, lower)
   hi = convert(Array{Cdouble}, upper)

   return ccall((:Highs_changeColsBoundsByMask, "libhighs.so"), Cint, (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cdouble}, Ptr{Cdouble}),
   highs, msk, lo, hi)
end

function Highs_changeRowBounds(highs, row, lower, upper)
   idx = convert(Cint, row)
   lo = convert(Cdouble, lower)
   hi = convert(Cdouble, upper)

   return ccall((:Highs_changeRowBounds, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Cdouble, Cdouble),
   highs, idx, lo, hi)
end

function Highs_changeRowsBoundsBySet(highs, set, lower, upper)
   nst = convert(Cint, size(set, 1))
   st = convert(Array{Cint}, set)

   lo = convert(Array{Cdouble}, lower)
   hi = convert(Array(Cdouble), upper)

   return ccall((:Highs_changeRowsBoundsBySet, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Ptr{Cint}, Ptr{Cdouble}, Ptr{Cdouble}),
   highs, nst, st, lo, hi)
end

function Highs_changeRowsBoundsByMask(highs, mask, lower, upper)
   msk = convert(Array{Cint}, mask)

   lo = convert(Array{Cdouble}, lower)
   hi = convert(Array{Cdouble}, upper)

   return ccall((:Highs_changeRowsBoundsByMask, "libhighs.so"), Cint, (Ptr{Cvoid}, Ptr{Cint}, Ptr{Cdouble}, Ptr{Cdouble}),
   highs, msk, lo, hi)
end

#=

int Highs_getColsByRange(
    void *highs,          //!< HiGHS object reference
    const int from_col,   //!< The index of the first column to
                          //!< get from the model
    const int to_col,     //!< One more than the last column to get
                          //!< from the model
    int* num_col,          //!< Number of columns got from the model
    double *costs,        //!< Array of size num_col with costs
    double *lower,        //!< Array of size num_col with lower bounds
    double *upper,        //!< Array of size num_col with upper bounds
    int* num_nz,           //!< Number of nonzeros got from the model
    int *matrix_start,    //!< Array of size num_col with start
                          //!< indices of the columns
    int *matrix_index,    //!< Array of size num_nz with row
                          //!< indices for the columns
    double *matrix_value  //!< Array of size num_nz with row
                          //!< values for the columns
);

int Highs_getColsBySet(
    void *highs,                //!< HiGHS object reference
    const int num_set_entries,  //!< The number of indides in the set
    const int *set,             //!< Array of size num_set_entries with indices
                                //!< of columns to get
    int* num_col,                //!< Number of columns got from the model
    double *costs,              //!< Array of size num_col with costs
    double *lower,              //!< Array of size num_col with lower bounds
    double *upper,              //!< Array of size num_col with upper bounds
    int* num_nz,                 //!< Number of nonzeros got from the model
    int *matrix_start,          //!< Array of size num_col with start indices
                                //!< of the columns
    int *matrix_index,          //!< Array of size num_nz with row indices
                                //!< for the columns
    double *matrix_value        //!< Array of size num_nz with row values
                                //!< for the columns
);

int Highs_getColsByMask(
    void *highs,          //!< HiGHS object reference
    const int *mask,      //!< Full length array with 1 => get; 0 => not
    int* num_col,          //!< Number of columns got from the model
    double *costs,        //!< Array of size num_col with costs
    double *lower,        //!< Array of size num_col with lower bounds
    double *upper,        //!< Array of size num_col with upper bounds
    int* num_nz,           //!< Number of nonzeros got from the model
    int *matrix_start,    //!<  Array of size num_col with start
                          //!<  indices of the columns
    int *matrix_index,    //!<  Array of size num_nz with row indices
                          //!<  for the columns
    double *matrix_value  //!<  Array of size num_nz with row values
                          //!<  for the columns
);

int Highs_getRowsByRange(
    void *highs,          //!< HiGHS object reference
    const int from_row,   //!< The index of the first row to get from the model
    const int to_row,     //!< One more than the last row get from the model
    int* num_row,          //!< Number of rows got from the model
    double *lower,        //!< Array of size num_row with lower bounds
    double *upper,        //!< Array of size num_row with upper bounds
    int* num_nz,           //!< Number of nonzeros got from the model
    int *matrix_start,    //!< Array of size num_row with start indices of the
                          //!< rows
    int *matrix_index,    //!< Array of size num_nz with column indices for the
                          //!< rows
    double *matrix_value  //!< Array of size num_nz with column values for the
                          //!< rows
);

int Highs_getRowsBySet(
    void *highs,                //!< HiGHS object reference
    const int num_set_entries,  //!< The number of indides in the set
    const int *set,             //!< Array of size num_set_entries with indices
                                //!< of rows to get
    int* num_row,                //!< Number of rows got from the model
    double *lower,              //!< Array of size num_row with lower bounds
    double *upper,              //!< Array of size num_row with upper bounds
    int* num_nz,                 //!< Number of nonzeros got from the model
    int *matrix_start,          //!< Array of size num_row with start indices
                                //!< of the rows
    int *matrix_index,          //!< Array of size num_nz with column indices
                                //!< for the rows
    double *matrix_value        //!< Array of size num_nz with column
                                //!< values for the rows
);

int Highs_getRowsByMask(
    void *highs,          //!< HiGHS object reference
    const int *mask,      //!< Full length array with 1 => get; 0 => not
    int* num_row,          //!< Number of rows got from the model
    double *lower,        //!< Array of size num_row with lower bounds
    double *upper,        //!< Array of size num_row with upper bounds
    int* num_nz,           //!< Number of nonzeros got from the model
    int *matrix_start,    //!< Array of size num_row with start indices
                          //!< of the rows
    int *matrix_index,    //!< Array of size num_nz with column indices
                          //!< for the rows
    double *matrix_value  //!< Array of size num_nz with column
                          //!< values for the rows
);

=#

function Highs_deleteColsByRange(highs, from, to)
   f = convert(Cint, from)
   t = convert(Cint, to)

   return ccall((:Highs_deleteColsByRange, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Cint), highs, f, t)
end

function Highs_deleteColsBySet(highs, set)
   nset = convert(Cint, size(set, 1))

   st = convert(Array{Cint}, set)

   return ccall((:Highs_deleteColsBySet, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Ptr{Cint}), highs, nset, st)
end

function Highs_deleteColsByMask(highs, mask)
   msk = convert(Array{Cint}, mask)

   return ccall((:Highs_deleteColsByMask, "libighs.so"), Cint, (Ptr{Cvoid}, Ptr{Cint}), highs, msk)
end

function Highs_deleteRowsByRange(highs, from, to)
   f = convert(Cint, from)
   t = convert(Cint, to)

   return ccall((:Highs_deleteRowsByRange, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Cint), highs, f, t)
end

function Highs_deleteRowsBySet(highs, set)
   nset = convert(Cint, size(set, 1))

   st = convert(Array{Cint}, set)

   return ccall((:Highs_deleteRowsBySet, "libhighs.so"), Cint, (Ptr{Cvoid}, Cint, Ptr{Cint}), highs, nset, st)
end

function Highs_deleteRowsByMask(highs, mask)
   msk = convert(Array{Cint}, mask)

   return ccall((:Highs_deleteRowsByMask, "libhighs.so"), Cint, (Ptr{Cvoid}, Ptr{Cint}), highs, msk)
end


using BenchmarkTools

function to_pari(x::fmpz_mat, varname = "x")
  s = "$varname = ["
  y = Hecke._eltseq(x)
  n = ncols(x)
  for i in 1:length(y)
    s = s * "$(y[i])"

    if i % n != 0
      s = s * ","
    else
      if i != length(y)
        s = s * ";"
      end
    end
  end

  s = s * "]"
  return s
end

function to_magma(x::fmpz_mat, varname = "x")
  n = nrows(x)
  m = ncols(x)
  s = "$varname := Matrix(Integers(), $n, $m, ["
  y = Hecke._eltseq(x)
  for i in 1:length(y)
    s = s * "$(y[i])"
    if i < length(y)
      s = s * ","
    end
  end
  s = s * "]);"
end

function to_magma(x::fmpq_mat, varname = "x")
  n = nrows(x)
  m = ncols(x)
  s = "$varname := Matrix(Rationals(), $n, $m, ["
  y = Hecke._eltseq(x)
  for i in 1:length(y)
    s = s * "$(y[i])"
    if i < length(y)
      s = s * ","
    end
  end
  s = s * "]);"
  return s
end

function to_sage(x::fmpq_mat, varname = "x")
  n = nrows(x)
  m = ncols(x)
  s = "$varname = Matrix(QQ, $n, $m, ["
  y = Hecke._eltseq(x)
  for i in 1:length(y)
    s = s * "$(y[i])"
    if i < length(y)
      s = s * ","
    end
  end
  s = s * "]);"
  return s
end

function benchmark_code_sage(L::Vector{Hecke.ZLat}, repetitions::Int = 10; out_file::String = "sage_timings")
  s = """
  from sage.all import *
  import timeit
  res = []
  out_file = \"$(out_file)\"
  """
  for i in 1:length(L)
    G = gram_matrix(L[i])
    s = s * to_sage(G, "G$i") * "\n"
    s = s * """
    print(\"Doing\", $(i))
    t = 0.0
    for i in range($repetitions):
      L = IntegralLattice(G$(i))
      s = timeit.repeat(lambda: L.automorphisms(), repeat = 1, number = 1)
      t = t + s[0]
    t = t/$(repetitions)
    t = t/1000
    res.append(t)
    """
  end
  s = s * """
f = open(out_file, 'w')
f.write(\"%s\\n\" % res)
f.close()
quit()

  """
  return s
end

function benchmark_code_magma(L::Vector{Hecke.ZLat}, repetitions::Int = 10; out_file::String = "magma_timings")
  default_repetitions = repetitions
  s = """
  res := [];
  """
  for i in 1:length(L)
    if rank(L[i]) <= 15
      repititions = 100
    else
      repititions = default_repetitions
    end
    G = gram_matrix(L[i])
    s = s * to_magma(G, "G$i") * "\n"
    s = s * """
    t0 := 0.0;
    \"Doing $(i)\";
    for i in [1..$(repetitions)] do
      L := LatticeWithGram(G$(i));
      t := Cputime();
      _ := AutomorphismGroup(L);
      t0 := t0 + Cputime(t);
    end for;
    t0av := RealField()!t0/$(repetitions);
    Append(~res, [t0av, RealField()!t0]);
    """
  end
  s = s * """
  PrintFile(\"$(out_file)\", res);
  quit;
  """
  return s
end

function benchmark_magma(L::Vector{Hecke.ZLat})
  t, fp = mktemp()
  @show t
  out_file, fp_out = mktemp()
  close(fp_out)
  mcode = benchmark_code_magma(L, out_file = out_file)
  print(fp, mcode)
  close(fp)

  run(`magma $(t)`)
  rm(t)

  s = read(out_file, String)

  rm(out_file)

  t = Meta.eval(Meta.parse(s))
  return Float64[x[1] for x in t]
end

function benchmark_sage(L::Vector{Hecke.ZLat})
  t, fp = mktemp()
  @show t
  out_file, fp_out = mktemp()
  close(fp_out)
  scode = benchmark_code_sage(L, out_file = out_file)
  print(fp, scode)
  close(fp)

  run(`/home/thofmann/SageMath/sage $(t)`)
  rm(t)

  s = read(out_file, String)

  rm(out_file)

  return Meta.eval(Meta.parse(s))
end

function benchmark_hecke(L::Vector{Hecke.ZLat})
  res = Float64[]
  for i in 1:length(L)
    LL = L[i]
    println("Doing $i")
    t = @belapsed Hecke.assert_has_automorphisms($LL, redo = true) samples=10
    push!(res, t)
  end
  return res
end

function benchmark_all(L::Vector{Hecke.ZLat})
  bM = benchmark_magma(L)
  bH = benchmark_hecke(L)
  bS = benchmark_sage(L)
end


abstract type CutData end

struct BendersCutData <: CutData
  θk
  λk
  xk
end

struct LLIntegerCutData <: CutData
  θlb
  yk
end

struct IntegerCutData <: CutData
  yk
end

(==)(cd1::BendersCutData,cd2::BendersCutData) = (cd1.θk == cd2.θk) && (cd1.λk == cd2.λk) && (cd1.xk == cd2.xk)
(==)(cd1::LLIntegerCutData,cd2::LLIntegerCutData) = (cd1.θlb == cd2.θlb) &&  (cd1.yk == cd2.yk)
(==)(cd1::IntegerCutData,cd2::IntegerCutData) = (cd1.yk == cd2.yk)

"""
bendersolve
"""
function bendersolve(graph::ModelGraph; max_iterations::Int64=10, cuts::Array{Symbol,1}=[:LP], ϵ=1e-5,UBupdatefrequency=1,timelimit=3600,verbose=false)
  starttime = time()
  s = Solution(method=:benders)
  updatebound = true

  verbose && info("Preparing graph")
  bdprepare(graph)
  n = getattribute(graph, :normalized)

  verbose && info("Solve relaxation and set LB")
  mf = getattribute(graph, :mflat)
  solve(mf,relaxation=true)
  LB = getobjectivevalue(getattribute(graph, :mflat))
  UB = Inf

  # Set bound to root node
  rootnode = getattribute(graph, :roots)[1]
  rootmodel = getmodel(rootnode)
  @constraint(rootmodel, rootmodel.obj.aff >= LB)

  # Begin iterations
  for i in 1:max_iterations
    tic()
    updatebound = ((i-1) % UBupdatefrequency) == 0
    LB,UB = forwardstep(graph, cuts, updatebound)

    tstamp = time() - starttime

    itertime = toc()
    if n == 1
      saveiteration(s,tstamp,[UB,LB,itertime,tstamp],n)
    else
      saveiteration(s,tstamp,[n*LB,n*UB,itertime,tstamp],n)
    end
    printiterationsummary(s,singleline=false)

    if abs(UB-LB) < ϵ
      s.termination = "Optimal"
      return s
    end

    # Check time limit
    if tstamp > timelimit
      s.termination = "Time Limit"
      return s
    end

    if getattribute(graph, :stalled)
      s.termination = "Stalled"
      return s
    end
  end

  s.termination = "Max Iterations"
  return s
end

function forwardstep(graph::ModelGraph, cuts::Array{Symbol,1}, updatebound::Bool)
  levels = getattribute(graph, :levels)
  numlevels = length(levels)
  for level in 1:numlevels
    nodeslevel = levels[level]
    for node in nodeslevel
      solveprimalnode(node,graph,cuts,updatebound)
    end
  end
  LB = getattribute(graph, :LB)
  if updatebound
    iterUB = sum(getattribute(node, :preobjval) for node in getnodes(graph))
    setattribute(graph, :iterUB, iterUB)
    UB = min(getattribute(graph, :UB),iterUB)
    setattribute(graph, :UB, UB)
  else
    UB = getattribute(graph, :UB)
  end
  return LB,UB
end

function solveprimalnode(node::ModelNode, graph::ModelGraph, cuts::Array{Symbol,1}, updatebound::Bool)
  # 1. Add cuts
  generatecuts(node,graph)
  # 2. Take x
  takex(node)
  # 3. solve
  if :LP in cuts
    solvelprelaxation(node)
  end
  if updatebound
    solvenodemodel(node,graph)
  end
  # 4. put x
  putx(node,graph)
  # 5. put cuts and nodebound
  putcutdata(node,graph,cuts)
end

function solvelprelaxation(node::ModelNode)
  model = getmodel(node)
  status = solve(model, relaxation = true)

  @assert status == :Optimal

  dualconstraints = getattribute(node, :linkconstraints)

  λnode = getdual(dualconstraints)
  nodebound = getobjectivevalue(model)

  setattribute(node, :bound, nodebound)
  setattribute(node, :λ, λnode)

  return status
end

function solvenodemodel(node::ModelNode,graph::ModelGraph)
  model = getmodel(node)
  solve(model)
  if in_degree(graph,node) == 0 # Root node
    setattribute(graph, :LB, getobjectivevalue(model))
  end
  setattribute(node, :preobjval, JuMP.getvalue(model.ext[:preobj]))
end

function takex(node::ModelNode)
  xinvals = getattribute(node, :xin)
  xinvars = getattribute(node, :xinvars)
  if length(xinvals) > 0
    fix.(xinvars,xinvals)
  end
end

function putx(node::ModelNode,graph::ModelGraph)
  childvars = getattribute(node,:childvars)
  children = out_neighbors(graph,node)
  length(children) == 0 && return true

  for child in children
    xnode = getvalue(childvars[getindex(graph,child)])
    setattribute(child,:xin, xnode)
  end
end

function putcutdata(node::ModelNode,graph::ModelGraph,cuts::Array{Symbol,1})
  parents = in_neighbors(graph,node)
  length(parents) == 0 && return true
  parent = parents[1]    # Assume only one parent
  parentcuts = getattribute(parent, :cutdata)
  θk = getattribute(node,:bound)
  λk = getattribute(node,:λ)
  xk = getattribute(node,:xin)
  nodeindex = getindex(graph,node)
  if :LP in cuts || :Root in cuts
    bcd = BendersCutData(θk, λk, xk)
    push!(parentcuts[nodeindex],bcd)
  end
  if :LLinteger in cuts
    llintcd = LLIntegerCutData(θlb,xk)
    push!(parentcuts[nodeindex],llintcd)
  end
  if :Integer in cuts
    intcd = IntegerCutData(xk)
    push!(parentcuts[nodeindex],intcd)
  end
end

function generatecuts(node::ModelNode,graph::ModelGraph)
  children = out_neighbors(graph,node)
  length(children) == 0 && return true

  cutdataarray = getattribute(node,:cutdata)
  previouscuts = getattribute(node,:prevcuts)
  thisitercuts = Dict()
  samecuts = Dict()
  for child in children
    childindex = getindex(graph,child)
    thisitercuts[childindex] = CutData[]
    samecuts[childindex] = Bool[]

    while length(cutdataarray[childindex]) > 0
      cutdata = pop!(cutdataarray[childindex])
      samecut = in(cutdata,previouscuts[childindex])
      push!(samecuts[childindex],samecut)
      samecut && continue
      if typeof(cutdata) == BendersCutData
        generatebenderscut(node,cutdata,childindex)
      elseif typeof(cutdata) == LLIntegerCutData
        generateLLintegercut(node,cutdata)
      elseif typeof(cutdata) == IntegerCutData
        generateintegercut(node,cutdata)
      end
      push!(thisitercuts[childindex],cutdata)
    end
    samecuts[childindex] = reduce(*,samecuts[childindex]) && length(samecuts[childindex]) > 0
  end
  setattribute(node,:prevcuts, thisitercuts)
  nodesamecuts = collect(values(samecuts))
  setattribute(node, :stalled, reduce(*,nodesamecuts))
  getattribute(node, :stalled) && warn("Node $(node.label) stalled")
  if in(node,getattribute(graph, :roots) ) && getattribute(node, :stalled)
    setattribute(graph, :stalled,  true)
 end
end

function generatebenderscut(node::ModelNode, cd::BendersCutData,index)
  model = getmodel(node)
  θ = getindex(model, :θ)
  x = getattribute(node, :childvars)[index]
  @constraint(model, θ[index] >= cd.θk + cd.λk'*(cd.xk - x))
end

function identifylevels(graph::ModelGraph)
  #Create lists of root and leaf nodes in graph
  setattribute(graph, :roots, ModelNode[])
  roots = getattribute(graph, :roots)
  setattribute(graph, :leaves, ModelNode[])
  leaves = getattribute(graph, :leaves)
  #Create dictionary to keep track of levels of nodes
  setattribute(graph, :levels, Dict())
  levels = getattribute(graph, :levels)
  #Iterate through every node to check for root/leaf nodes
  for node in getnodes(graph)
    setattribute(node,:xin, [])
    setattribute(node,:λ, [])
    setattribute(node,:bound, NaN)
    setattribute(node,:xinvars, [])
    setattribute(node,:preobjval, NaN)
    setattribute(node,:linkconstraints, [])
    #If the node does not have parents it is a root node
    if in_degree(graph,node) == 0
      push!(roots,node)
    end
    #If the node does not have children it is a leaf node
    if out_degree(graph,node) == 0
      push!(leaves,node)
    end
  end

  #Start mapping level from the root nodes
  current = roots
  level = 1
  levels[1]  = roots
  while length(current) > 0
    levels[level] = current
    children = []
    for node in current
        push!(children,out_neighbors(graph,node)...)
        setattribute(node,:childvars, Dict(getindex(graph,child) => [] for child in out_neighbors(graph,node)))
        setattribute(node,:cutdata, Dict(getindex(graph,child) => CutData[] for child in out_neighbors(graph,node)))
        setattribute(node,:prevcuts, Dict(getindex(graph,child) => CutData[] for child in out_neighbors(graph,node)))
        setattribute(node,:stalled, false)
    end
    current = children
    level += 1
  end
  setattribute(graph,:numlevels,level - 1)
end

function bdprepare(graph::ModelGraph)
  if Plasmo.hasattribute(graph,:preprocessed)
    return true
  end

  identifylevels(graph)
  setattribute(graph, :normalized, normalizegraph(graph))
  setattribute(graph, :stalled, false)
  setattribute(graph, :mflat ,create_jump_graph_model(graph))
  setattribute(graph, :UB, Inf)
  JuMP.setsolver(getattribute(graph, :mflat) ,getsolver(graph))

  links = getlinkconstraints(graph)
  numlinks = length(links)

  for index in 1:length(getnodes(graph))
    node = getnode(graph, index)
    model = getmodel(node)
    if model.solver == JuMP.UnsetSolver()
      model.solver = getsolver(graph)
    end
    model.ext[:preobj] = model.obj
    setattribute(node, :cgmodel, deepcopy(model))
    #Add theta to parent nodes
    if out_degree(graph,node) != 0
      childrenindices = [getindex(graph,child) for child in out_neighbors(graph,node)]
      sort!(childrenindices)
      @variable(model, θ[i in childrenindices] >= -1e6)
      model.obj += sum(θ[i] for i in childrenindices)
    end
  end
  #Add dual constraint to child nodes using the linking constraints
  for (numlink,link) in enumerate(links)
    #Take the two variables of the constraint
    var1 = link.terms.vars[1]
    var2 = link.terms.vars[2]
    #Determine which nodes they belong to
    nodeV1 = getnode(var1)
    nodeV2 = getnode(var2)
    #Set the order of the nodes
    if ischildnode(graph,nodeV1,nodeV2)
      childnode = nodeV1
      childvar = var1
      parentnode = nodeV2
      parentvar = var2
    else
      childnode = nodeV2
      childvar = var2
      parentnode = nodeV1
      parentvar = var1
    end
    childindex = getindex(graph,childnode)
    childmodel = getmodel(childnode)
    push!(getattribute(parentnode, :childvars)[childindex],parentvar)
    linkvar = @variable(childmodel)
    setname(linkvar,"linkvar$numlink")
    push!(getattribute(childnode, :xinvars),linkvar)
    conref = @constraint(childmodel, linkvar - childvar == 0)
    push!(getattribute(childnode, :linkconstraints), conref)
  end
  setattribute(graph, :preprocessed, true)
end


"""initialises netCDF files for data output."""
function output_ini(u,v,η,sst)
    # only process with rank 0 defines the netCDF file
    if output #&& prank == 0

        # Attributes
        Dictu = Dict{String,Any}("description"=>"Data from shallow-water model juls.")
        Dictu["details"] = "Cartesian coordinates, f or beta-plane, Arakawa C-grid"
        Dictu["reference"] = "github.com/milankl/juls"

        Dictu["nx"] = nx
        Dictu["Lx"] = Lx
        Dictu["L_ratio"] = L_ratio
        Dictu["delta"] = Δ

        Dictu["halo"] = halo
        Dictu["haloeta"] = haloη
        Dictu["halosstx"] = halosstx
        Dictu["halossty"] = halossty

        Dictu["g"] = gravity
        Dictu["water_depth"] = water_depth
        Dictu["phi"] = ϕ
        Dictu["density"] = ρ

        Dictu["wind_forcing"] = wind_forcing
        Dictu["Fx0"] = Fx0

        Dictu["topography_feature"] = topography_feature
        Dictu["topofeat_height"] = topofeat_height
        Dictu["topofeat_width"] = topofeat_width

        Dictu["surface_forcing"] = string(surface_forcing)
        Dictu["t_relax"] = t_relax
        Dictu["eta_refh"] = η_refh
        Dictu["η_refw"] = η_refw

        Dictu["Numtype"] = string(Numtype)
        Dictu["output_dt"] = output_dt
        Dictu["nout"] = nout
        Dictu["nadvstep"] = nadvstep

        Dictu["RKo"] = RKo
        Dictu["cfl"] = cfl
        Dictu["Ndays"] = Ndays

        Dictu["bc_x"] = bc_x
        Dictu["lbc"] = lbc

        Dictu["adv_scheme"] = adv_scheme

        Dictu["bottom_friction"] = bottom_friction
        Dictu["drag"] = drag
        Dictu["taudrag"] = τdrag

        Dictu["diffusion"] = diffusion
        Dictu["nuConst"] = ν_const
        Dictu["c_smag"] = c_smag

        Dictu["tracer_advcetion"] = string(tracer_advection)
        Dictu["tracer_relaxation"] = string(tracer_relaxation)
        Dictu["injection_region"] = injection_region
        Dictu["sstrestart"] = string(sstrestart)
        Dictu["Uadv"] = Uadv
        Dictu["SSTmax"] = SSTmax
        Dictu["SSTmin"] = SSTmin
        Dictu["tauSST"] = τSST
        Dictu["SSTw"] = SSTw
        Dictu["SSTphi"] = SSTϕ

        Dictu["initial_cond"] = initial_cond
        Dictu["init_run_id"] = init_run_id
        Dictu["initpath"] = initpath

        if "u" in output_vars
            xudim = NcDim("x",nux,values=x_u)
            yudim = NcDim("y",nuy,values=y_u)
            tdim = NcDim("t",0,unlimited=true)

            uvar = NcVar("u",[xudim,yudim,tdim],t=Float32)
            tvaru = NcVar("t",tdim,t=Int64)

            ncu = NetCDF.create(runpath*"u.nc",[uvar,tvaru],mode=NC_NETCDF4)
            NetCDF.putatt(ncu,"u",Dict("units"=>"m/s","long_name"=>"zonal velocity"))
        else
            ncu = 0
        end

        if "v" in output_vars
            xvdim = NcDim("x",nvx,values=x_v)
            yvdim = NcDim("y",nvy,values=y_v)
            tdim = NcDim("t",0,unlimited=true)

            vvar = NcVar("v",[xvdim,yvdim,tdim],t=Float32)
            tvarv = NcVar("t",tdim,t=Int64)

            ncv = NetCDF.create(runpath*"v.nc",[vvar,tvarv],mode=NC_NETCDF4)
            NetCDF.putatt(ncv,"v",Dict("units"=>"m/s","long_name"=>"meridional velocity"))
        else
            ncv = 0
        end

        if "eta" in output_vars
            xTdim = NcDim("x",nx,values=x_T)
            yTdim = NcDim("y",ny,values=y_T)
            tdim = NcDim("t",0,unlimited=true)

            ηvar = NcVar("eta",[xTdim,yTdim,tdim],t=Float32)
            tvarη = NcVar("t",tdim,t=Int64)

            ncη = NetCDF.create(runpath*"eta.nc",[ηvar,tvarη],mode=NC_NETCDF4)
            NetCDF.putatt(ncη,"eta",Dict("units"=>"m","long_name"=>"sea surface height"))
        else
            ncη = 0
        end

        if "sst" in output_vars
            xTdim = NcDim("x",nx,values=x_T)
            yTdim = NcDim("y",ny,values=y_T)
            tdim = NcDim("t",0,unlimited=true)

            ηvar = NcVar("sst",[xTdim,yTdim,tdim],t=Float32)
            tvarη = NcVar("t",tdim,t=Int64)

            ncsst = NetCDF.create(runpath*"sst.nc",[ηvar,tvarη],mode=NC_NETCDF4)
            NetCDF.putatt(ncsst,"sst",Dict("units"=>"1","long_name"=>"sea surface temperature"))
        else
            ncsst = 0
        end

        ncs = (ncu,ncv,ncη,ncsst)

        # Write attributes and units
        for nc in ncs
            if nc != 0
                NetCDF.putatt(nc,"global",Dictu)
                NetCDF.putatt(nc,"t",Dict("units"=>"s","long_name"=>"time"))
                NetCDF.putatt(nc,"x",Dict("units"=>"m","long_name"=>"zonal coordinate"))
                NetCDF.putatt(nc,"y",Dict("units"=>"m","long_name"=>"meridional coordinate"))
            end
        end

        # write initial conditions
        iout = 1   # counter for output time steps
        ncs,iout = output_nc(ncs,u,v,η,sst,0,iout)

        # also output scripts
        scripts_output()

        return ncs,iout
    else
        return nothing, nothing
    end
end

""" Writes data to a netCDF file."""
function output_nc(ncs,u,v,η,sst,i,iout)

    # if nprocs > 1
    #     #TODO MPI Gather data
    #     #TODO rename u,v,η necessary? To distinguish between the loval u,v,η and the gathered u,v,η?
    # end

    # output only every nout time steps
    # only process 0 will do the output
    if i % nout == 0 && output #&& prank == 0

        # cut off the halo
        if ncs[1] != 0
            NetCDF.putvar(ncs[1],"u",Float32.(u[halo+1:end-halo,halo+1:end-halo]),start=[1,1,iout],count=[-1,-1,1])
        end
        if ncs[2] != 0
            NetCDF.putvar(ncs[2],"v",Float32.(v[halo+1:end-halo,halo+1:end-halo]),start=[1,1,iout],count=[-1,-1,1])
        end
        if ncs[3] != 0
            NetCDF.putvar(ncs[3],"eta",Float32.(η[haloη+1:end-haloη,haloη+1:end-haloη]),start=[1,1,iout],count=[-1,-1,1])
        end
        if ncs[4] != 0
            NetCDF.putvar(ncs[4],"sst",Float32.(sst[halosstx+1:end-halosstx,halossty+1:end-halossty]),start=[1,1,iout],count=[-1,-1,1])
        end

        for nc in ncs
            if nc !=0
                NetCDF.putvar(nc,"t",Int64[i*dtint],start=[iout])
                NetCDF.sync(nc)     # sync to view netcdf while model is still running
            end
        end

        iout += 1
    end

    #TODO MPI Barrier, Waitall?

    return ncs,iout
end

"""Closes netCDF and progress.txt files."""
function output_close(ncs,progrtxt)
    if output #&& prank == 0
        for nc in ncs
            if nc !=0
                NetCDF.close(nc)
            end
        end
        println("All data stored.")
        write(progrtxt,"All data stored.")
        close(progrtxt)
    end
end

"""Checks output folders to determine a 4-digit run id number."""
function get_run_id_path(order="continue",run_id=nothing)

    """Finds the first gap in a list of integers."""
    function gap(a::Array{Int,1})
        try
            return minimum([i for i in minimum(a):maximum(a) if ~(i in a)])
        catch
            return maximum(a)+1
        end
    end

    # only process rank 0 checks existing folders
    if output #&& prank == 0
        runlist = filter(x->startswith(x,"run"),readdir(outpath))
        existing_runs = [parse(Int,id[4:end]) for id in runlist]
        if length(existing_runs) == 0           # if no runfolder exists yet
            runpath = outpath*"run0000/"
            mkdir(runpath)
            return 0,runpath
        else                                    # create next folder
            if order == "fill"  # find the smallest gap in runfolders
                run_id = gap(existing_runs)
                runpath = outpath*"run"*@sprintf("%04d",run_id)*"/"
                mkdir(runpath)

            elseif order == "specific" # specify the run_id as input argument
                runpath = outpath*"run"*@sprintf("%04d",run_id)*"/"
                try # create folder if not existent
                    mkdir(runpath)
                catch # else rm folder and create new one
                    rm(runpath,recursive=true)
                    mkdir(runpath)
                end

            elseif order == "continue" # find largest folder and count one up
                run_id = maximum(existing_runs)+1
                runpath = outpath*"run"*@sprintf("%04d",run_id)*"/"
                mkdir(runpath)
            else
                throw(error("Order $order is not valid for get_run_id_path(), chose continue, specific or fill."))
            end
            return run_id,runpath
        end
    else
        return 0,"no runpath"
    end
end

#TODO in ensemble mode, the .jl files might have changed since the start and do not correspond to what
#TODO is actually executed!
"""Archives all .jl files of juls in the output folder to make runs reproducible."""
function scripts_output()
    if output #&& prank == 0
        # copy all files in juls main folder
        mkdir(runpath*"scripts")
        for juliafile in filter(x->endswith(x,".jl"),readdir())
            cp(juliafile,runpath*"scripts/"*juliafile)
        end

        # and also in the src folder
        mkdir(runpath*"scripts/src")
        for juliafile in filter(x->endswith(x,".jl"),readdir("src"))
            cp("src/"*juliafile,runpath*"scripts/src/"*juliafile)
        end
    end
end


#
#  CreateRunDirectory.jl
#  CollagenPolymer
#
#  Created by Christopher Revell on 02/05/2020.
#
#

module CreateRunDirectory

# Import Julia packages
using Dates

# Import local program modules

function createRunDirectory(nMonomers,L,nDomains,μ,kT,ϵLJ,σ,k,rₑ,Ebend,D,tMax,outputInterval,nGrid,boxSize,intrctnThrshld)

    # Create directory for run data labelled with current time.
    foldername = Dates.format(Dates.now(),"yyyy-mm-dd-HH-MM-SS")
    mkdir("output/$(foldername)")
    # Store system parameters.
    open("output/$(foldername)/conditions.txt","w") do conditionsfile
        println(conditionsfile,"nMonomers      $nMonomers      ")
        println(conditionsfile,"L              $L              ")
        println(conditionsfile,"σ              $σ              ")
        println(conditionsfile,"nDomains       $nDomains       ")
        println(conditionsfile,"nGrid          $nGrid          ")
        println(conditionsfile,"intrctnThrshld $intrctnThrshld ")
        println(conditionsfile,"μ              $μ              ")
        println(conditionsfile,"kT             $kT             ")
        println(conditionsfile,"ϵLJ            $ϵLJ            ")
        println(conditionsfile,"k              $k              ")
        println(conditionsfile,"rₑ             $rₑ             ")
        println(conditionsfile,"Ebend          $Ebend          ")
        println(conditionsfile,"D              $D              ")
        println(conditionsfile,"tMax           $tMax           ")
        println(conditionsfile,"outputInterval $outputInterval ")
        println(conditionsfile,"boxSize        $boxSize        ")
    end

    return foldername
end

export createRunDirectory

end


"""
Holds a specification for composing a simple species.
"""
type SimpleSpeciesDefinition <: SpeciesDefinition
  """
  The fitness scheme used by members of this species.
  Defaults to (maximised) scalar fitness if unspecified.
  """
  fitness::FitnessScheme

  """
  The representation used by members of this species.
  """
  representation::RepresentationDefinition

  SimpleSpeciesDefinition() =
    new(fitness.scalar())
end

"""
Composes a simple specification from a provided specification.
Operates by transforming the simple specification into a complex one and then
composing that specification.
"""
function compose!(spec::SimpleSpeciesDefinition)
  cspec = ComplexSpeciesDefinition()
  genome = stage("genome", spec.representation)
  cspec.stages = [genome]
  cspec.fitness = spec.fitness
  compose!(cspec)
end

"""
A simple species uses only a single representation, and defaults to using
scalar fitness if no fitness scheme is specified. For most problems, simple
species will suffice.

The genome of an individual belonging to a simple species may be retrieved
through its `genome` property.

**Properties:**

* `representation::RepresentationDefinition`, the representation used by
members of this species.
* `fitness::FitnessScheme`, the fitness scheme used by members of this
species. Defaults to (maximised) scalar fitness if unspecified.
"""
function simple(def::Function)
  spec = SimpleSpeciesDefinition()
  def(spec)
  spec
end


struct FreeFlow
    rs # Riemann solver
end

function calcLeftFlux(bc::FreeFlow,
                      state::HydroSnapshot,
                      edge_velocities::Array{Float64,1},
                      eos,
                      cached)::Conserved
    rpcond = RPCond(state.cells[1].density,
                    state.cells[1].pressure,
                    state.cells[1].velocity,
                    cached["sound_speeds"][1],
                    cached["energies"][1])
    return calcFlux(bc.rs,
                    rpcond,
                    rpcond,
                    edge_velocities[1],
                    eos)
end

function calcRightFlux(bc::FreeFlow,
                       state::HydroSnapshot,
                       edge_velocities::Array{Float64,1},
                       eos,
                       cached)::Conserved
    rpcond = RPCond(state.cells[end].density,
                    state.cells[end].pressure,
                    state.cells[end].velocity,
                    cached["sound_speeds"][end],
                    cached["energies"][end])
    return calcFlux(bc.rs,
                    rpcond,
                    rpcond,
                    edge_velocities[end],
                    eos)
end


module TKTDsimulations

    import DiffEqBase
    import DifferentialEquations

    include("model.jl")

    export runTK, runTK_MCMC
    export runSD, runSD_MCMC
    export runIT, runIT_MCMC

end


using OpenStreetMapX
using OpenStreetMapXPlot
using LightGraphs
using MetaGraphs
using Luxor
using Colors
import Plots

#=
visualization.jl

this script implements visualization alternatives for LPBasic problems and solutions

=#

#both of these work ok, but are quite slow and the osm information isn't really useful
export PlotInstance, PlotSolution

"""
    function PlotInstance(instance::ProblemInstance, osmpath::String)
Return a plot of the given instance using the given Open Street Map information.

Substantially slower than the PlotInstance method not using OSM information, but the result is better looking.
"""
function PlotInstance(instance::ProblemInstance, osmpath::String)
    map_data = OpenStreetMapX.get_map_data(osmpath)

    Plots.gr() #using the GR back-end
    p = OpenStreetMapXPlot.plotmap(map_data,width=600,height=400);

    # "convert" from our instance's network to a format frienldy to the OpenStreetMapX library

    for line in instance.lines

        #a route is simply an array of MapData nodes
        route = Array{Int64, 1}() #where T = Int64
        for stop_tuple in line.walk
            from, to = stop_tuple
            #latitude and longitude for to and from stops
            from_coord = (get_prop(instance.network, vertices(instance.network)[from], :latitude), get_prop(instance.network, vertices(instance.network)[from], :longitude))
            to_coord = (get_prop(instance.network, vertices(instance.network)[to], :latitude), get_prop(instance.network, vertices(instance.network)[to], :longitude))

            #the DataMap nodes
            from_node = OpenStreetMapXPlot.point_to_nodes(from_coord, map_data)
            to_node = OpenStreetMapXPlot.point_to_nodes(to_coord, map_data)

            #append to route
            push!(route, from_node)

        end

        #add route to plot
        addroute!(p,map_data,route,route_color="red",start_name="",end_name="");

    end

    return p
end

"""
    PlotSolution(instance::ProblemInstance, solution::Array{Bool, 1}, osmpath::String)
Identical to PlotInstance(instance::ProblemInstance, osmpath::String), but paint unused lines differently

Substantially slower than the PlotSolution method not using OSM information, but the result is better looking.
"""
function PlotSolution(instance::ProblemInstance, solution::Array{Bool, 1}, osmpath::String)
    map_data = OpenStreetMapX.get_map_data(osmpath)

    Plots.gr() #using the GR back-end
    p = OpenStreetMapXPlot.plotmap(map_data,width=600,height=400);

    # "convert" from our instance's network to a format frienldy to the OpenStreetMapX library

    line_i = 1
    for line in instance.lines

        #a route is simply an array of MapData nodes
        route = Array{Int64, 1}() #where T = Int64
        for stop_tuple in line.walk
            from, to = stop_tuple
            #latitude and longitude for to and from stops
            from_coord = (get_prop(instance.network, vertices(instance.network)[from], :latitude), get_prop(instance.network, vertices(instance.network)[from], :longitude))
            to_coord = (get_prop(instance.network, vertices(instance.network)[to], :latitude), get_prop(instance.network, vertices(instance.network)[to], :longitude))

            #the DataMap nodes
            from_node = OpenStreetMapXPlot.point_to_nodes(from_coord, map_data)
            to_node = OpenStreetMapXPlot.point_to_nodes(to_coord, map_data)

            #append to route
            push!(route, from_node)

        end

        #add route to plot
        if solution[line_i]
            addroute!(p,map_data,route,route_color="red",start_name="",end_name="")
        else
            addroute!(p,map_data,route,route_color="blue",start_name="",end_name="")
        end

        line_i +=1

    end

    return p

end

#visualization alternative using Luxor
"""
    PlotInstance(instance::ProblemInstance; out_path::String = "./", should_preview = true)
Return a simple Luxor drawing of the given instance

Substantially faster than the PlotInstance method using Open Street Map information, but draws only simple lines on a blank background.
"""
function PlotInstance(instance::ProblemInstance; out_path::String = "./", should_preview = true)
    PlotSolution(instance, [true for i in 1:length(instance.lines)], out_path = out_path, should_preview = should_preview)
end

"""
    PlotSolution(instance::ProblemInstance, solution::Array{Bool, 1};  <keyword arguments>)
Return a simple Luxor drawing of the given instance, painting lines used in solution differently

# Keywork arguments:
- [out_path]: optional path determining where to save the image. If this isn't set, a time-stamped name will be used
- should_preview=true: should the image be previewed in the default image viewer?
"""
function PlotSolution(instance::ProblemInstance, solution::Array{Bool, 1}; out_path::Union{String, Nothing} = nothing, should_preview = true)
    #min and max of latitudes and longitudes, used for scaling
    min_lat, max_lat = Inf, -Inf
    min_lon, max_lon = Inf, -Inf
    for i in 1:nv(instance.network)
        lat, lon = get_prop(instance.network, vertices(instance.network)[i], :latitude), get_prop(instance.network, vertices(instance.network)[i], :longitude)
        min_lat = min(min_lat, lat)
        max_lat = max(max_lat, lat)
        min_lon = min(min_lon, lon)
        max_lon = max(max_lon, lon)
    end

    #not sure if this is correct
    if max_lat - min_lat > max_lon - min_lon
        #latitude has bigger span than longitude
        slat = 600 / (max_lat - min_lat)
        slon = slat
    else
        #longitude has bigger span than latitude
        slon = 600 / (max_lon - min_lon)
        slat = slon
    end

    Drawing(600, 600, :png, if out_path != nothing out_path else "./$(Dates.minute(now())).png" end)
    origin()
     #default size is 600x600

        cx, cy = get_prop(instance.network, :buffer_center_lat), get_prop(instance.network, :buffer_center_lon)

        #random, different Colors
        # if seed != nothing
        #     dcolors = Colors.distinguishable_colors(length(instance.lines), seed)
        # else
        #     dcolors = Colors.distinguishable_colors(length(instance.lines))
        # end

        sf = 1
        println("slat: $(slat)  e slon:$(slon)")
        for i in 1:(length(instance.lines))
            line = instance.lines[i]
            Luxor.setcolor(if solution[i] "red" else "black" end) #red if included in solution, blue otherwise
            Luxor.setdash("solid")
            #Luxor.setcolor(if solution[i] dcolors[i] else "black" end)
            #Luxor.setdash(if solution[i] "solid" else "dashed" end)

            #Luxor.setcolor(dcolors[i])
            #Luxor.setopacity(if solution[i] 1.0 else 0.5 end)

            #if solution[i] action = :fillstroke else action = :dotted end
            #action = if (solution[i]) :fillstroke else :dotted end
            #Luxor.setcolor("black")
            for j in 1:(length(line.walk) - 1) #stop at -1 because we'll draw a line between every two stops
                from, to = line.walk[j]
                p1_x = slat * sf * (get_prop(instance.network, vertices(instance.network)[from], :latitude) - cx)
                p1_y = slon * -sf * (get_prop(instance.network, vertices(instance.network)[from], :longitude) -cy) #negated bc y axis points downwards

                p2_x = slat * sf * (get_prop(instance.network, vertices(instance.network)[to], :latitude) -cx)
                p2_y = slon * -sf * (get_prop(instance.network, vertices(instance.network)[to], :longitude) -cy)

                #might raise ERROR: cant draw arrow between two identical points. Maybe bc of imprecision?
                try
                    #Luxor.arrow(Luxor.Point(p1_x, p1_y), Luxor.Point(p2_x, p2_y))
                    Luxor.line(Luxor.Point(p1_x, p1_y), Luxor.Point(p2_x, p2_y), :stroke)
                catch e
                    @warn "ERROR: cant draw arrow between two identical points????"
                end
            end
        end

    finish()
    !should_preview || preview()
end

#to-do: plotting based on edge frequency, and not lines


module Manifests
include("DBFs.jl"); using .DBFs;
include("PointArrays.jl"); using .PointArrays;

# using JLD2
using BigArrays
using BigArrays.GSDicts

using ..RealNeuralNetworks.NodeNets
#import ..RealNeuralNetworks.NodeNets.DBFs
#import ..RealNeuralNetworks.NodeNets.PointArrays
using OffsetArrays
using JSON
#import Distributed: pmap

const MIP_LEVEL = 4

export Manifest

struct Manifest
    # the bigarray for cutout of segmentation
    ba          ::AbstractBigArray
    # the id of object
    obj_id      ::Integer
    # the unit range list for cutout
    rangeList   ::Vector
end

"""
Parameters:
    dir
"""
function Manifest(manifestDirPath::AbstractString, manifestKey::AbstractString,
                  bigArrayPath::AbstractString, mip::Integer )
    println("big array path: ", bigArrayPath)

    ba = BigArray( GSDict( bigArrayPath ); mip=mip )
    h = GSDict( manifestDirPath )
    d = h[manifestKey]
    ranges = d[:fragments]
    Manifest( ranges, ba )
end

"""
example: ["770048087:0:2968-3480_1776-2288_16912-17424"]
"""
function Manifest( ranges::Vector, ba::BigArray{D,T} ) where {D,T}
    obj_id = Meta.parse( split(ranges[1], ":")[1] )
    obj_id = convert(T, obj_id)
    ranges = map(x-> split(x,":")[end], ranges)
    rangesList = map( BigArrays.Indexes.string2unit_range, ranges )
    # convert from z,y,x to x,y,z
    # this is due to a bug in chunkflow, we should not need this in the future
    rangesList = map( reverse, rangesList)
    @show rangesList
    Manifest( ba, obj_id, rangesList )
end

function Base.length(self::Manifest) length(get_range_list(self)) end

function get_range_list(self::Manifest) self.rangeList end

"""
the voxel offset in the neuroglancer precomputed info file
"""
function get_voxel_offset(self::Manifest)
    voxel_offset =  self.ba.kvStore.configDict[:offset]
    #real voxel offset should based on the highest resolution 5x5x45
    # we are using mip level 4 with (80x80x45 nm) now
    # voxel_offset = map((x,y)->UInt32(x*y), voxel_offset, (2^MIP_LEVEL, 2^MIP_LEVEL,1))
    voxel_offset = Vector{UInt32}(voxel_offset)
    @show voxel_offset
    return (voxel_offset...,)
end

"""
iterate the chunks containing the neuron with specified neuronId
build point cloud and dbf when iterating the chunks
"""
function trace(self::Manifest)
    println("extract point clouds and distance from boundary fields ...")

    pointCloudDBFList = map(x->_get_point_cloud_dbf(self, x), self.rangeList )

    pointClouds = map( x->x[1], pointCloudDBFList )
    pointCloud = vcat(pointClouds ...)
    dbfs = map(x->x[2], pointCloudDBFList)
    dbf = vcat(dbfs ...)
    # save temporal variables for debug
    # @save "/tmp/$(neuronId).jld" pointClouds, pointCloud, dbf
    println("skeletonization from global point cloud and dbf using RealNeuralNetworks algorithm...")
    @time nodeNet = NodeNet(pointCloud; dbf=dbf)
    return nodeNet
end

function _get_point_cloud_dbf(self::Manifest, ranges::Vector)
    # example: [2456:2968, 1776:2288, 16400:16912]
    offset = (map(x-> UInt32(x.start-1), ranges)...,)
    seg = self.ba[ranges...] |> parent
    bin_im = DBFs.create_binary_image( seg; obj_id = self.obj_id )
    @assert any(bin_im)
    point_cloud = PointArrays.from_binary_image( bin_im )
    # distance from boundary field
    dbf = DBFs.compute_DBF(point_cloud, bin_im)
    PointArrays.add_offset!(point_cloud, offset)
    # no need to use voxel_offset since the file name encoded the global coordinate
    # PointArrays.add_offset!(point_cloud, get_voxel_offset(self))
    return point_cloud, dbf
end

end # module


module UnixTimes

using Dates
using TimeZones

export UnixTime
export unix_now
export UNIX_EPOCH

struct UnixTime <: Dates.AbstractDateTime
    instant::Dates.UTInstant{Nanosecond}
end

function UnixTime(
    y::Integer,
    m::Integer = 1,
    d::Integer = 1,
    h::Integer = 0,
    mi::Integer = 0,
    s::Integer = 0,
    ms::Integer = 0,
    us::Integer = 0,
    ns::Integer = 0)
    convert(UnixTime, DateTime(y, m, d, h, mi, s, ms)) + Nanosecond(us * 1000 + ns)
end

const UNIX_EPOCH = UnixTime(Dates.UTInstant(Nanosecond(0)))

Dates.days(x::UnixTime) = Dates.days(convert(DateTime, x))
Dates.hour(x::UnixTime) = mod(fld(Dates.value(x), 3600_000_000_000), 24)
Dates.minute(x::UnixTime) = mod(fld(Dates.value(x), 60_000_000_000), 60)
Dates.second(x::UnixTime) = mod(fld(Dates.value(x), 1_000_000_000), 60)
Dates.millisecond(x::UnixTime) = mod(fld(Dates.value(x), 1_000_000), 1000)
Dates.microsecond(x::UnixTime) = mod(fld(Dates.value(x), 1_000), 1000)
Dates.nanosecond(x::UnixTime) = mod(Dates.value(x), 1_000)

Base.:+(x::UnixTime, p::Period) =
    UnixTime(Dates.UTInstant(x.instant.periods + Nanosecond(Dates.tons(p))))

Base.:-(x::UnixTime, p::Period) = x + (-p)

function Base.:+(x::UnixTime, p::Union{Month, Year})
    trunc_ns = mod(Dates.value(x), 1_000_000)
    convert(UnixTime, convert(DateTime, x) + p) + Nanosecond(trunc_ns)
end

function Dates.DateTime(x::UnixTime)
    instant_ms = Dates.UNIXEPOCH + div(x.instant.periods.value, 1_000_000)
    DateTime(Dates.UTM(instant_ms))
end

Dates.Date(x::UnixTime) = Date(DateTime(x))
Dates.Time(x::UnixTime) = Time(Nanosecond(Dates.value(x)))

Base.convert(::Type{DateTime}, x::UnixTime) = DateTime(x)

function UnixTime(x::DateTime)
    instant_ns = (Dates.value(x) - Dates.UNIXEPOCH) * 1_000_000
    UnixTime(Dates.UTInstant(Nanosecond(instant_ns)))
end

UnixTime(x::Date) = UnixTime(DateTime(x))

Base.convert(::Type{UnixTime}, x::DateTime) = UnixTime(x)

UnixTime(x::ZonedDateTime) = UnixTime(DateTime(x, UTC))
ZonedDateTime(x::UnixTime, tz::TimeZone) = ZonedDateTime(DateTime(x), tz; from_utc = true)

function Base.show(io::IO, x::UnixTime)
    xdt = convert(DateTime, x)
    print(io, Dates.format(xdt, dateformat"yyyy-mm-ddTHH:MM:SS.sss"))
    v = x.instant.periods.value
    d = 100_000
    for i in 1:6
        print(io, div(v, d) % 10 + '0')
        d = div(d, 10)
    end
    nothing
end

function Base.floor(x::UnixTime, p::Union{DatePeriod, TimePeriod})
    convert(UnixTime, floor(convert(DateTime, x), p))
end

Dates.guess(a::UnixTime, b::UnixTime, c) = guess(DateTime(a), DateTime(b), c)

Dates.default_format(::Type{UnixTime}) = nothing
Dates.format(x::UnixTime, fmt::Nothing) = string(x)

if Sys.islinux()
    struct LinuxTimespec
        seconds::Clong
        nanoseconds::Cuint
    end
    @inline function unix_now()
        ts = Ref{LinuxTimespec}()
        ccall(:clock_gettime, Cint, (Cint, Ref{LinuxTimespec}), 0, ts)
        x = ts[]
        UnixTime(Dates.UTInstant(Nanosecond(x.seconds * 1_000_000_000 + x.nanoseconds)))
    end
else
    @inline function unix_now()
        tv = Libc.TimeVal()
        UnixTime(Dates.UTInstant(Nanosecond(tv.sec * 1_000_000_000 + tv.usec * 1_000)))
    end
end

end


"""
    load_rda(rda_file)

Load header dictionary and spectroscopy data from a Siemens .rda file.  Returns
`header,data`, where `header` is a simple dictionary containing the ASCII
header, and `data` is the time domain induced magnetization signal.

Single voxel spectroscopy data will be loaded as a vector containing the single
magnentization signal.

For spectroscopy with a spatial component (that is, CSI data with at least one
header field `CSIMatrixSize[i]` not equal to 1), an array of size N×I×K×J will
be returned where I,J,K are the CSI indices 0,1,2 and the FID length is N.
"""
function load_rda(io::IO)
    firstline = readline(io)
    if firstline != ">>> Begin of header <<<"
        error("Expected RDA header, got: $firstline")
    end
    # Read header
    header = OrderedDict{String,String}()
    while true
        if eof(io)
            throw(EOFError("RDA file ended while reading header"))
        end
        # TODO: Is the encoding correct here?
        line = readline(io)
        if line == ">>> End of header <<<"
            break
        end
        # To split array indices...
        # r"^(\w+)(?:\[(\d+)\])?:\s*(.*)$",
        m = match(r"^(\S+)\s*:\s*(.*)$", line)
        if m == nothing
            @error "BUG: could not match RDA header line \"$line\""
        end
        header[m[1]] = m[2]
    end
    vecsize = parse(Int, header["VectorSize"])
    csi_dims = [parse(Int, get(header, "CSIMatrixSize[$i]", "1")) for i=0:2]
    # RDA should always be channel combined (& averaged?), so expect a single
    # FID per voxel.
    data_dims = prod(csi_dims) == 1 ? (vecsize,) : (vecsize, csi_dims...)
    raw_data = read(io)
    if sizeof(raw_data) != prod(data_dims)*sizeof(ComplexF64)
        error("""
              Wrong amount of raw data in .rda file.
              Expected $data_dims ComplexF64's. Got $(length(raw_data)) bytes instead.""")
    end
    data = Array{ComplexF64}(undef, data_dims)
    copy!(data, reshape(reinterpret(ComplexF64, raw_data), data_dims))
    header, data
end

load_rda(fname::AbstractString) = open(load_rda, fname)

"""
    save_rda(file, header, data)

Save a single FID as Siemens format .rda.  `header` should be a dictionary of
key values for the rda ASCII header.
"""
function save_rda(io::IO, header::AbstractDict, data)
    if parse(Int, header["VectorSize"]) != size(data,1)
        error("rda header VectorSize mismatches with length(data)")
    end
    for dim in 2:4
        csi_key = "CSIMatrixSize[$(dim-2)]"
        h_size = parse(Int, get(header, csi_key, "1"))
        d_size = size(data, dim)
        if h_size != d_size
            error("rda header entry $csi_key=$h_size is inconsistent with data dimension $dim of length $d_size")
        end
    end
    write(io, ">>> Begin of header <<<\r\n")
    for (key,val) in header
        write(io, "$key: $val\r\n")
    end
    write(io, ">>> End of header <<<\r\n")
    write(io, ComplexF64.(data))
    nothing
end

save_rda(fname::String, header, data) = open(io->save_rda(io, header, data), fname, "w")


const VALID_BAR_SIZES = (
    Second(1), Second(5), Second(10), Second(15), Second(30),
    Minute(1), Minute(2), Minute(3), Minute(5), Minute(10), Minute(15), Minute(20), Minute(30),
    Hour(1), Hour(2), Hour(3), Hour(4), Hour(8),
    Day(1),
    Week(1),
    Month(1),
)

const VALID_DURATION_UNITS = (Second, Day, Week, Month, Year)

const VALID_HISTORICAL_DATA_TYPE = (
    "TRADES",
    "MIDPOINT",
    "BID",
    "ASK",
    "BID_ASK",
    "ADJUSTED_LAST",
    "HISTORICAL_VOLATILITY",
    "OPTION_IMPLIED_VOLATILITY",
    "REBATE_RATE",
    "FEE_RATE",
    "YIELD_BID",
    "YIELD_ASK",
    "YIELD_BID_ASK",
    "YIELD_LAST",
)

# TODO Timezone
function historical_data(
    ib::IB,
    asset::Asset;
    end_datetime::DateTime = now(),
    duration::TimePeriod = Week(1),
    barsize::TimePeriod = Minute(1),
    use_regular_trading_hours::Bool=true,
    datatype::String = "MIDPOINT",
)
    check_end_datetime(end_datetime)
    check_duration(dt)
    check_barsize(sz)
    check_historical_data_type(datatype)
    datas = []
    while true
        data = ib.reqHistoricalData(
            asset,
            endDateTime = end_datetime,
            durationStr = duration,
            barsizeSetting = barsize,
            whatToShow = datatype,
            useRTH = use_regular_trading_hours
        )
        @info data
        data === nothing && break
        push!(datas, data)
        @info data[1].data
        break
    end
end

function duration_str(dt::TimePeriod)
    check_duration(dt)
    unit = dt isa Second ? "S" :
        dt isa Day ? "D" :
        dt isa Week ? "W" :
        dt isa Month ? "M" :
        dt isa Year ? "Y" :
        error("This shouldn't happen")
    return "$(dt.value) $unit"
end

function barsize_str(sz::TimePeriod)
    check_barsize(sz)
    unit = sz isa Second ? "secs" :
        sz isa Minute ? (sz == Minute(1) ? "min" : "mins") :
        sz isa Hour ? (sz == Hour(1) ? "hour" : "hours") :
        sz isa Day ? "day" :
        sz isa Week ? "week" :
        sz isa Month ? "month" :
        error("This shouldn't happen")
    return "$(dt.value) $unit"
end

function check_end_datetime(t::DateTime)
    t > now() && error("End date time cannot be in the future: $t")
    return nothing
end

function check_duration(dt::TimePeriod)
    idx = findfirst(T -> typeof(dt) === T, VALID_DURATION_UNITS)
    idx === nothing && error("Invalid duration, expected unit of one of $(VALID_DURATION_UNITS)")
    return nothing
end

function check_barsize(sz::TimePeriod)
    Tlargest = typeof(first(canonicalize(sz).periods))
    sz = round(sz, Tlargest)
    idx = findfirst(x -> x == sz, VALID_BAR_SIZES)
    idx === nothing && error("Invalid dt $sz, expected one of $VALID_BAR_SIZES")
    return nothing
end

function check_historical_data_type(type::String)
    idx = findfirst(t -> t == type, VALID_HISTORICAL_DATA_TYPE)
    idx === nothing && error("Invalid historical data type, expected unit of one of $(VALID_HISTORICAL_DATA_TYPE)")
    return nothing
end


# type
mutable struct DQMCStack{GreensEltype<:Number, HoppingEltype<:Number} <: AbstractDQMCStack
  eye_flv::Matrix{Float64}
  eye_full::Matrix{Float64}
  ones_vec::Vector{Float64}

  u_stack::Array{GreensEltype, 3}
  d_stack::Matrix{Float64}
  t_stack::Array{GreensEltype, 3}

  Ul::Matrix{GreensEltype}
  Ur::Matrix{GreensEltype}
  Dl::Vector{Float64}
  Dr::Vector{Float64}
  Tl::Matrix{GreensEltype}
  Tr::Matrix{GreensEltype}

  greens::Matrix{GreensEltype}
  greens_temp::Matrix{GreensEltype}
  log_det::Float64 # contains logdet of greens_{mc.p.slices+1} === greens_1
                            # after we calculated a fresh greens in propagate()

  U::Matrix{GreensEltype}
  D::Vector{Float64}
  T::Matrix{GreensEltype}
  tmp::Matrix{GreensEltype}

  ranges::Array{UnitRange, 1}
  n_elements::Int
  current_slice::Int # running internally over 0:mc.p.slices+1, where 0 and mc.p.slices+1 are artifcial to prepare next sweep direction.
  direction::Int

  # # -------- Global update backup
  # gb_u_stack::Array{GreensEltype, 3}
  # gb_d_stack::Matrix{Float64}
  # gb_t_stack::Array{GreensEltype, 3}

  # gb_greens::Matrix{GreensEltype}
  # gb_log_det::Float64

  # gb_conf::Array{Float64, 3}
  # # --------

  # preallocated, reused arrays
  curr_U::Matrix{GreensEltype}
  eV::Matrix{GreensEltype}

  # hopping matrices
  hopping_matrix_exp::Matrix{HoppingEltype} # mu included
  hopping_matrix_exp_inv::Matrix{HoppingEltype} # mu included

  # checkerboard hopping matrices
  checkerboard::Matrix{Int} # src, trg, bondid
  groups::Vector{UnitRange}
  n_groups::Int
  chkr_hop_half::Vector{SparseMatrixCSC{HoppingEltype, Int64}}
  chkr_hop_half_inv::Vector{SparseMatrixCSC{HoppingEltype, Int64}}
  chkr_hop_half_dagger::Vector{SparseMatrixCSC{HoppingEltype, Int64}}
  chkr_hop::Vector{SparseMatrixCSC{HoppingEltype, Int64}} # without prefactor 0.5 in matrix exponentials
  chkr_hop_inv::Vector{SparseMatrixCSC{HoppingEltype, Int64}}
  chkr_hop_dagger::Vector{SparseMatrixCSC{HoppingEltype, Int64}}
  chkr_mu_half::SparseMatrixCSC{HoppingEltype, Int64}
  chkr_mu_half_inv::SparseMatrixCSC{HoppingEltype, Int64}
  chkr_mu::SparseMatrixCSC{HoppingEltype, Int64}
  chkr_mu_inv::SparseMatrixCSC{HoppingEltype, Int64}

  DQMCStack{GreensEltype, HoppingEltype}() where {GreensEltype<:Number, HoppingEltype<:Number} = begin
    # @assert isleaftype(GreensEltype);
    # @assert isleaftype(HoppingEltype);
    @assert isconcretetype(GreensEltype);
    @assert isconcretetype(HoppingEltype);
    new()
  end
end

# type helpers
geltype(::Type{DQMCStack{G,H}}) where {G,H} = G
heltype(::Type{DQMCStack{G,H}}) where {G,H} = H
geltype(mc::DQMC{M, CB, CT, S}) where {M, CB, CT, S} = geltype(S)
heltype(mc::DQMC{M, CB, CT, S}) where {M, CB, CT, S} = heltype(S)

# type initialization
function initialize_stack(mc::DQMC)
  GreensEltype = geltype(mc)
  HoppingEltype = heltype(mc)
  N = mc.model.l.sites
  flv = mc.model.flv

  mc.s.eye_flv = Matrix{Float64}(I, flv,flv)
  mc.s.eye_full = Matrix{Float64}(I, flv*N,flv*N)
  mc.s.ones_vec = ones(flv*N)

  mc.s.n_elements = convert(Int, mc.p.slices / mc.p.safe_mult) + 1

  mc.s.u_stack = zeros(GreensEltype, flv*N, flv*N, mc.s.n_elements)
  mc.s.d_stack = zeros(Float64, flv*N, mc.s.n_elements)
  mc.s.t_stack = zeros(GreensEltype, flv*N, flv*N, mc.s.n_elements)

  mc.s.greens = zeros(GreensEltype, flv*N, flv*N)
  mc.s.greens_temp = zeros(GreensEltype, flv*N, flv*N)

  mc.s.Ul = Matrix{GreensEltype}(I, flv*N, flv*N)
  mc.s.Ur = Matrix{GreensEltype}(I, flv*N, flv*N)
  mc.s.Tl = Matrix{GreensEltype}(I, flv*N, flv*N)
  mc.s.Tr = Matrix{GreensEltype}(I, flv*N, flv*N)
  mc.s.Dl = ones(Float64, flv*N)
  mc.s.Dr = ones(Float64, flv*N)

  mc.s.U = zeros(GreensEltype, flv*N, flv*N)
  mc.s.D = zeros(Float64, flv*N)
  mc.s.T = zeros(GreensEltype, flv*N, flv*N)
  mc.s.tmp = zeros(GreensEltype, flv*N, flv*N)

  # # Global update backup
  # mc.s.gb_u_stack = zero(mc.s.u_stack)
  # mc.s.gb_d_stack = zero(mc.s.d_stack)
  # mc.s.gb_t_stack = zero(mc.s.t_stack)
  # mc.s.gb_greens = zero(mc.s.greens)
  # mc.s.gb_log_det = 0.
  # mc.s.gb_conf = zero(mc.conf)

  mc.s.ranges = UnitRange[]

  for i in 1:mc.s.n_elements - 1
    push!(mc.s.ranges, 1 + (i - 1) * mc.p.safe_mult:i * mc.p.safe_mult)
  end

  mc.s.curr_U = zero(mc.s.U)
  mc.s.eV = zeros(GreensEltype, flv*N, flv*N)

  # mc.s.hopping_matrix_exp = zeros(HoppingEltype, flv*N, flv*N)
  # mc.s.hopping_matrix_exp_inv = zeros(HoppingEltype, flv*N, flv*N)
  nothing
end

# hopping
function init_hopping_matrices(mc::DQMC{M,CB}, m::Model) where {M, CB<:Checkerboard}
  init_hopping_matrix_exp(mc, m)
  CB <: CheckerboardTrue && init_checkerboard_matrices(mc, m)
  nothing
end
function init_hopping_matrix_exp(mc::DQMC, m::Model)
  N = m.l.sites
  flv = m.flv
  dtau = mc.p.delta_tau

  T = hopping_matrix(mc, m)
  size(T) == (flv*N, flv*N) || error("Hopping matrix should have size "*
                                "$((flv*N, flv*N)) but has size $(size(T)) .")
  mc.s.hopping_matrix_exp = exp(-0.5 * dtau * T)
  mc.s.hopping_matrix_exp_inv = exp(0.5 * dtau * T)
  nothing
end

# checkerboard
rem_eff_zeros!(X::AbstractArray) = map!(e -> abs.(e)<1e-15 ? zero(e) : e,X,X)
function init_checkerboard_matrices(mc::DQMC, m::Model)
  s = mc.s
  l = m.l
  flv = m.flv
  H = heltype(mc)
  N = m.l.sites
  dtau = mc.p.delta_tau
  mu = m.mu

  s.checkerboard, s.groups, s.n_groups = build_checkerboard(l)
  n_grps = s.n_groups
  cb = s.checkerboard

  T = reshape(hopping_matrix(mc, m), (N, flv, N, flv))

  s.chkr_hop_half = Vector{SparseMatrixCSC{H, Int}}(undef, n_grps)
  s.chkr_hop_half_inv = Vector{SparseMatrixCSC{H, Int}}(undef, n_grps)
  s.chkr_hop = Vector{SparseMatrixCSC{H, Int}}(undef, n_grps)
  s.chkr_hop_inv = Vector{SparseMatrixCSC{H, Int}}(undef, n_grps)

  for (g, gr) in enumerate(s.groups)
    Tg = zeros(H, N, flv, N, flv)
    for i in gr
      src, trg = cb[1:2,i]
      for f1 in 1:flv, f2 in 1:flv
        Tg[trg, f1, src, f2] = T[trg, f1, src, f2]
      end
    end

    Tgg = reshape(Tg, (N*flv, N*flv))
    s.chkr_hop_half[g] = sparse(rem_eff_zeros!(exp(-0.5 * dtau * Tgg)))
    s.chkr_hop_half_inv[g] = sparse(rem_eff_zeros!(exp(0.5 * dtau * Tgg)))
    s.chkr_hop[g] = sparse(rem_eff_zeros!(exp(- dtau * Tgg)))
    s.chkr_hop_inv[g] = sparse(rem_eff_zeros!(exp(dtau * Tgg)))
  end

  s.chkr_hop_half_dagger = adjoint.(s.chkr_hop_half)
  s.chkr_hop_dagger = adjoint.(s.chkr_hop)

  mus = diag(reshape(T, (N*flv, N*flv)))
  s.chkr_mu_half = spdiagm(0 => exp.(-0.5 * dtau * mus))
  s.chkr_mu_half_inv = spdiagm(0 => exp.(0.5 * dtau * mus))
  s.chkr_mu = spdiagm(0 => exp.(-dtau * mus))
  s.chkr_mu_inv = spdiagm(0 => exp.(dtau * mus))

  # hop_mat_exp_chkr = foldl(*,s.chkr_hop_half) * sqrt.(s.chkr_mu)
  # r = effreldiff(s.hopping_matrix_exp,hop_mat_exp_chkr)
  # r[find(x->x==zero(x),hop_mat_exp_chkr)] = 0.
  # println("Checkerboard - Exact ≈ ", round(maximum(absdiff(s.hopping_matrix_exp,hop_mat_exp_chkr)), 4))
  nothing
end

# stack construction
"""
Build stack from scratch.
"""
function build_stack(mc::DQMC)
  mc.s.u_stack[:, :, 1] = mc.s.eye_full
  mc.s.d_stack[:, 1] = mc.s.ones_vec
  mc.s.t_stack[:, :, 1] = mc.s.eye_full

  @inbounds for i in 1:length(mc.s.ranges)
    add_slice_sequence_left(mc, i)
  end

  mc.s.current_slice = mc.p.slices + 1
  mc.s.direction = -1

  nothing
end
"""
Updates stack[idx+1] based on stack[idx]
"""
function add_slice_sequence_left(mc::DQMC, idx::Int)
  copyto!(mc.s.curr_U, mc.s.u_stack[:, :, idx])

  # println("Adding slice seq left $idx = ", mc.s.ranges[idx])
  for slice in mc.s.ranges[idx]
    multiply_slice_matrix_left!(mc, mc.model, slice, mc.s.curr_U)
  end

  mc.s.curr_U *= spdiagm(0 => mc.s.d_stack[:, idx])
  mc.s.u_stack[:, :, idx + 1], mc.s.d_stack[:, idx + 1], T = udt(mc.s.curr_U)
  mc.s.t_stack[:, :, idx + 1] =  T * mc.s.t_stack[:, :, idx]
end
"""
Updates stack[idx] based on stack[idx+1]
"""
function add_slice_sequence_right(mc::DQMC, idx::Int)
  copyto!(mc.s.curr_U, mc.s.u_stack[:, :, idx + 1])

  for slice in reverse(mc.s.ranges[idx])
    multiply_daggered_slice_matrix_left!(mc, mc.model, slice, mc.s.curr_U)
  end

  mc.s.curr_U *=  spdiagm(0 => mc.s.d_stack[:, idx + 1])
  mc.s.u_stack[:, :, idx], mc.s.d_stack[:, idx], T = udt(mc.s.curr_U)
  mc.s.t_stack[:, :, idx] = T * mc.s.t_stack[:, :, idx + 1]
end

# Green's function calculation
"""
Calculates G(slice) using mc.s.Ur,mc.s.Dr,mc.s.Tr=B(slice)' ... B(M)' and
mc.s.Ul,mc.s.Dl,mc.s.Tl=B(slice-1) ... B(1)
"""
function calculate_greens(mc::DQMC)
    # U, D, T = udt(mc.s.greens) after this
    mc.s.U, mc.s.D, mc.s.T = udt_inv_one_plus(
        UDT(mc.s.Ul, mc.s.Dl, mc.s.Tl),
        UDT(mc.s.Ur, mc.s.Dr, mc.s.Tr),
        tmp = mc.s.U, tmp2 = mc.s.T, tmp3 = mc.s.tmp,
        internaluse = true
    )
    mul!(mc.s.tmp, mc.s.U, Diagonal(mc.s.D))
    mul!(mc.s.greens, mc.s.tmp, mc.s.T)
    mc.s.greens
end

"""
Only reasonable immediately after calculate_greens()!
"""
function calculate_logdet(mc::DQMC)
    mc.s.log_det = real(
        log(complex(det(mc.s.U))) +
        sum(log.(mc.s.D)) +
        log(complex(det(mc.s.T)))
    )
  # mc.s.log_det = real(logdet(mc.s.U) + sum(log.(mc.s.D)) + logdet(mc.s.T))
end

# Green's function propagation
@inline function wrap_greens!(mc::DQMC, gf::Matrix, curr_slice::Int, direction::Int)
  if direction == -1
    multiply_slice_matrix_inv_left!(mc, mc.model, curr_slice - 1, gf)
    multiply_slice_matrix_right!(mc, mc.model, curr_slice - 1, gf)
  else
    multiply_slice_matrix_left!(mc, mc.model, curr_slice, gf)
    multiply_slice_matrix_inv_right!(mc, mc.model, curr_slice, gf)
  end
  nothing
end
# @inline function wrap_greens(mc::DQMC, gf::Matrix,slice::Int,direction::Int)
#   temp = copy(gf)
#   wrap_greens!(mc, temp, slice, direction)
#   return temp
# end
function propagate(mc::DQMC)
  if mc.s.direction == 1
    if mod(mc.s.current_slice, mc.p.safe_mult) == 0
      mc.s.current_slice +=1 # slice we are going to
      if mc.s.current_slice == 1
        mc.s.Ur[:, :], mc.s.Dr[:], mc.s.Tr[:, :] = mc.s.u_stack[:, :, 1], mc.s.d_stack[:, 1], mc.s.t_stack[:, :, 1]
        mc.s.u_stack[:, :, 1] = mc.s.eye_full
        mc.s.d_stack[:, 1] = mc.s.ones_vec
        mc.s.t_stack[:, :, 1] = mc.s.eye_full
        mc.s.Ul[:,:], mc.s.Dl[:], mc.s.Tl[:,:] = mc.s.u_stack[:, :, 1], mc.s.d_stack[:, 1], mc.s.t_stack[:, :, 1]

        calculate_greens(mc) # greens_1 ( === greens_{m+1} )
        calculate_logdet(mc)

      elseif 1 < mc.s.current_slice <= mc.p.slices
        idx = Int((mc.s.current_slice - 1)/mc.p.safe_mult)

        mc.s.Ur[:, :], mc.s.Dr[:], mc.s.Tr[:, :] = mc.s.u_stack[:, :, idx+1], mc.s.d_stack[:, idx+1], mc.s.t_stack[:, :, idx+1]
        add_slice_sequence_left(mc, idx)
        mc.s.Ul[:,:], mc.s.Dl[:], mc.s.Tl[:,:] = mc.s.u_stack[:, :, idx+1], mc.s.d_stack[:, idx+1], mc.s.t_stack[:, :, idx+1]

        if mc.p.all_checks
          mc.s.greens_temp = copy(mc.s.greens)
        end

        wrap_greens!(mc, mc.s.greens_temp, mc.s.current_slice - 1, 1)

        calculate_greens(mc) # greens_{slice we are propagating to}

        if mc.p.all_checks
          greensdiff = maximum(abs.(mc.s.greens_temp - mc.s.greens)) # OPT: could probably be optimized through explicit loop
          if greensdiff > 1e-7
            @printf("->%d \t+1 Propagation instability\t %.1e\n", mc.s.current_slice, greensdiff)
          end
        end

      else # we are going to mc.p.slices+1
        idx = mc.s.n_elements - 1
        add_slice_sequence_left(mc, idx)
        mc.s.direction = -1
        mc.s.current_slice = mc.p.slices+1 # redundant
        propagate(mc)
      end

    else
      # Wrapping
      wrap_greens!(mc, mc.s.greens, mc.s.current_slice, 1)
      mc.s.current_slice += 1
    end

  else # mc.s.direction == -1
    if mod(mc.s.current_slice-1, mc.p.safe_mult) == 0
      mc.s.current_slice -= 1 # slice we are going to
      if mc.s.current_slice == mc.p.slices
        mc.s.Ul[:, :], mc.s.Dl[:], mc.s.Tl[:, :] = mc.s.u_stack[:, :, end], mc.s.d_stack[:, end], mc.s.t_stack[:, :, end]
        mc.s.u_stack[:, :, end] = mc.s.eye_full
        mc.s.d_stack[:, end] = mc.s.ones_vec
        mc.s.t_stack[:, :, end] = mc.s.eye_full
        mc.s.Ur[:,:], mc.s.Dr[:], mc.s.Tr[:,:] = mc.s.u_stack[:, :, end], mc.s.d_stack[:, end], mc.s.t_stack[:, :, end]

        calculate_greens(mc) # greens_{mc.p.slices+1} === greens_1
        calculate_logdet(mc) # calculate logdet for potential global update

        # wrap to greens_{mc.p.slices}
        wrap_greens!(mc, mc.s.greens, mc.s.current_slice + 1, -1)

      elseif 0 < mc.s.current_slice < mc.p.slices
        idx = Int(mc.s.current_slice / mc.p.safe_mult) + 1
        mc.s.Ul[:, :], mc.s.Dl[:], mc.s.Tl[:, :] = mc.s.u_stack[:, :, idx], mc.s.d_stack[:, idx], mc.s.t_stack[:, :, idx]
        add_slice_sequence_right(mc, idx)
        mc.s.Ur[:,:], mc.s.Dr[:], mc.s.Tr[:,:] = mc.s.u_stack[:, :, idx], mc.s.d_stack[:, idx], mc.s.t_stack[:, :, idx]

        if mc.p.all_checks
          mc.s.greens_temp = copy(mc.s.greens)
        end

        calculate_greens(mc)

        if mc.p.all_checks
          greensdiff = maximum(abs.(mc.s.greens_temp - mc.s.greens)) # OPT: could probably be optimized through explicit loop
          if greensdiff > 1e-7
            @printf("->%d \t-1 Propagation instability\t %.1e\n", mc.s.current_slice, greensdiff)
          end
        end

        wrap_greens!(mc, mc.s.greens, mc.s.current_slice + 1, -1)

      else # we are going to 0
        idx = 1
        add_slice_sequence_right(mc, idx)
        mc.s.direction = 1
        mc.s.current_slice = 0 # redundant
        propagate(mc)
      end

    else
      # Wrapping
      wrap_greens!(mc, mc.s.greens, mc.s.current_slice, -1)
      mc.s.current_slice -= 1
    end
  end
  nothing
end


using OrdinaryDiffEq, Test
using DiffEqProblemLibrary.ODEProblemLibrary: importodeproblems; importodeproblems()
import DiffEqProblemLibrary.ODEProblemLibrary: van

prob1 = ODEProblem(van,[0,2.],(0.0,6),inv(0.003))
# out-of-place test
function _van(u, p, t)
  μ = p[1]
  [μ*((1-u[2]^2)*u[1] - u[2]),
   1*u[1]]
end
prob2 = ODEProblem(_van,[0,2.],(0.0,6),inv(0.003))
probArr = [prob1, prob2]
# Test if switching back and forth
is_switching_fb(sol) = maximum(diff(findall(x->x==2, sol.alg_choice))) > 5
for prob in probArr
  alg = AutoTsit5(Rodas5(); maxstiffstep=5, maxnonstiffstep=5, stiffalgfirst=true)
  sol = solve(prob, alg)
  @test length(sol.t) < 280
  @test typeof(alg.algs[sol.alg_choice[1]]) <: Rodas5
  @test is_switching_fb(sol)
  sol = solve(prob, AutoDP5(Rodas5(); maxstiffstep=2, maxnonstiffstep=2,
                            stifftol=11//10, nonstifftol=9//10),
                            reltol=1e-5, abstol=1e-5)
  @test length(sol.t) < 625
  @test is_switching_fb(sol)

  sol = solve(prob,AutoVern6(Kvaerno3(); maxstiffstep=4, maxnonstiffstep=4))
  @test length(sol.t) < 690
  @test is_switching_fb(sol)
  sol = solve(prob,AutoVern7(Hairer42(); maxstiffstep=4, maxnonstiffstep=4))
  @test length(sol.t) < 540
  @test is_switching_fb(sol)
  sol = solve(prob,AutoVern8(Rosenbrock23(); maxstiffstep=4, maxnonstiffstep=4))
  @test length(sol.t) < 910
  @test is_switching_fb(sol)
  sol = solve(prob,AutoVern9(KenCarp3(); maxstiffstep=4, maxnonstiffstep=4))
  @test length(sol.t) < 470
  @test is_switching_fb(sol)
end


#
# Dimension: translates sequences of Symbols, Strings, or Ints to
# ordinal values, so the Dimension keys can be used for lookup in
# Arrays in Mimi. A Dimension can be declared one of several ways:
# dim = Dimension(:foo, :bar, :baz)      # varargs
# dim = Dimension([:foo, :bar, :baz])    # Vector
# dim = Dimension(2010:2100)             # Range
# dim = Dimension(4)                     # Same as 1:4
#
# Similarly, values can be referenced several ways:
# value  = Dimension[:foo]               # single lookup
# values = Dimension[(:foo, :bar)]       # Tuple of values
# values = Dimension[[2010, 2020, 2030]] # Vector of values
# values = Dimension[2010, 2020, 2030]   # Varargs
# values = Dimension[2010:10:2030]       # AbstractRange
#
using DataStructures

key_type(dim::Dimension{T}) where {T <: DimensionKeyTypes} = T

#
# Iteration and basic dictionary methods are delegated to the internal dict
#
Base.length(dim::Dimension) = length(dim.dict)
Base.iterate(dim::Dimension, state...) = iterate(dim.dict, state...)

Base.keys(dim::Dimension)   = keys(dim.dict)
Base.values(dim::Dimension) = values(dim.dict)

Base.get(dim::Dimension, key::Union{Number, Symbol, String}, default::Any) = get(dim.dict, key, default)

Base.getindex(dim::Dimension, key::Colon) = collect(values(dim.dict))

Base.getindex(dim::AbstractDimension, key::Union{Number, Symbol, String}) = getindex(dim.dict, key)

# Support dim[[:foo, :bar, :baz]], dim[(:foo, :bar, :baz)], and dim[2010:2020]
Base.getindex(dim::AbstractDimension, keys::Union{Vector{T} where T, Tuple, AbstractRange}) = [getindex(dim.dict, key) for key in keys]

Base.getindex(dim::AbstractDimension, keys...) = getindex(dim, keys)

Base.length(dim::RangeDimension) = length(dim.range)
Base.iterate(dim::RangeDimension, state...)  = iterate(dim.range, state...)

Base.keys(dim::RangeDimension)   = collect(dim.range)
Base.values(dim::RangeDimension) = collect(1:length(dim.range))

# Get first/last value from OrderedDict of keys
Base.firstindex(dim::AbstractDimension) = dim.dict.keys[1]
Base.lastindex(dim::AbstractDimension)  = dim.dict.keys[length(dim)]

#
# Compute the index of a "key" (e.g., a year) in the range.
#
function Base.get(dim::RangeDimension, key::Int, default::Any=0)
    r = dim.range
    i = key - r.start
    return i == 0 ? 1 : (i % r.step != 0 ? default : 1 + div(i, r.step))
end

# Support dim[[2010, 2020, 2030]], dim[(:foo, :bar, :baz)], and dim[2010:2050]
Base.getindex(dim::RangeDimension, keys::Union{Vector{Int}, Tuple, AbstractRange}) = [get(dim, key, 0) for key in keys]

# Symbols are added to the dim_dict in @defcomp (with value of nothing), but are set later using set_dimension!
has_dim(obj::AbstractCompositeComponentDef, name::Symbol) = (haskey(obj.dim_dict, name) && obj.dim_dict[name] !== nothing)

isuniform(obj::AbstractCompositeComponentDef) = obj.is_uniform

set_uniform!(obj::AbstractCompositeComponentDef, value::Bool) = (obj.is_uniform = value)

dimension(obj::AbstractCompositeComponentDef, name::Symbol) = obj.dim_dict[name]

dim_names(obj::AbstractCompositeComponentDef, dims::Vector{Symbol}) = [dimension(obj, dim) for dim in dims]

dim_count_dict(obj::AbstractCompositeComponentDef) = Dict([name => length(value) for (name, value) in dim_dict(obj)])

dim_counts(obj::AbstractCompositeComponentDef, dims::Vector{Symbol}) = [length(dim) for dim in dim_names(obj, dims)]
dim_count(obj::AbstractCompositeComponentDef, name::Symbol) = length(dimension(obj, name))

dim_keys(obj::AbstractCompositeComponentDef, name::Symbol)   = collect(keys(dimension(obj, name)))
dim_values(obj::AbstractCompositeComponentDef, name::Symbol) = collect(values(dimension(obj, name)))

"""
    set_dimension!(ccd::CompositeComponentDef, name::Symbol, keys::Union{Int, Vector, Tuple, AbstractRange})

Set the values of `ccd` dimension `name` to integers 1 through `count`, if `keys` is
an integer; or to the values in the vector or range if `keys` is either of those types.
"""
function set_dimension!(ccd::AbstractCompositeComponentDef, name::Symbol, keys::Union{Int, Vector, Tuple, AbstractRange})
    redefined = has_dim(ccd, name)
    # if redefined
    #     @warn "Redefining dimension :$name"
    # end

    dim = Dimension(keys)

    if name == :time
        _set_run_period!(ccd, keys[1], keys[end])
        propagate_time!(ccd, dim)
        set_uniform!(ccd, isuniform(keys))
    end

    return set_dimension!(ccd, name, dim)
end

function set_dimension!(obj::AbstractComponentDef, name::Symbol, dim::Dimension)
    dirty!(obj)
    obj.dim_dict[name] = dim

    if name == :time
        for subcomp in compdefs(obj)
            set_dimension!(subcomp, :time, dim)
        end
    end
    return dim
end

function add_dimension!(comp::AbstractComponentDef, name)
    # generally, we add dimension name with nothing instead of a Dimension instance,
    # but in the case of an Int name, we create the "anonymous" dimension on the fly.
    dim = (name isa Int) ? Dimension(name) : nothing
    comp.dim_dict[Symbol(name)] = dim                         # TBD: test this
end

# Note that this operates on the registered comp, not one added to a composite
add_dimension!(comp_id::ComponentId, name) = add_dimension!(compdef(comp_id), name)

function dim_names(ccd::AbstractCompositeComponentDef)
    dims = OrderedSet{Symbol}()             # use a set to eliminate duplicates
    for cd in compdefs(ccd)
        union!(dims, keys(dim_dict(cd)))    # TBD: test this
    end

    return collect(dims)
end

dim_names(comp_def::AbstractComponentDef, datum_name::Symbol) = dim_names(datumdef(comp_def, datum_name))

dim_count(def::AbstractDatumDef) = length(dim_names(def))


using OpenCV

function detect(img::OpenCV.InputArray, cascade)
    rects = OpenCV.detectMultiScale(cascade, img, scaleFactor=1.3, minNeighbors=Int32(4), minSize=OpenCV.Size{Int32}(30, 30), flags=OpenCV.CASCADE_SCALE_IMAGE)
    processed_rects = []
    for rect in rects
        push!(processed_rects, (rect.x, rect.y, rect.width+rect.x, rect.height+rect.y))
    end
    return processed_rects
end

function draw_rects(img, rects, color)
    for x in rects
        OpenCV.rectangle(img, OpenCV.Point{Int32}(x[1], x[2]), OpenCV.Point{Int32}(x[3], x[4]), color, thickness = Int32(2))
    end
end

cap = OpenCV.VideoCapture(Int32(0))

# Replace the paths for the classifiers before running

cascade = OpenCV.CascadeClassifier("haarcascade_frontalface_alt.xml")
nested = OpenCV.CascadeClassifier("haarcascade_eye.xml")

OpenCV.namedWindow("facedetect")

while true
    ret, img = OpenCV.read(cap)
    if ret==false
        print("Webcam stopped")
        break
    end
    gray = OpenCV.cvtColor(img, OpenCV.COLOR_BGR2GRAY)
    gray = OpenCV.equalizeHist(gray)

    rects = detect(gray, cascade)
    vis = copy(img)
    draw_rects(vis, rects, (0.0, 255.0, 0.0))

    if ~OpenCV.empty(nested)
        for x in rects
            roi = view(gray, :, Int(x[1]):Int(x[3]), Int(x[2]):Int(x[4]))
            subrects = detect(roi, nested)
            draw_view = view(vis, :, Int(x[1]):Int(x[3]), Int(x[2]):Int(x[4]))
            draw_rects(draw_view, subrects, (255.0, 0.0, 0.0))
        end
    end

    OpenCV.imshow("facedetect", vis)
    if OpenCV.waitKey(Int32(5))==27
        break
    end
end

OpenCV.release(cap)

OpenCV.destroyAllWindows()

print("Stopped")


"""

    DumbAggregate{P<:Primitive} <: Aggregate

A dumb aggregate is the simplest way to store primitives: it's just a list and all the
primitives are checked for intersection one by one. It's mostly used for testing purposes,
as the `BVHAccelerator` will be faster for any reasonable number of primitives.

"""
struct DumbAggregate{P<:Primitive} <: Aggregate
    primitives::Array{P,1}
    bounds::BoundingBox
end

"""

    DumbAggregate(primitives::Array{P,1})

Construct a `DumbAggregate` from a list of primitives, computing the bounding box.

"""
function DumbAggregate(primitives::Array{P,1}) where P<:Primitive
    BB = BoundingBox([world_bounds(p) for p in primitives])
    return DumbAggregate(primitives, BB)
end

"""
    world_bounds(B::DumbAggregate)

Return the bounding box containing all the primitives in the `DumbAggregate`.

"""
world_bounds(A::DumbAggregate) = A.bounds

"""
    intersect(ray::Ray, A::DumbAggregate)

Find nearest intersection, if any, between `ray` and a `DumbAggregate`.

Returns `Intersection` or `nothing`.

"""
function intersect(ray::Ray, A::DumbAggregate)
    best_tmin = Inf
    best_isect = nothing
    if !intersectP(ray, A.bounds)
        return nothing
    end
    for primitive in A.primitives
        best_isect = update_isect(best_isect, intersect(ray, primitive))
    end
    return best_isect
end

"""
    intersectP(ray::Ray, A::DumbAggregate)

Returns `true` if the given ray intersects any primitive in the `DumbAggregate`, else
returns false. This is much faster to do than [`intersect`](@ref), since it's enough to
find any intersection, not specifically the nearest one.

Possibly the most often called function in the entire ray-tracer.

"""
function intersectP(R::Ray, A::DumbAggregate)
    if !intersectP(R, A.bounds)
        return false
    end
    for prim = A.primitives
        if intersectP(R, prim)
            return true
        end
    end
    return false
end


abstract type AbstractNode end

"""
    Node
This data type holds the basic Node structure. The type T is used to specify the type of the data
stored in the node.
* If `nchild` is `0` the Node is a leaf node.
* If `root` is `False` the Node is a child of another node.
* `inc_length` specifies the length of the incomming branch.
* `binary` specifies the path from the root to the Node. `1` and `0` represent left and right turns respectively.
"""
mutable struct GeneralNode{R<:Real, I<:Integer} <: AbstractNode
    name::String
    mother::Union{GeneralNode{R,I}, Missing}
    children::Vector{GeneralNode{R,I}}
    nchild::I
    root::Bool
    inc_length::R
    binary::String
    num::I
    height::R
    IntExtMap::Vector{I}
    blv::Vector{R}
    stats::Dict{String, Float64}
end # struct Node

"""
    function Node()::GeneralNode{Float64, Int64}
This function will initialize an empty node.
"""
function Node()::GeneralNode{Float64, Int64}
        GeneralNode("no_name", missing, GeneralNode{Float64, Int64}[], 0, true, 1.0, "0",1, 1.0,Int64[], Float64[], Dict{String, Float64}())
end

function Node(name::String)::GeneralNode{Float64, Int64}
        GeneralNode(name, missing, GeneralNode{Float64, Int64}[], 0, true, 1.0, "0", 1, 1.0, Int64[], Float64[], Dict{String, Float64}())
end

function Node(name::String, inc_len::T)::GeneralNode{T, Int64} where T<:Real
    GeneralNode(name, missing, GeneralNode{T, Int64}[], 0, true, inc_len, "0", 1, 1.0, Int64[], Float64[], Dict{String, Float64}())
end

#################### Base functionality ####################

Base.:(==)(x::T, y::T) where T<:AbstractNode = x.num == y.num
Base.size(x::T) where T<:AbstractNode = size(post_order(x))
Base.length(x::T) where T<:AbstractNode = x.nchild

function Base.summary(io::IO, d::N) where N <: GeneralNode
    summary(io, d.name)
end

function Base.show(io::IO, d::N) where N <: GeneralNode
    print(io, "Tree with root:\n")
    show(io, d.name)
    print(io, "\nLength:\n")
    show(io, "text/plain", tree_length(d))
    print(io, "\nHeight:\n")
    show(io, "text/plain", tree_height(d))

end

function showall(io::IO, d::N) where N <: GeneralNode
  show(io, d)
  print(io, "\nNode:\n")
  show(io, "text/plain", d.name)
  print(io, "\n#children:\n")
  show(io, d.nchild)
  print(io, "\nbinary:\n")
  show(io, d.binary)
end


using Vectorize
if VERSION >= v"0.5-"
    using Base.Test
else
    using BaseTestNext
    const Test = BaseTestNext
end

## Test Apple Accelerate
@static if is_apple()
    include("AccelerateTests.jl")
else
    println("Accelerate not present. Aborting Accelerate tests")
end

## Test Yeppp
include("YepppTests.jl")

## Check for presence of VML and test VML
if Libdl.find_library(["libmkl_vml_avx"], ["/opt/intel/mkl/lib"]) != ""
    include("VMLTests.jl")
end

## Running tests over benchmarked Vectorized functions
include("VectorizeTests.jl")


#!/usr/bin/env julia

using Luxor, Test, Random

Random.seed!(42)

function image_testing(fname)
    width, height = 4000, 4000
    margin = 500
    Drawing(width, height, fname)
    origin()
    background("grey25")
    setline(5)
    sethue("green")

    # try a nonpng
    @test_throws ErrorException readpng(dirname(dirname(pathof(Luxor))) * "/test/test-image.jl")

    image = readpng(dirname(dirname(pathof(Luxor))) * "/test/stackoverflow.png")
    w = image.width
    h = image.height
    pagetiles = Tiler(width, height, 7, 9)
    tw = pagetiles.tilewidth/2
    for (pos, n) in pagetiles
        circle(pos, tw, :stroke)
        circle(pos, tw, :clip)
        gsave()
        translate(pos)
        scale(25, 25)
        rotate(rand(0.0:pi/8:2pi))
        placeimage(image, O, centered=true)
        grestore()
        clipreset()
    end
    @test finish() == true
end

function placetesting(fname)
    # make svg file
    Drawing(200, 200, "svgimage.svg")
    origin()
    setline(40)
    sethue("rebeccapurple")
    squircle(O, 80, 80, :strokepreserve)
    sethue("white")
    fillpath()
    fontsize(40)
    sethue("black")
    text("SVG", halign=:center, valign=:middle)
    finish()

    # make png file
    Drawing(200, 200, "pngimage.png")
    origin()
    setline(40)
    sethue("red")
    squircle(O, 80, 80, :strokepreserve)
    sethue("white")
    fillpath()
    fontsize(40)
    sethue("black")
    text("PNG", halign=:center, valign=:middle)
    finish()

    Drawing(1200, 1200, fname)
    origin()
    svgimg = readsvg("svgimage.svg")
    pngimg = readpng("pngimage.png")
    for (pt, n) in Tiler(1200, 1200, 4, 4)
        @layer begin
            translate(pt)
            isodd(n) && placeimage(svgimg, O, centered=true)
            iseven(n) && placeimage(pngimg, O, centered=true)
        end
    end
    finish()
end

fname = "test-image.png"
image_testing(fname)

fname = "test-image-place.png"
placetesting(fname)

println("...finished test: output in $(fname)")


using HDF5

export save_hdf, load_hdf

function save_hdf(hdf_file_path, model; group_name::String="")
    h5open(hdf_file_path, "w") do file
        to_hdf(file, group_name, model)
    end
end

function load_hdf(hdf_file_path; group_name::String="")
    model = h5open(hdf_file_path, "r") do file
        matfac_from_hdf(file, group_name)
    end
    return model
end

####################################
# MatFacModel
####################################

function to_hdf(hdf_file, path::String, model::MatFacModel)

    # Factors
    write(hdf_file, string(path,"/X"), model.X)
    write(hdf_file, string(path,"/Y"), model.Y)

    # covariate coefficients
    if model.instance_covariate_coeff != nothing
        write(hdf_file, string(path,"/instance_covariate_coeff"),
                        model.instance_covariate_coeff)
    end
    to_hdf(hdf_file, string(path,"/instance_covariate_coeff_reg"),
                     model.instance_covariate_coeff_reg)

    # Offsets
    write(hdf_file, string(path,"/instance_offset"), model.instance_offset)
    write(hdf_file, string(path,"/feature_offset"), model.feature_offset)

    # Offset Regularizers
    to_hdf(hdf_file, string(path,"/instance_offset_reg"), model.instance_offset_reg)
    to_hdf(hdf_file, string(path,"/feature_offset_reg"), model.feature_offset_reg)

    # Precisions
    write(hdf_file, string(path,"/feature_precision"), model.feature_precision)

    # Losses
    loss_strs = String[string(typeof(loss)) for loss in model.losses]
    write(hdf_file, string(path, "/loss_types"), loss_strs)
    loss_scales = Float64[loss.scale for loss in model.losses]
    write(hdf_file, string(path, "/loss_scales"), loss_scales)

    # Instance regularizer matrices
    for (i, regmat) in enumerate(model.instance_reg_mats)
        to_hdf(hdf_file, string(path, "/inst_reg/mat_", i), regmat)
    end

    # Feature regularizer matrices
    for (i, regmat) in enumerate(model.feature_reg_mats)
        to_hdf(hdf_file, string(path, "/feat_reg/mat_", i), regmat)
    end

    write(hdf_file, string(path,"/factor_importances"), model.factor_importances)

end

function matfac_from_hdf(hdf_file, path::String)
    # Factors
    X = hdf_file[string(path,"/X")][:,:]
    Y = hdf_file[string(path,"/Y")][:,:]

    # Covariate coefficients
    if "instance_covariate_coeff" in keys(hdf_file[string(path,"/")])
        covariate_coeff = hdf_file[string(path,"/instance_covariate_coeff")][:,:]
    end
    covariate_coeff_reg = spmat_from_hdf(hdf_file, string(path,"/instance_covariate_coeff_reg"))

    # Offsets
    instance_offset = hdf_file[string(path,"/instance_offset")][:]
    feature_offset = hdf_file[string(path,"/feature_offset")][:]

    # Offset Regularizers
    instance_offset_reg = spmat_from_hdf(hdf_file, string(path, "/instance_offset_reg"))
    feature_offset_reg = spmat_from_hdf(hdf_file, string(path, "/feature_offset_reg"))

    # Precisions
    feature_precision = hdf_file[string(path,"/feature_precision")][:]

    # Losses
    loss_names = hdf_file[string(path,"/loss_types")][:]
    loss_scales = hdf_file[string(path,"/loss_scales")][:]
    losses = Loss[eval(Meta.parse(lname))(lscale) for (lname, lscale) in zip(loss_names, loss_scales)]

    # Instance regularizers
    inst_reg_gp = hdf_file[string(path, "/inst_reg")]
    inst_reg_mats = SparseMatrixCSC[spmat_from_hdf(inst_reg_gp, k) for k in sort(keys(inst_reg_gp))]

    # Feature regularizers
    feat_reg_gp = hdf_file[string(path, "/feat_reg")]
    feat_reg_mats = SparseMatrixCSC[spmat_from_hdf(inst_reg_gp, k) for k in sort(keys(feat_reg_gp))]

    # Factor importances
    factor_importances = hdf_file[string(path,"/factor_importances")][:]

    return MatFacModel(X, Y, inst_reg_mats, feat_reg_mats,
                             instance_offset, instance_offset_reg,
                             feature_offset, feature_offset_reg,
                             feature_precision,
                             covariate_coeff, covariate_coeff_reg,
                             losses, factor_importances)
end

##################################
# SparseMatrixCSC
##################################

function to_hdf(hdf_file, path::String, mat::SparseMatrixCSC)

    write(hdf_file, string(path, "/colptr"), mat.colptr)
    write(hdf_file, string(path, "/m"), mat.m)
    write(hdf_file, string(path, "/n"), mat.n)
    write(hdf_file, string(path, "/nzval"), mat.nzval)
    write(hdf_file, string(path, "/rowval"), mat.rowval)

end

function spmat_from_hdf(hdf_file, path::String)
    colptr = read(hdf_file, string(path, "/colptr"))
    m = read(hdf_file, string(path, "/m"))
    n = read(hdf_file, string(path, "/n"))
    nzval = read(hdf_file, string(path, "/nzval"))
    rowval = read(hdf_file, string(path, "/rowval"))

    return SparseMatrixCSC(m, n, colptr, rowval, nzval)
end