"""Convenience iterable for state-dependent controllers"""
struct ControllerIterator{C,E,B}
    controller::C
    env::E
    T::Int
    trajectory::B
    plotiter::Int
    randstart::Bool
    alreadyran::Base.RefValue{Bool}

    function ControllerIterator(
        controller,
        env::AbstractEnvironment;
        T::Integer = 1000,
        plotiter::Integer = 1,
        randstart::Bool = false
    )
        trajectory = (
            states = Array(undef, statespace(env), T),
            observations = Array(undef, observationspace(env), T),
            actions = Array(undef, actionspace(env), T),
            rewards = Array(undef, rewardspace(env), T),
        )
        new{typeof(controller),typeof(env),typeof(trajectory)}(
            controller,
            env,
            T,
            trajectory,
            plotiter,
            randstart,
            Ref(false)
        )
    end
end

function Base.iterate(it::ControllerIterator, t::Int = 1)
    it.alreadyran[] && error("Cannot iterate on a ControllerIterator more than once")
    if t > it.T
        it.alreadyran[] = true
        return
    end

    rolloutstep!(it.controller, it.trajectory, it.env, t)
    if mod(t, it.plotiter) == 0
        rew = @views Tools.Line(it.trajectory.rewards[1:t], "Reward")
        eval = @views Tools.Line(it.trajectory.evaluations[1:t], "Eval")
        plt = Tools.termplot(
            rew,
            eval,
            title = "ControllerIterator Iteration=$t/$(it.T)",
            width = 40,
        )
        display(plt)
    end

    (t, it), t + 1
end

Base.length(it::ControllerIterator) = it.T

function rolloutstep!(controller, traj, env, t)
    st = view(traj.states, .., t)
    ot = view(traj.observations, .., t)
    at = view(traj.actions, .., t)

    getstate!(st, env)
    getobservation!(ot, env)
    controller(at, st, ot)
    setaction!(env, at)

    step!(env)
    r = getreward(st, at, ot, env)
    done = isdone(st, ot, env)

    traj.rewards[t] = r

    traj
end


module Isaac

include("testsets.jl")

# linsearch  method
include("linesearch.jl")

# specialised Krylov subspace methods using finite-differences
include("fdkrylov.jl")

# the Newton-Krylov solver based on C T Kelley's nsoli.m
include("nsoli.jl")

# # the main transformed Newton solver
# include("solvermain.jl")

# # interfaces to Optim.jl and NLsolve.jl
# include("interfaces.jl")

end


### A Pluto.jl notebook ###
# v0.17.3

using Markdown
using InteractiveUtils

# ╔═╡ 8ae2593a-cb48-44c9-b9dc-ab901c358a06
begin
	import Pkg
	Pkg.develop("PoreMatMod")
end

# ╔═╡ 16f0e183-f48d-4dcd-8751-d3c61c875e18
# load required packages (Pluto.jl will automatically install them)
using PoreMatMod, PlutoUI

# ╔═╡ 3cca5289-a829-4717-b796-0834229995d1
using PoreMatMod.ExampleHelpers

# ╔═╡ 8d523993-6e85-443a-9949-12030552b457
md"""
## example: generate a hypothetical MOF structure by decorating its linkers with functional groups
"""

# ╔═╡ 74b45651-21d5-4332-a4a5-866ea1bb02b8
input_file_message()

# ╔═╡ 3ff0cd63-8bdf-4ba5-92b6-e2a52f7573c2
xtal_folder_message()

# ╔═╡ ccc7fc86-5f48-4cc9-bd5c-349fc1d55b0c
rc[:paths][:crystals]

# ╔═╡ b66dd0d5-5bac-4b23-a33f-17305b0d4728
moiety_folder_message()

# ╔═╡ a76e79ee-c82c-4771-818d-5380a5fb4c18
rc[:paths][:moieties]

# ╔═╡ be86f07e-0669-46c2-8c79-80e65dfcc6f2
md"""
!!! example \"the task\"
	we have the IRMOF-1 crystal structure, and wish to append acetamido functional groups to six of its (randomly chosen) BDC (1,4-benzodicarboxylate) linkers to give a mixed-linker IRMOF-1 derivative.

**Parent crystal structure**: first, we read in the `.cif` file describing the parent IRMOF-1 crystal structure.
"""

# ╔═╡ b53e7c38-d8f5-4f28-a9dc-f7902a86fdb2
begin
	# read in the parent xtal
	parent = Crystal("IRMOF-1.cif")	# load .cif file
	infer_bonds!(parent, true)      # infer bonds
	view_structure(parent)          # view structure
end

# ╔═╡ b402e79a-784b-4d8b-82f1-df4fe1cedad1
fragment_construction_note()

# ╔═╡ 9b2d534a-4f78-4950-add1-9ba645669bb9
md"""
**Query fragment**: next, we construct a query fragment as a _p_-phenylene fragment to match that on the BCD linker of the IRMOF-1 parent structure. we mark one hydrogen atom on the query fragment as "masked" (shown in pink) by tagging its species label with `!` in the input. we need to mask this hydrogen atom because it will eventually be replaced by the acetamido functional group.
"""

# ╔═╡ 4be03110-61ab-4cd6-b3fe-7d51ac5ee771
query = moiety("2-!-p-phenylene.xyz");

# ╔═╡ e5eaaef4-13a0-48bd-9f2b-5040b2d10ac1
view_query_or_replacement("2-!-p-phenylene.xyz")

# ╔═╡ 559ef3c3-a176-4d65-8958-810c9b0b32c5
with_terminal() do
	display_query_or_replacement_file("2-!-p-phenylene.xyz")
end

# ╔═╡ d1aa8a19-3702-40f6-b73e-b9ebfc1a7a71
md"""
**Replacement fragment**: next, we construct a replacement fragment as a modified version of the query fragment (with acetamido group in place of one hydrogen atom).
"""

# ╔═╡ 44e7e665-ae68-4cd4-b45f-138a0fb8910e
replacement = moiety("2-acetylamido-p-phenylene.xyz");

# ╔═╡ af606f02-fbd5-4033-a5f3-56a8f740b1a1
view_query_or_replacement("2-acetylamido-p-phenylene.xyz")

# ╔═╡ c8f8dbd3-4191-460d-944f-f5e456ce8b83
with_terminal() do
	display_query_or_replacement_file("2-!-p-phenylene.xyz")
end

# ╔═╡ 127a2bef-3903-4801-bc75-00a6dde2bc6e
md"""
**Find and replace**: finally, we (i) search the parent MOF for the query fragment and (ii) effect the replacements. Ta-da; we have a hypothetical derivatized IRMOF-1 structure. 🎈

n.b.
* the `nb_loc=6` kwarg indicates that we wish to randomly select 6 matches on the IRMOF-1 parent structure to effect the replacement. the `loc` kwarg grants more control over which BDC linkers are functionalized.
* the acetamido functional groups are appended at random positions on each BDC ligand. the `ori` kwarg grants more control over which positions on each linker are functionalized.
* if we omit `nb_loc=6` as a kwarg, a functional group is appended on all BDC linkers of the parent.
"""

# ╔═╡ 74aa19d2-b1a4-4333-9ff9-e6ea74e7d989
begin
	child = replace(parent, query => replacement, nb_loc=6)
	view_structure(child)
end

# ╔═╡ 986ecdc4-455f-457e-a964-f00ddfeb53a2
write_cif_message()

# ╔═╡ 71209370-c445-4ff2-a873-b6e31c46419b
write_cif(child, "acetamido-functionalized_IRMOF-1.cif")

# ╔═╡ Cell order:
# ╟─8ae2593a-cb48-44c9-b9dc-ab901c358a06
# ╟─8d523993-6e85-443a-9949-12030552b457
# ╠═16f0e183-f48d-4dcd-8751-d3c61c875e18
# ╠═3cca5289-a829-4717-b796-0834229995d1
# ╠═74b45651-21d5-4332-a4a5-866ea1bb02b8
# ╟─3ff0cd63-8bdf-4ba5-92b6-e2a52f7573c2
# ╠═ccc7fc86-5f48-4cc9-bd5c-349fc1d55b0c
# ╟─b66dd0d5-5bac-4b23-a33f-17305b0d4728
# ╠═a76e79ee-c82c-4771-818d-5380a5fb4c18
# ╟─be86f07e-0669-46c2-8c79-80e65dfcc6f2
# ╠═b53e7c38-d8f5-4f28-a9dc-f7902a86fdb2
# ╟─b402e79a-784b-4d8b-82f1-df4fe1cedad1
# ╟─9b2d534a-4f78-4950-add1-9ba645669bb9
# ╠═4be03110-61ab-4cd6-b3fe-7d51ac5ee771
# ╟─e5eaaef4-13a0-48bd-9f2b-5040b2d10ac1
# ╟─559ef3c3-a176-4d65-8958-810c9b0b32c5
# ╟─d1aa8a19-3702-40f6-b73e-b9ebfc1a7a71
# ╠═44e7e665-ae68-4cd4-b45f-138a0fb8910e
# ╟─af606f02-fbd5-4033-a5f3-56a8f740b1a1
# ╟─c8f8dbd3-4191-460d-944f-f5e456ce8b83
# ╟─127a2bef-3903-4801-bc75-00a6dde2bc6e
# ╠═74aa19d2-b1a4-4333-9ff9-e6ea74e7d989
# ╟─986ecdc4-455f-457e-a964-f00ddfeb53a2
# ╠═71209370-c445-4ff2-a873-b6e31c46419b


import SymmetricDominance
using Base.Test

const sd = SymmetricDominance

const gdb = sd.GeneDB()

for _ = 1:2
    sd.insert!(sd.WithNewAllele, gdb, 1, 0)
end
for i = 1:2, _ = 1:2
    sd.insert!(sd.WithNewAllele, gdb, 2, i)
end
for i = 3:6, _ = 1:2
    sd.insert!(sd.WithNewAllele, gdb, 3, i)
end
@test gdb.currentid == 14
@test gdb.currentstate == 14
sd.insert!(sd.WithoutNewAllele, gdb, 4, 1, sd.select(gdb, 1, :state)[1])
@test gdb.currentid == 15
@test gdb.currentstate == 14

@test sd.isidbystate(gdb, 1, 1) == true
@test sd.isidbystate(gdb, 1, 15) == true
@test sd.isidbystate(gdb, 15, 1) == true
@test sd.isidbystate(gdb, 1, 2) != true
@test sd.isidbystate(gdb, 2, 1) != true
@test sd.isidbystate(gdb, 2, 6) != true
@test sd.isidbystate(gdb, 6, 2) != true

@test sd.getancestor(gdb, 1, 1) == 1
@test sd.getancestor(gdb, 2, 1) == 2
@test sd.getancestor(gdb, 3, 1) == 1
@test sd.getancestor(gdb, 4, 1) == 1
@test sd.getancestor(gdb, 5, 1) == 2
@test sd.getancestor(gdb, 6, 1) == 2
@test sd.getancestor(gdb, 7, 1) == 1
@test sd.getancestor(gdb, 8, 1) == 1
@test sd.getancestor(gdb, 9, 1) == 1
@test sd.getancestor(gdb, 10, 1) == 1
@test sd.getancestor(gdb, 11, 1) == 2
@test sd.getancestor(gdb, 12, 1) == 2
@test sd.getancestor(gdb, 13, 1) == 2
@test sd.getancestor(gdb, 14, 1) == 2
@test sd.getancestor(gdb, 15, 1) == 1
@test sd.getancestor(gdb, 3, 2) == 3
@test sd.getancestor(gdb, 4, 2) == 4
@test sd.getancestor(gdb, 5, 2) == 5
@test sd.getancestor(gdb, 6, 2) == 6
@test sd.getancestor(gdb, 7, 2) == 3
@test sd.getancestor(gdb, 8, 2) == 3
@test sd.getancestor(gdb, 9, 2) == 4
@test sd.getancestor(gdb, 10, 2) == 4
@test sd.getancestor(gdb, 11, 2) == 5
@test sd.getancestor(gdb, 12, 2) == 5
@test sd.getancestor(gdb, 13, 2) == 6
@test sd.getancestor(gdb, 14, 2) == 6
@test sd.getancestor(gdb, 15, 2) == 1
@test sd.getancestor(gdb, 7, 3) == 7
@test sd.getancestor(gdb, 8, 3) == 8
@test sd.getancestor(gdb, 9, 3) == 9
@test sd.getancestor(gdb, 10, 3) == 10
@test sd.getancestor(gdb, 11, 3) == 11
@test sd.getancestor(gdb, 12, 3) == 12
@test sd.getancestor(gdb, 13, 3) == 13
@test sd.getancestor(gdb, 14, 3) == 14
@test sd.getancestor(gdb, 15, 3) == 1

const o = sd.Organism(3)
@test sd.nloci(o) == 3

const pop = sd.Population(2, 1)
pop[1, 1, 1] = 7
pop[2, 1, 1] = 8
pop[1, 1, 2] = 9
pop[2, 1, 2] = 10
@test sd.hascoalesced(gdb, pop, 1, 1)
@test !sd.hascoalesced(gdb, pop, 1, 2)

for i = 1:15
    gdb.state[i] += 10
end

sdata3 = Array(Int, 2, 1, 2)
sdata3[1, 1, 1] = 7
sdata3[2, 1, 1] = 8
sdata3[1, 1, 2] = 9
sdata3[2, 1, 2] = 10
@test sd.toarray(gdb, pop, :id) == sdata3
sdata3[1, 1, 1] = 17
sdata3[2, 1, 1] = 18
sdata3[1, 1, 2] = 19
sdata3[2, 1, 2] = 20
@test sd.toarray(gdb, pop, :state) == sdata3
sdata3[1, 1, 1] = 3
sdata3[2, 1, 1] = 3
sdata3[1, 1, 2] = 4
sdata3[2, 1, 2] = 4
@test sd.toarray(gdb, pop, :parent) == sdata3
sdata3[1, 1, 1] = 3
sdata3[2, 1, 1] = 3
sdata3[1, 1, 2] = 3
sdata3[2, 1, 2] = 3
@test sd.toarray(gdb, pop, :epoch) == sdata3

const acs, gcs, hcs = sd.counts(gdb, pop)
@test acs == [Dict{Int,Int}(17=>1, 18=>1, 19=>1, 20=>1)]
@test gcs == [Dict{Tuple{Int,Int}, Int}((17, 19)=>1, (18, 20)=>1)]
@test hcs == Dict{Tuple{Int}, Int}((17,)=>1, (18,)=>1, (19,)=>1, (20,)=>1)

const afs, gfs, hfs = sd.spectra(gdb, pop)
@test afs == [Dict{Int,Int}(1=>4)]
@test gfs == [Dict{Int,Int}(1=>2)]
@test hfs == Dict{Int,Int}(1=>4)

@test sd.history(gdb, 1, 1) == [1]
@test sd.history(gdb, 2, 1) == [2]
@test sd.history(gdb, 3, 1) == [1, 3]
@test sd.history(gdb, 4, 1) == [1, 4]
@test sd.history(gdb, 5, 1) == [2, 5]
@test sd.history(gdb, 6, 1) == [2, 6]
@test sd.history(gdb, 7, 1) == [1, 3, 7]
@test sd.history(gdb, 8, 1) == [1, 3, 8]
@test sd.history(gdb, 9, 1) == [1, 4 ,9]
@test sd.history(gdb, 10, 1) == [1, 4, 10]
@test sd.history(gdb, 11, 1) == [2, 5, 11]
@test sd.history(gdb, 12, 1) == [2, 5, 12]
@test sd.history(gdb, 13, 1) == [2, 6, 13]
@test sd.history(gdb, 14, 1) == [2, 6, 14]
@test sd.history(gdb, 15, 1) == [1, 15]
@test sd.history(gdb, 3, 2) == [3]
@test sd.history(gdb, 4, 2) == [4]
@test sd.history(gdb, 5, 2) == [5]
@test sd.history(gdb, 6, 2) == [6]
@test sd.history(gdb, 7, 2) == [3, 7]
@test sd.history(gdb, 8, 2) == [3, 8]
@test sd.history(gdb, 9, 2) == [4 ,9]
@test sd.history(gdb, 10, 2) == [4, 10]
@test sd.history(gdb, 11, 2) == [5, 11]
@test sd.history(gdb, 12, 2) == [5, 12]
@test sd.history(gdb, 13, 2) == [6, 13]
@test sd.history(gdb, 14, 2) == [6, 14]
@test sd.history(gdb, 15, 2) == [1, 15]
@test sd.history(gdb, 7, 3) == [7]
@test sd.history(gdb, 8, 3) == [8]
@test sd.history(gdb, 9, 3) == [9]
@test sd.history(gdb, 10, 3) == [10]
@test sd.history(gdb, 11, 3) == [11]
@test sd.history(gdb, 12, 3) == [12]
@test sd.history(gdb, 13, 3) == [13]
@test sd.history(gdb, 14, 3) == [14]
@test sd.history(gdb, 15, 3) == [1, 15]

ds = sd.distances(gdb, pop, 1)
@test ds == [1 17 18 2;
             1 17 19 4;
             1 17 20 4;
             1 18 19 4;
             1 18 20 4;
             1 19 20 2]
ds = sd.distances(gdb, pop, 2)
@test ds == [1 17 18 2;
             1 17 19 -1;
             1 17 20 -1;
             1 18 19 -1;
             1 18 20 -1;
             1 19 20 2]
ds = sd.distances(gdb, pop, 3)
@test ds == [1 17 18 -1;
             1 17 19 -1;
             1 17 20 -1;
             1 18 19 -1;
             1 18 20 -1;
             1 19 20 -1]

@test sd.nsegsites(gdb, pop, 1) == [6]
@test sd.nsegsites(gdb, pop, 2) == [-1]
@test sd.nsegsites(gdb, pop, 3) == [-1]

@test sd.tmrca(gdb, pop) == [1]


# Clause set operation

struct RESDB
  cdb
  ldb
  pgr
end

#==
 CDB(clauses) -> LDB(Literals), PGR(Graph)
==#

include("common.jl")
include("reso.jl")

#==
 primitives for CDB
==#

function getvars(cdb, cid)
 cdb[cid][1]
end

function getvars(db::RESDB, cid)
 db.cdb[cid][1]
end

function getcls(cdb, cid)
 if cid == 0; return [[],EmptyClause] end
 cdb[cid]
end

function getcls(rdb::RESDB, cid)
 if cid == 0; return [[],EmptyClause] end
 rdb.cdb[cid]
end

function getlit(cdb, clp)
 cid,lid=clp
 return cdb[cid][2][lid].args[1], cdb[cid][2][lid].args[2]
end

function getlit(db::RESDB, clp)
 cid,lid=clp
 return db.cdb[cid][2][lid].args[1], db.cdb[cid][2][lid].args[2]
end

function litcounts(cdb)
 map(x->length(x[2]),cdb)
end

function litcounts(db::RESDB)
 map(x->length(x[2]),db.cdb)
end

#==
 maker for CDB,LDB,PGR
==#
makecent(cls) = (cls.args[1].args, cls.args[2].args[1].args)

function makecdb(cdb,cls)
  vars, cl = makecent(cls)
  push!(cdb, (vars, cl))
  return cdb
end

function addcdb(cdb, clss)
 for cls in clss
  makecdb(cdb,cls)
 end
 cdb
end

function makeldb(ldb, cid, cls)
@show cls
 for lid in 1:length(cls[2])
  lit=cls[2][lid]
@show lid
@show lit
  ldb[(cid,lid)]=lit
 end
 return ldb
end

function makepg(ldb)
 pgr=Dict()

 # ADHOC
 psyms = map(x->x.args[2].args[1],values(ldb))
 psyms = union(psyms,psyms)
 map(p->pgr[p]=[[],[]],psyms)

 for lid in keys(ldb)
  sign = ldb[lid].args[1]
  psym = ldb[lid].args[2].args[1]

  if sign == :+
    push!(pgr[psym][1],lid)
  else # sign == :-
    push!(pgr[psym][2],lid)
  end

 end
 return pgr
end

function makedb(clauses)
 cdb = []
 ldb = Dict()
 for cid in 1:length(clauses)
  cls = clauses[cid]
  cdb = makecdb(cdb, cls)
  ldb = makeldb(ldb, cid, cls)
 end
 pgr = makepg(ldb)
 return RESDB(cdb, ldb, pgr)
end

##
newcd(cdb) = length(cdb) + 1

function putcls(cls,cdb)::Int
 cid = 1
 for ci in cdb
  if equalclause(ci, cls)
   return cid
  end
  cid += 1
 end
 push!(cdb, cls)
 return cid +1
end

function putcdb(cdb,cls)
 push!(cdb, cls)
end

function putcdb(cls, cdb,ldb,pgr)
 cid = newcid(cdb)
 cdb = putcdb(cdb,cls)
 ldb = putldb(ldb,cid, cls)
 pgr = putpgr(pgr,cls)

 return cdb, ldb, pgr
end

function replacecdb(clsb, clsa, db)
@nyi :replacecdb
end

findunit(cdb) = find(x->x==1,litcounts(cdb))

#==
printing
==#

function printvars(vars)
 print("[")
 for i in 1:length(vars)
  if i < length(vars); print("$(vars[i]),") end
  if i == length(vars); print("$(vars[i])") end
 end
 print("]")
end

function printcdb(cdb)
 for i in 1:length(cdb)
  print("$i ")
  printvars(cdb[i][1])
  println(cdb[i][2])
 end
end

function printplist(plist)
 for i in 1:length(plist)
  if i < length(plist); print("$(plist[i]),") end
  if i == length(plist); print("$(plist[i])") end
 end
end

function printldb(ldb)
 for pr in keys(ldb)
  println("$pr => $(ldb[pr])")
 end
end

function printpgr(pgr)
 for p in keys(pgr)
  print("$p: ")
  print("\n +")
  printplist(pgr[p][1])
  print("\n -")
  printplist(pgr[p][2])
  println()
 end
end

## introduce a type for DB(cdb,ldb,pgr)

function printdb(resdb)
println("Clauses")
  printcdb(resdb.cdb)
println("\nLiterals")
  printldb(resdb.ldb)
println("\nGraph")
  printpgr(resdb.pgr)
end


"""
    testapply(tfm, item)
    testapply(tfm, I)

Test `apply` invariants of `tfm` on `item` or item type `I`.

1. With a constant `randstate` parameter, `apply` should always return the
    same result.
"""
function testapply(tfm::Transform, item::AbstractItem)
    # Invariant 1
    r = getrandstate(tfm)
    titems = [apply(tfm, item; randstate = r) for i = 1:8]
    @test all(map(titem -> itemdata(titem) == itemdata(titems[1]), titems))

    #
    @test_nowarn apply(tfm, (item, item))
end

testapply(tfm::Transform, I::Type{<:AbstractItem}) = testapply(tfm, testitem(I))
testapply(tfm, items::Tuple) = foreach(i -> testapply(tfm, i), items)

"""
    testapply!(tfm, Items)
    testapply!(tfm, Item)
    testapply!(tfm, item1, item2)

Test `apply!` invariants.

1. With a constant `randstate` parameter, `apply!` should always return the
    same result.
2. Given a different item than was used to create the buffer, the buffer's data
    should be modified.
"""
function testapply!(tfm::Transform, item1::I, item2::I) where I<:AbstractItem
    # Invariant 1
    r = getrandstate(tfm)
    titems = [apply(tfm, deepcopy(item1); randstate = r) for i = 1:8]
    @test allequal(itemdata.(titems))

    # Invariant 2
    buf = makebuffer(tfm, item1)
    cbuf = deepcopy(buf)
    apply!(buf, tfm, item2)
    if buf isa AbstractItem
        @test itemdata(buf) != itemdata(cbuf)
    end
end

testapply!(tfm::Transform, I::Type{<:AbstractItem}) = testapply!(tfm, testitem(I), testitem(I))
testapply!(tfm, items::Tuple) = foreach(i -> testapply!(tfm, i), items)

"""
    testprojective(tfm)

Test invariants of a `ProjectiveTransform`.

1. `getprojection` is defined, and, given a constant `randstate` parameter,
    always returns the same result.
2. It preserves the item type, i.e. `apply(tfm, ::I) -> I`.
3. Applying it to multiple items with the same bounds results in the same bounds
    for all items.
"""
function testprojective(tfm::ProjectiveTransform, items::Tuple)
    # All bounds must be equal for the test to work
    @assert allequal(getbounds.(items))

    # Invariant 1
    r = getrandstate(tfm)
    bs = getbounds(items[1])
    Ps = [getprojection(tfm, bs; randstate = r) for i = 1:10]
    @test allequal(Ps)

    # Invariant 2
    titems = apply(tfm, items)
    @test all(typeof.(items) .== typeof.(titems))

    # Invariant 3
    @test allequal(getbounds.(titems))
end

function testprojective(tfm::ProjectiveTransform, Is::NTuple{N, <:Type}) where {N}
    testprojective(tfm, testitem.(Is))
end

testprojective(tfm::ProjectiveTransform) =
    testprojective(tfm, (Image, MaskBinary, MaskMulti, Keypoints))

"""
    testitem(TItem)

Create an instance of an item with type `TItem`. If it has spatial bounds,
should return an instance with bounds with ranges (1:16, 1:16).
"""
function testitem end

testitem(::Type{ArrayItem}) = testitem(ArrayItem{2, Float32})
testitem(::Type{ArrayItem{N, T}}) where {N, T} = ArrayItem(rand(T, ntuple(i -> 16, N)))

testitem(::Type{Image}) = testitem(Image{2, RGB{N0f8}})
testitem(::Type{Image{N, T}}) where {N, T} = Image(rand(T, ntuple(i -> 16, N)))

testitem(::Type{MaskBinary}) = testitem(MaskBinary{2})
testitem(::Type{MaskBinary{N}}) where {N} = MaskBinary(rand(Bool, ntuple(i -> 16, N)))

testitem(::Type{MaskMulti}) = testitem(MaskMulti{2, UInt8})
function testitem(::Type{MaskMulti{N, T}}) where {N, T}
    n = rand(2:10)
    data = T.(rand(1:n, ntuple(i -> 16, N)))
    MaskMulti(data, 1:n)
end

testitem(::Type{Keypoints}) = testitem(Keypoints{2, Float32})
function testitem(::Type{Keypoints{N, T}}) where {N, T}
    n = rand(8:16)
    data = map(v -> v .* 15, rand(SVector{N, T}, n))
    return Keypoints(data, (16, 16))
end

testitem(::Type{Polygon}) = testitem(Polygon{2, Float32})
function testitem(::Type{Polygon{N, T}}) where {N, T}
    n = rand(8:16)
    data = map(v -> v .* 15, rand(SVector{N, T}, n))
    return Polygon(Keypoints(data, (16, 16)))
end

testitem(::Type{BoundingBox}) = testitem(BoundingBox{2, Float32})
function testitem(::Type{BoundingBox{N, T}}) where {N, T}
    n = 2
    data = map(v -> v .* 15, rand(SVector{N, T}, n))
    return BoundingBox(Keypoints(data, (16, 16)))
end

allequal(xs) = all(x == xs[1] for x in xs)


using Foundation
using Test

@test (+).name == :+


# websafe colors
# using Colors, ColorSchemes
# cs = ColorScheme([parse(RGB{Float64}, "#$(string.([r,g,b], base=16)...)") for r in 0x0:3:0xf for g in 0x0:3:0xf for b in 0x0:3:0xf])

loadcolorscheme(:websafe, [
        RGB{Float64}(0.0,0.0,0.0),
        RGB{Float64}(0.0,0.0,0.2),
        RGB{Float64}(0.0,0.0,0.4),
        RGB{Float64}(0.0,0.0,0.6),
        RGB{Float64}(0.0,0.0,0.8),
        RGB{Float64}(0.0,0.0,1.0),
        RGB{Float64}(0.0,0.2,0.0),
        RGB{Float64}(0.0,0.2,0.2),
        RGB{Float64}(0.0,0.2,0.4),
        RGB{Float64}(0.0,0.2,0.6),
        RGB{Float64}(0.0,0.2,0.8),
        RGB{Float64}(0.0,0.2,1.0),
        RGB{Float64}(0.0,0.4,0.0),
        RGB{Float64}(0.0,0.4,0.2),
        RGB{Float64}(0.0,0.4,0.4),
        RGB{Float64}(0.0,0.4,0.6),
        RGB{Float64}(0.0,0.4,0.8),
        RGB{Float64}(0.0,0.4,1.0),
        RGB{Float64}(0.0,0.6,0.0),
        RGB{Float64}(0.0,0.6,0.2),
        RGB{Float64}(0.0,0.6,0.4),
        RGB{Float64}(0.0,0.6,0.6),
        RGB{Float64}(0.0,0.6,0.8),
        RGB{Float64}(0.0,0.6,1.0),
        RGB{Float64}(0.0,0.8,0.0),
        RGB{Float64}(0.0,0.8,0.2),
        RGB{Float64}(0.0,0.8,0.4),
        RGB{Float64}(0.0,0.8,0.6),
        RGB{Float64}(0.0,0.8,0.8),
        RGB{Float64}(0.0,0.8,1.0),
        RGB{Float64}(0.0,1.0,0.0),
        RGB{Float64}(0.0,1.0,0.2),
        RGB{Float64}(0.0,1.0,0.4),
        RGB{Float64}(0.0,1.0,0.6),
        RGB{Float64}(0.0,1.0,0.8),
        RGB{Float64}(0.0,1.0,1.0),
        RGB{Float64}(0.2,0.0,0.0),
        RGB{Float64}(0.2,0.0,0.2),
        RGB{Float64}(0.2,0.0,0.4),
        RGB{Float64}(0.2,0.0,0.6),
        RGB{Float64}(0.2,0.0,0.8),
        RGB{Float64}(0.2,0.0,1.0),
        RGB{Float64}(0.2,0.2,0.0),
        RGB{Float64}(0.2,0.2,0.2),
        RGB{Float64}(0.2,0.2,0.4),
        RGB{Float64}(0.2,0.2,0.6),
        RGB{Float64}(0.2,0.2,0.8),
        RGB{Float64}(0.2,0.2,1.0),
        RGB{Float64}(0.2,0.4,0.0),
        RGB{Float64}(0.2,0.4,0.2),
        RGB{Float64}(0.2,0.4,0.4),
        RGB{Float64}(0.2,0.4,0.6),
        RGB{Float64}(0.2,0.4,0.8),
        RGB{Float64}(0.2,0.4,1.0),
        RGB{Float64}(0.2,0.6,0.0),
        RGB{Float64}(0.2,0.6,0.2),
        RGB{Float64}(0.2,0.6,0.4),
        RGB{Float64}(0.2,0.6,0.6),
        RGB{Float64}(0.2,0.6,0.8),
        RGB{Float64}(0.2,0.6,1.0),
        RGB{Float64}(0.2,0.8,0.0),
        RGB{Float64}(0.2,0.8,0.2),
        RGB{Float64}(0.2,0.8,0.4),
        RGB{Float64}(0.2,0.8,0.6),
        RGB{Float64}(0.2,0.8,0.8),
        RGB{Float64}(0.2,0.8,1.0),
        RGB{Float64}(0.2,1.0,0.0),
        RGB{Float64}(0.2,1.0,0.2),
        RGB{Float64}(0.2,1.0,0.4),
        RGB{Float64}(0.2,1.0,0.6),
        RGB{Float64}(0.2,1.0,0.8),
        RGB{Float64}(0.2,1.0,1.0),
        RGB{Float64}(0.4,0.0,0.0),
        RGB{Float64}(0.4,0.0,0.2),
        RGB{Float64}(0.4,0.0,0.4),
        RGB{Float64}(0.4,0.0,0.6),
        RGB{Float64}(0.4,0.0,0.8),
        RGB{Float64}(0.4,0.0,1.0),
        RGB{Float64}(0.4,0.2,0.0),
        RGB{Float64}(0.4,0.2,0.2),
        RGB{Float64}(0.4,0.2,0.4),
        RGB{Float64}(0.4,0.2,0.6),
        RGB{Float64}(0.4,0.2,0.8),
        RGB{Float64}(0.4,0.2,1.0),
        RGB{Float64}(0.4,0.4,0.0),
        RGB{Float64}(0.4,0.4,0.2),
        RGB{Float64}(0.4,0.4,0.4),
        RGB{Float64}(0.4,0.4,0.6),
        RGB{Float64}(0.4,0.4,0.8),
        RGB{Float64}(0.4,0.4,1.0),
        RGB{Float64}(0.4,0.6,0.0),
        RGB{Float64}(0.4,0.6,0.2),
        RGB{Float64}(0.4,0.6,0.4),
        RGB{Float64}(0.4,0.6,0.6),
        RGB{Float64}(0.4,0.6,0.8),
        RGB{Float64}(0.4,0.6,1.0),
        RGB{Float64}(0.4,0.8,0.0),
        RGB{Float64}(0.4,0.8,0.2),
        RGB{Float64}(0.4,0.8,0.4),
        RGB{Float64}(0.4,0.8,0.6),
        RGB{Float64}(0.4,0.8,0.8),
        RGB{Float64}(0.4,0.8,1.0),
        RGB{Float64}(0.4,1.0,0.0),
        RGB{Float64}(0.4,1.0,0.2),
        RGB{Float64}(0.4,1.0,0.4),
        RGB{Float64}(0.4,1.0,0.6),
        RGB{Float64}(0.4,1.0,0.8),
        RGB{Float64}(0.4,1.0,1.0),
        RGB{Float64}(0.6,0.0,0.0),
        RGB{Float64}(0.6,0.0,0.2),
        RGB{Float64}(0.6,0.0,0.4),
        RGB{Float64}(0.6,0.0,0.6),
        RGB{Float64}(0.6,0.0,0.8),
        RGB{Float64}(0.6,0.0,1.0),
        RGB{Float64}(0.6,0.2,0.0),
        RGB{Float64}(0.6,0.2,0.2),
        RGB{Float64}(0.6,0.2,0.4),
        RGB{Float64}(0.6,0.2,0.6),
        RGB{Float64}(0.6,0.2,0.8),
        RGB{Float64}(0.6,0.2,1.0),
        RGB{Float64}(0.6,0.4,0.0),
        RGB{Float64}(0.6,0.4,0.2),
        RGB{Float64}(0.6,0.4,0.4),
        RGB{Float64}(0.6,0.4,0.6),
        RGB{Float64}(0.6,0.4,0.8),
        RGB{Float64}(0.6,0.4,1.0),
        RGB{Float64}(0.6,0.6,0.0),
        RGB{Float64}(0.6,0.6,0.2),
        RGB{Float64}(0.6,0.6,0.4),
        RGB{Float64}(0.6,0.6,0.6),
        RGB{Float64}(0.6,0.6,0.8),
        RGB{Float64}(0.6,0.6,1.0),
        RGB{Float64}(0.6,0.8,0.0),
        RGB{Float64}(0.6,0.8,0.2),
        RGB{Float64}(0.6,0.8,0.4),
        RGB{Float64}(0.6,0.8,0.6),
        RGB{Float64}(0.6,0.8,0.8),
        RGB{Float64}(0.6,0.8,1.0),
        RGB{Float64}(0.6,1.0,0.0),
        RGB{Float64}(0.6,1.0,0.2),
        RGB{Float64}(0.6,1.0,0.4),
        RGB{Float64}(0.6,1.0,0.6),
        RGB{Float64}(0.6,1.0,0.8),
        RGB{Float64}(0.6,1.0,1.0),
        RGB{Float64}(0.8,0.0,0.0),
        RGB{Float64}(0.8,0.0,0.2),
        RGB{Float64}(0.8,0.0,0.4),
        RGB{Float64}(0.8,0.0,0.6),
        RGB{Float64}(0.8,0.0,0.8),
        RGB{Float64}(0.8,0.0,1.0),
        RGB{Float64}(0.8,0.2,0.0),
        RGB{Float64}(0.8,0.2,0.2),
        RGB{Float64}(0.8,0.2,0.4),
        RGB{Float64}(0.8,0.2,0.6),
        RGB{Float64}(0.8,0.2,0.8),
        RGB{Float64}(0.8,0.2,1.0),
        RGB{Float64}(0.8,0.4,0.0),
        RGB{Float64}(0.8,0.4,0.2),
        RGB{Float64}(0.8,0.4,0.4),
        RGB{Float64}(0.8,0.4,0.6),
        RGB{Float64}(0.8,0.4,0.8),
        RGB{Float64}(0.8,0.4,1.0),
        RGB{Float64}(0.8,0.6,0.0),
        RGB{Float64}(0.8,0.6,0.2),
        RGB{Float64}(0.8,0.6,0.4),
        RGB{Float64}(0.8,0.6,0.6),
        RGB{Float64}(0.8,0.6,0.8),
        RGB{Float64}(0.8,0.6,1.0),
        RGB{Float64}(0.8,0.8,0.0),
        RGB{Float64}(0.8,0.8,0.2),
        RGB{Float64}(0.8,0.8,0.4),
        RGB{Float64}(0.8,0.8,0.6),
        RGB{Float64}(0.8,0.8,0.8),
        RGB{Float64}(0.8,0.8,1.0),
        RGB{Float64}(0.8,1.0,0.0),
        RGB{Float64}(0.8,1.0,0.2),
        RGB{Float64}(0.8,1.0,0.4),
        RGB{Float64}(0.8,1.0,0.6),
        RGB{Float64}(0.8,1.0,0.8),
        RGB{Float64}(0.8,1.0,1.0),
        RGB{Float64}(1.0,0.0,0.0),
        RGB{Float64}(1.0,0.0,0.2),
        RGB{Float64}(1.0,0.0,0.4),
        RGB{Float64}(1.0,0.0,0.6),
        RGB{Float64}(1.0,0.0,0.8),
        RGB{Float64}(1.0,0.0,1.0),
        RGB{Float64}(1.0,0.2,0.0),
        RGB{Float64}(1.0,0.2,0.2),
        RGB{Float64}(1.0,0.2,0.4),
        RGB{Float64}(1.0,0.2,0.6),
        RGB{Float64}(1.0,0.2,0.8),
        RGB{Float64}(1.0,0.2,1.0),
        RGB{Float64}(1.0,0.4,0.0),
        RGB{Float64}(1.0,0.4,0.2),
        RGB{Float64}(1.0,0.4,0.4),
        RGB{Float64}(1.0,0.4,0.6),
        RGB{Float64}(1.0,0.4,0.8),
        RGB{Float64}(1.0,0.4,1.0),
        RGB{Float64}(1.0,0.6,0.0),
        RGB{Float64}(1.0,0.6,0.2),
        RGB{Float64}(1.0,0.6,0.4),
        RGB{Float64}(1.0,0.6,0.6),
        RGB{Float64}(1.0,0.6,0.8),
        RGB{Float64}(1.0,0.6,1.0),
        RGB{Float64}(1.0,0.8,0.0),
        RGB{Float64}(1.0,0.8,0.2),
        RGB{Float64}(1.0,0.8,0.4),
        RGB{Float64}(1.0,0.8,0.6),
        RGB{Float64}(1.0,0.8,0.8),
        RGB{Float64}(1.0,0.8,1.0),
        RGB{Float64}(1.0,1.0,0.0),
        RGB{Float64}(1.0,1.0,0.2),
        RGB{Float64}(1.0,1.0,0.4),
        RGB{Float64}(1.0,1.0,0.6),
        RGB{Float64}(1.0,1.0,0.8),
        RGB{Float64}(1.0,1.0,1.0),
    ], "general", "216 websafe colors")


export CoreTracker, CoreTrackerResult, CoreTrackerStatus, CoreTrackerOptions,
        coretracker, coretracker_startsolutions, affine_tracking,
        track, track!, setup!, iterator,
        current_x, current_t, current_Δt, iters, status, accuracy, max_corrector_iters,
        max_step_size , refinement_accuracy, max_refinement_iters,
        set_accuracy!, set_max_corrector_iters!, set_refinement_accuracy!,
        set_max_refinement_iters!, set_max_step_size!, digits_lost, options

const coretracker_supported_keywords = [:corrector, :predictor, :patch,
    :initial_step_size, :min_step_size , :max_step_size,
    :accuracy, :refinement_accuracy, :max_corrector_iters, :max_refinement_iters,
    :max_steps, :simple_step_size_alg, :auto_scaling, :terminate_ill_conditioned,
    :log_transform, :precision, :steps_jacobian_info_update,
    :max_lost_digits]

####################
# CoreTrackerState #
####################

module CoreTrackerStatus

    @doc """
        CoreTrackerStatus.states

    The possible states the coretracker can achieve are

    * `CoreTrackerStatus.success`
    * `CoreTrackerStatus.tracking`
    * `CoreTrackerStatus.terminated_maximal_iterations`
    * `CoreTrackerStatus.terminated_invalid_startvalue`
    * `CoreTrackerStatus.terminated_step_size_too_small`
    * `CoreTrackerStatus.terminated_singularity`
    * `CoreTrackerStatus.terminated_ill_conditioned`
    """
    @enum states begin
        success
        tracking
        terminated_maximal_iterations
        terminated_invalid_startvalue
        terminated_step_size_too_small
        terminated_singularity
        terminated_ill_conditioned
    end
end

####################
# CoreTrackerResult #
####################

"""
     CoreTrackerResult{V<:AbstractVector}

Containing the result of a tracked path. The fields are
* `returncode::CoreTrackerStatus.states` If the tracking was successfull then it is `CoreTrackerStatus.success`.
* `x::V` The result.
* `t::ComplexF64` The `t` when the path tracker stopped.
* `accuracy::Float64`: The estimated accuracy of `x`.
"""
struct CoreTrackerResult{V <: AbstractVector}
     returncode::CoreTrackerStatus.states
     x::V
     t::ComplexF64
     accuracy::Float64
     accepted_steps::Int
     rejected_steps::Int
end

function CoreTrackerResult(tracker)
    state = tracker.state
     CoreTrackerResult(state.status,
          copy(state.x), state.segment[state.s],
          state.accuracy,
          state.accepted_steps,
          state.rejected_steps)
end

Base.show(io::IO, result::CoreTrackerResult) = print_fieldnames(io, result)
Base.show(io::IO, ::MIME"application/prs.juno.inline", result::CoreTrackerResult) = result

##################
## PATH TRACKER ##
##################

"""
    AutoScalingOptions(;scale_min=0.01,
                     scale_abs_min=min(scale_min^2, 200 * sqrt(eps()),
                     scale_max=1.0 / eps() / sqrt(2)

Parameters for the auto scaling of the variables.
"""
struct AutoScalingOptions
    scale_min::Float64
    scale_abs_min::Float64
    scale_max::Float64
end
function AutoScalingOptions(;scale_min=0.01,
                             scale_abs_min=min(scale_min^2, 200 * sqrt(eps())),
                             scale_max=1.0 / eps() / sqrt(2))
    AutoScalingOptions(scale_min, scale_abs_min, scale_max)
end
Base.show(io::IO, opts::AutoScalingOptions) = print_fieldnames(io, opts)
Base.show(io::IO, ::MIME"application/prs.juno.inline", opts::AutoScalingOptions) = opts

mutable struct StepSizeModel
    ω::Float64
    expected_Δx₀::Float64
end
StepSizeModel() = StepSizeModel(NaN, NaN)

function reset!(model::StepSizeModel)
    model.ω = model.expected_Δx₀ = NaN
    model
end
Base.show(io::IO, opts::StepSizeModel) = print_fieldnames(io, opts)
Base.show(io::IO, ::MIME"application/prs.juno.inline", opts::StepSizeModel) = opts

######################
# CoreTrackerOptions #
######################

"""
    CoreTrackerOptions

The set of options set for a [`CoreTracker`](@ref). See the description of [`CoreTracker`](@ref)
for all possible options.
"""
mutable struct CoreTrackerOptions
    accuracy::Float64
    max_corrector_iters::Int
    refinement_accuracy::Float64
    max_refinement_iters::Int
    max_steps::Int
    initial_step_size::Float64
    min_step_size ::Float64
    max_step_size ::Float64
    simple_step_size_alg::Bool
    update_patch::Bool
    max_lost_digits::Float64
    auto_scaling::Bool
    auto_scaling_options::AutoScalingOptions
    terminate_ill_conditioned::Bool
    precision::PrecisionOption
    steps_jacobian_info_update::Int
end

function CoreTrackerOptions(;
    accuracy=1e-7,
    auto_scaling=true,
    auto_scaling_options=AutoScalingOptions(),
    initial_step_size=0.1,
    max_corrector_iters::Int=2,
    max_refinement_iters=5,
    max_step_size=Inf,
    min_step_size=1e-14,
    parameter_homotopy=false,
    max_steps=parameter_homotopy ? 10_000 : 1_000,
    precision::PrecisionOption=PRECISION_FIXED_64,
    max_lost_digits=default_max_lost_digits(precision, accuracy),
    refinement_accuracy=1e-8,
    simple_step_size_alg=false,
    steps_jacobian_info_update::Int=3,
    terminate_ill_conditioned::Bool=true,
    update_patch=true)

    CoreTrackerOptions(accuracy, max_corrector_iters, refinement_accuracy,
            max_refinement_iters, max_steps, initial_step_size, min_step_size,
            max_step_size, simple_step_size_alg, update_patch, max_lost_digits,
            auto_scaling, auto_scaling_options, terminate_ill_conditioned,
            precision, steps_jacobian_info_update)
end

Base.show(io::IO, opts::CoreTrackerOptions) = print_fieldnames(io, opts)
Base.show(io::IO, ::MIME"application/prs.juno.inline", opts::CoreTrackerOptions) = opts

const eps_double_64 = Float64(eps(Double64))
function default_max_lost_digits(prec::PrecisionOption, accuracy::Float64)
    if prec == PRECISION_FIXED_64
        # maximal_digits_available - digits necessary - buffer
        -log10(eps()) + log10(accuracy) + 1
    else
        # if higher precision is available we will more like be constrained
        # by the fact that the jacobian cannot be too ill-conditioned
        min(-log10(eps()) - 3, -log10(eps_double_64) + log10(accuracy) - 3)
    end
end

####################
# CoreTrackerState #
####################

mutable struct CoreTrackerState{T, AV<:AbstractVector{T}, MaybePatchState <: Union{AbstractAffinePatchState, Nothing}, IP}
    x::AV # current x
    x̂::AV # last prediction
    x̄::AV # canidate for new x
    ẋ::Vector{T} # derivative at current x
    inner_product::IP
    step_size::StepSizeModel
    segment::ComplexSegment
    s::Float64 # current step length (0 ≤ s ≤ length(segment))
    Δs::Float64 # current step size
    Δs_prev::Float64 # previous step size
    accuracy::Float64
    jacobian::Jacobian{T}
    steps_jacobian_info_update::Int
    status::CoreTrackerStatus.states
    patch::MaybePatchState
    accepted_steps::Int
    rejected_steps::Int
    last_step_failed::Bool
    consecutive_successfull_steps::Int
end

function CoreTrackerState(H, x₁::AbstractVector, t₁, t₀, options::CoreTrackerOptions,
                           patch::Union{Nothing, AbstractAffinePatchState}=nothing,
                           inner_product::AbstractInnerProduct=EuclideanIP())
    if isa(x₁, SVector)
        x = Vector(x₁)
    else
        x = copy(x₁)
    end
    x̂, x̄ = copy(x), copy(x)
    ẋ = Vector(x)
    step_size_model = StepSizeModel()
    segment = ComplexSegment(promote(complex(t₁), complex(t₀))...)
    s = 0.0
    Δs = convert(Float64, min(options.initial_step_size, length(segment), options.max_step_size ))
    Δs_prev = 0.0
    accuracy = 0.0
    Jac = Jacobian(jacobian(H, x, t₁))
    steps_jacobian_info_update = 0
    accepted_steps = rejected_steps = 0
    status = CoreTrackerStatus.tracking
    last_step_failed = false
    consecutive_successfull_steps = 0
    CoreTrackerState(x, x̂, x̄, ẋ, inner_product, step_size_model, segment, s, Δs, Δs_prev, accuracy,
        Jac, steps_jacobian_info_update, status, patch,
        accepted_steps, rejected_steps, last_step_failed, consecutive_successfull_steps)
end

Base.show(io::IO, state::CoreTrackerState) = print_fieldnames(io, state)
Base.show(io::IO, ::MIME"application/prs.juno.inline", state::CoreTrackerState) = state

function reset!(state::CoreTrackerState, x₁::AbstractVector, t₁, t₀, options::CoreTrackerOptions, setup_patch::Bool, loop::Bool)
    state.segment = ComplexSegment(promote(t₁, t₀)...)
    state.s = 0.0
    state.Δs = min(options.initial_step_size, length(state.segment), options.max_step_size )
    state.Δs_prev = 0.0
    state.status = CoreTrackerStatus.tracking
    embed!(state.x, x₁)
    setup_patch && state.patch !== nothing && setup!(state.patch, state.x)
    state.last_step_failed = false
    state.consecutive_successfull_steps = 0
    if !loop
        state.accuracy = 0.0
        state.accepted_steps = state.rejected_steps = 0
        reset!(state.step_size)

        if options.auto_scaling
            init_auto_scaling!(state.inner_product, state.x, options.auto_scaling_options)
        end
        reset!(state.jacobian)
        state.steps_jacobian_info_update = 0
    end

    state
end

embed!(x::ProjectiveVectors.PVector, y) = ProjectiveVectors.embed!(x, y)
embed!(x::AbstractVector, y) = x .= y

function init_auto_scaling!(ip::WeightedIP, x::AbstractVector, opts::AutoScalingOptions)
    point_norm = euclidean_norm(x)
    w = ip.weight
    for i in 1:length(w)
        wᵢ = abs(x[i])
        if wᵢ < opts.scale_min * point_norm
            wᵢ = opts.scale_min * point_norm
        elseif wᵢ > opts.scale_max * point_norm
            wᵢ = opts.scale_max * point_norm
        end
        w[i] = max(wᵢ, opts.scale_abs_min)
    end
    nothing
end
init_auto_scaling!(ip::EuclideanIP, x::AbstractVector, opts::AutoScalingOptions) = nothing

####################
# CoreTrackerCache #
####################
mutable struct CoreTrackerCache{H<:HomotopyWithCache, P<:AbstractPredictorCache,
             C<:AbstractCorrectorCache, T}
    homotopy::H
    predictor::P
    corrector::C
    out::Vector{T}
    r::Vector{T}
end
function CoreTrackerCache(H::HomotopyWithCache, predictor, corrector, state::CoreTrackerState)
    t = state.segment[state.s]
    pcache = cache(predictor, H, state.x, state.ẋ, t)
    ccache = cache(corrector, H, state.x, t)
    out = H(state.x, t)
    r = copy(out)
    CoreTrackerCache(H, pcache, ccache, out, r)
end

###############
# CoreTracker #
###############
"""
    CoreTracker(problem::AbstractProblem, x₁; kwargs...)

Construct a `CoreTracker` from the given `problem` to track elements of type `x₁`.
The path is tracked using a predictor-corrector scheme. The recommended methods to construct
a `CoreTracker` are [`coretracker`](@ref) and [`coretracker_startsolutions`](@ref).
Note that a `CoreTracker` is also a (mutable) iterator.

## Keyword arguments
* `accuracy=1e-7`: The accuracy required during the path tracking.
* `corrector::AbstractCorrector`: The corrector used during in the predictor-corrector scheme. The default is [`NewtonCorrector`](@ref).
* `initial_step_size=0.1`: The size of the first step.
* `max_corrector_iters=2`: The maximal number of correction steps in a single step.
* `max_lost_digits::Real`: The tracking is terminated if we estimate that we loose more than `max_lost_digits` in the linear algebra steps. This threshold depends on the `precision` argument.
* `max_refinement_iters=5`: The maximal number of correction steps used to refine the final value.
* `max_steps=1_000`: The maximal number of iterations the path tracker has available. Note that this changes to `10_000` for parameter homotopies.
* `max_step_size=Inf`: The maximal step size.
* `min_step_size=1e-14`: The minimal step size.
* `precision::PrecisionOption=PRECISION_FIXED_64`: The precision used for evaluating the residual in Newton's method.
* `predictor::AbstractPredictor`: The predictor used during in the predictor-corrector scheme. The default is [`Heun`](@ref)()`.
* `refinement_accuracy=1e-8`: The precision required for the final value.
* `simple_step_size_alg=false`: Use a more simple step size algorithm.
* `steps_jacobian_info_update::Int=1`: Every n-th step a linear system will be solved using a QR factorization to obtain an estimate for the condition number of the Jacobian.
* `terminate_ill_conditioned::Bool=true`: Indicates whether the path tracking should be terminated for ill-conditioned paths. A path is considerd ill-conditioned if the condition number of the Jacobian is larger than ≈1e14 or if it is larger than 1e`max_lost_digits`.
"""
struct CoreTracker{H<:AbstractHomotopy,
    Predictor<:AbstractPredictor,
    Corrector<:AbstractCorrector,
    MaybePatch<:Union{Nothing, AbstractAffinePatch},
    S<:CoreTrackerState,
    C<:CoreTrackerCache}
    # these are fixed
    homotopy::H
    predictor::Predictor
    corrector::Corrector
    affine_patch::MaybePatch
    # these are mutable
    state::S
    options::CoreTrackerOptions
    cache::C
end

function CoreTracker(prob::AbstractProblem, x₁; kwargs...)
    CoreTracker(prob.homotopy, embed(prob, x₁), complex(1.0), complex(0.0), prob; kwargs...)
end

# Tracking in Projective Space
function CoreTracker(homotopy::AbstractHomotopy, x₁::ProjectiveVectors.PVector, t₁, t₀, prob::AbstractProblem;
    patch=has_dedicated_homvars(prob.vargroups) ? EmbeddingPatch() : OrthogonalPatch(),
    corrector::AbstractCorrector=NewtonCorrector(),
    predictor::AbstractPredictor=default_predictor(x₁),
    log_transform=false,
    simple_step_size_alg=!isa(patch, EmbeddingPatch), kwargs...)

    options = CoreTrackerOptions(;
                    parameter_homotopy=isa(homotopy, ParameterHomotopy),
                    kwargs...)

    if homotopy isa PatchedHomotopy
        error("You cannot pass a `PatchedHomotopy` to CoreTracker. Instead pass the homotopy and patch separate.")
    end

    H = log_transform ? LogHomotopy(homotopy) : homotopy
    patch_state = state(patch, x₁)
    # We close over the patch state, the homotopy and its cache
    # to be able to pass things around more easily
    HC = HomotopyWithCache(PatchedHomotopy(H, patch_state), x₁, t₁)
    if is_global_patch(patch) && options.auto_scaling
        inner_product = WeightedIP(x₁)
    else
        inner_product = EuclideanIP()
    end
    # We have to make sure that the element type of x is invariant under evaluation
    tracker_state = CoreTrackerState(HC, indempotent_x(HC, x₁, t₁), t₁, t₀, options, patch_state, inner_product)
    cache = CoreTrackerCache(HC, predictor, corrector, tracker_state)

    CoreTracker(H, predictor, corrector, patch, tracker_state, options, cache)
end

# Tracking in affine space
function CoreTracker(homotopy::AbstractHomotopy, x₁::AbstractVector, t₁, t₀, prob::AbstractProblem;
    patch=nothing,
    corrector::AbstractCorrector=NewtonCorrector(),
    predictor::AbstractPredictor=default_predictor(x₁),
    log_transform=false, kwargs...)

    if patch !== nothing
        throw(ArgumentError("You can only pass `patch=$(patch)` if `affine_tracking=false`."))
    end

    options = CoreTrackerOptions(;parameter_homotopy=isa(homotopy, ParameterHomotopy), kwargs...)

    H = log_transform ? LogHomotopy(homotopy) : homotopy
    # We close over the patch state, the homotopy and its cache
    # to be able to pass things around more easily
    HC = HomotopyWithCache(H, Vector(x₁), t₁)

    inner_product = options.auto_scaling ? WeightedIP(x₁) : EuclideanIP()
    # We have to make sure that the element type of x is invariant under evaluation
    tracker_state = CoreTrackerState(HC, indempotent_x(HC, x₁, t₁), t₁, t₀, options, nothing, inner_product)
    cache = CoreTrackerCache(HC, predictor, corrector, tracker_state)

    CoreTracker(H, predictor, corrector, nothing, tracker_state, options, cache)
end

default_predictor(x::AbstractVector) = Heun()
# Do not really understand this but Heun doesn't work that great for multi-homogeneous tracking
default_predictor(x::ProjectiveVectors.PVector{T,1}) where {T} = Heun()
default_predictor(x::ProjectiveVectors.PVector{T,N}) where {T, N} = Euler()

"""
    indempotent_x(H, x₁, t₁)

This returns a vector similar to `x₁` but with an element type which is invariant under evaluation.
"""
function indempotent_x(H, x₁, t₁)
    u = Vector{Any}(undef, size(H)[1])
    evaluate!(u, H, x₁, t₁)

    if isa(x₁, SVector)
        indem_x = Vector{promote_type(typeof(u[1]), ComplexF64)}(undef, length(x₁))
    else
        indem_x = similar(x₁, promote_type(typeof(u[1]), ComplexF64))
    end
    indem_x .= x₁
end

Base.show(io::IO, C::CoreTracker) = print(io, "CoreTracker tracking a path of type $(typeof(C.state.x))")
Base.show(io::IO, ::MIME"application/prs.juno.inline", x::CoreTracker) = x

##############
## TRACKING ##
##############

"""
    track(tracker, x₁, t₁=1.0, t₀=0.0; options...)::CoreTrackerResult

Track a value `x₁` from `t₁` to `t₀` using the given `CoreTracker` `tracker`.
This returns a `CoreTrackerResult`. This modifies `tracker`.
See [`track!`](@ref) for the possible options.
"""
function track(tracker::CoreTracker, x₁::AbstractVector, t₁=1.0, t₀=0.0; kwargs...)
     track!(tracker, x₁, t₁, t₀; kwargs...)
     CoreTrackerResult(tracker)
end

"""
     track!(tracker, x₁, t₁=1.0, t₀=0.0; setup_patch=true, checkstartvalue=true, loop::Bool=false)::CoreTrackerResult

Track a value `x₁` from `t₁` to `t₀` using the given `CoreTracker` `tracker`.
Returns one of the enum values of `CoreTrackerStatus.states` indicating the status.
If the tracking was successfull it is `CoreTrackerStatus.success`.
If `setup_patch` is `true` then [`setup!`](@ref) is called at the beginning
of the tracking.

    track!(x₀, tracker, x₁, t₁=1.0, t₀=0.0; options...)::CoreTrackerStatus.states

Additionally also stores the result in `x₀` if the tracking was successfull.
"""
function track!(x₀, tracker::CoreTracker, x₁, t₁=1.0, t₀=0.0; setup_patch::Bool=tracker.options.update_patch,
            loop::Bool=false, checkstartvalue::Bool=!loop)
     _track!(tracker, x₁, t₁, t₀, setup_patch, checkstartvalue, loop)
     retcode = status(tracker)
     if retcode == CoreTrackerStatus.success
         x₀ .= current_x(tracker)
     end
     retcode
end
@inline function track!(tracker::CoreTracker, x₁, t₁=1.0, t₀=0.0;
        setup_patch::Bool=tracker.options.update_patch,
        checkstartvalue::Bool=true,
        loop::Bool=false)
    _track!(tracker, x₁, t₁, t₀, setup_patch, checkstartvalue, loop)
end

function _track!(tracker::CoreTracker, x₁, t₁, t₀,
                    setup_patch::Bool, checkstartvalue::Bool, loop::Bool)
    if t₁ == t₀
        tracker.state.status = CoreTrackerStatus.success
        return tracker.state.status
    end

    setup!(tracker, x₁, t₁, t₀, setup_patch, checkstartvalue, loop)

    while tracker.state.status == CoreTrackerStatus.tracking
        step!(tracker)
        check_terminated!(tracker)
    end
    if tracker.state.status == CoreTrackerStatus.success
        refine!(tracker)
    end

    tracker.state.status
end

"""
    setup!(coretracker, x₁, t₁=1.0, t₀=0.0, setup_patch=coretracker.options.update_patch, checkstartvalue=true, compute_ẋ=true)

Setup `coretracker` to track `x₁` from `t₁` to `t₀`. Use this if you want to use the
coretracker as an iterator.
"""
function setup!(tracker::CoreTracker, x₁::AbstractVector, t₁=1.0, t₀=0.0, setup_patch=tracker.options.update_patch, checkstartvalue=true, loop::Bool=false)
    @unpack state, cache = tracker

    try
        reset!(state, x₁, t₁, t₀, tracker.options, setup_patch, loop)
        reset!(cache.predictor, state.x, t₁)
        checkstartvalue && checkstartvalue!(tracker)
        if !loop
            compute_ẋ!(state, cache, tracker.options)
            setup!(cache.predictor, cache.homotopy, state.x, state.ẋ, current_t(state), state.jacobian)
        end
    catch err
        if !(err isa LinearAlgebra.SingularException)
            rethrow(err)
        end
        tracker.state.status = CoreTrackerStatus.terminated_singularity
    end
    tracker
end

function checkstartvalue(tracker::CoreTracker, x, t)
    embed!(tracker.state.x̄, x)
    init_auto_scaling!(tracker.state.inner_product, tracker.state.x̄, tracker.options.auto_scaling_options)
    result = correct!(tracker.state.x̄, tracker, x, t; update_jacobian_infos=true)
    isconverged(result)
end

function checkstartvalue!(tracker::CoreTracker)
    result = correct!(tracker.state.x̄, tracker; update_jacobian_infos=true)
    if isconverged(result)
        tracker.state.x .= tracker.state.x̄
    else
        tracker.state.status = CoreTrackerStatus.terminated_invalid_startvalue
    end
    nothing
end

function compute_ẋ!(state, cache, options::CoreTrackerOptions)
    @inbounds jacobian_and_dt!(state.jacobian.J, cache.out, cache.homotopy, state.x, current_t(state))
    # apply row scaling to J and compute factorization
    updated_jacobian!(state.jacobian)

    @inbounds for i in eachindex(cache.out)
        cache.out[i] = -cache.out[i]
    end
    solve!(state.ẋ, state.jacobian, cache.out)
    nothing
end

@inline function correct!(x̄, tracker::CoreTracker,
        x=tracker.state.x,
        t=tracker.state.segment[tracker.state.s],
        norm=tracker.state.inner_product;
        accuracy::Float64=tracker.options.accuracy,
        max_iters::Int=tracker.options.max_corrector_iters,
        precision::PrecisionOption=tracker.options.precision,
        update_jacobian_infos::Bool=false,
        use_qr::Bool=false)

    correct!(x̄, tracker.corrector, tracker.cache.corrector,
             tracker.cache.homotopy, x, t, norm, tracker.state.jacobian,
             accuracy, max_iters, tracker.state.step_size;
             precision=precision,
             update_jacobian_infos=update_jacobian_infos, use_qr=use_qr)
end

function step!(tracker::CoreTracker)
    @unpack state, cache, options = tracker
    @unpack x, x̂, x̄, ẋ = state
    H = cache.homotopy

    try
        t, Δt = current_t(state), current_Δt(state)
        predict!(x̂, tracker.predictor, cache.predictor, H, x, t, Δt, ẋ, tracker.state.jacobian)
        # check if we need to update the jacobian_infos
        update_jacobian_infos =
                state.last_step_failed ||
                state.steps_jacobian_info_update ≥ options.steps_jacobian_info_update
        # reset counter
        update_jacobian_infos && (state.steps_jacobian_info_update = 0)
        result = correct!(x̄, tracker, x̂, t + Δt; update_jacobian_infos=update_jacobian_infos)
        # println("ω: ", result.ω, " t: ", real(t))
        if isconverged(result)
            # Step is accepted, assign values
            state.accepted_steps += 1
            x .= x̄
            state.s += state.Δs
            state.accuracy = result.accuracy
            # Step size change
            state.Δs_prev = state.Δs
            update_stepsize!(tracker, result)
            if state.patch !== nothing && options.update_patch
                changepatch!(state.patch, x)
            end
            options.auto_scaling && auto_scaling!(state, options)
            # update derivative
            compute_ẋ!(state, cache, options)
            # tell the predictors about the new derivative if they need to update something
            update!(cache.predictor, H, x, ẋ, t + Δt, state.jacobian)
            state.steps_jacobian_info_update += 1

            # Check termination criterion: we became too ill-conditioned
            if options.terminate_ill_conditioned && is_ill_conditioned(state, options) && state.last_step_failed
                state.status = CoreTrackerStatus.terminated_ill_conditioned
                return nothing
            end

            state.last_step_failed = false
        else
            # We have to reset the patch
            state.rejected_steps += 1
            update_rank!(state.jacobian)
            # Step failed, so we have to try with a new (smaller) step size
            update_stepsize!(tracker, result)
            Δt = current_Δt(state)
            state.last_step_failed = true
        end
    catch err
        if !(err isa LinearAlgebra.SingularException)
            rethrow(err)
        end
        tracker.state.status = CoreTrackerStatus.terminated_singularity
    end
    nothing
end

function is_ill_conditioned(state::CoreTrackerState, options::CoreTrackerOptions)
    d = max(log₁₀(cond(state)), digits_lost(state))
    d > options.max_lost_digits || state.jacobian.corank > 0
end

function is_overdetermined_tracking(tracker)
    m, n = size(tracker.cache.homotopy)
    m > n
end

function check_min_step_size!(tracker::CoreTracker{<:LogHomotopy})
    @unpack state, options = tracker
    if abs((exp(-state.Δs) - 1) * exp(-state.s)) < options.min_step_size
        state.status = CoreTrackerStatus.terminated_step_size_too_small
    end
end
function check_min_step_size!(tracker::CoreTracker)
    @unpack state, options = tracker
    if state.Δs < options.min_step_size
        state.status = CoreTrackerStatus.terminated_step_size_too_small
    end
end

g(Θ) = sqrt(1+4Θ) - 1
# Choose 0.25 instead of 1.0 due to Newton-Kantorovich theorem
δ(opts::CoreTrackerOptions, ω, μ) = @fastmath min(√(0.5ω) * τ(opts, μ), 0.25)
τ(opts::CoreTrackerOptions, μ) = @fastmath opts.accuracy^(1/(2opts.max_corrector_iters - μ))

function update_stepsize!(tracker::CoreTracker, result::CorrectorResult)
    @unpack state, options = tracker
    model = state.step_size
    ord = order(tracker.predictor)

    if options.simple_step_size_alg
        simple_step_size_alg!(tracker, result)
        return nothing
    end

    # The step size control is described in https://arxiv.org/abs/1902.02968

    # we have to handle the special case that there is only 1 iteration
    # in this case we cannot estimate ω and therefore just assume ω = 2
    # Also note ||x̂-x|| = ||Δx₀||
    if result.iters == 1
        ω = isnan(model.ω) ? 2.0 : 0.5model.ω
        d_x̂_x̄ = result.norm_Δx₀
    else
        ω = result.ω₀
        d_x̂_x̄ = distance(state.x̂, state.x, state.inner_product)
    end
    Δx₀ = result.norm_Δx₀
    if isconverged(result)
        δ_N_ω = δ(options, ω, 0.25)
        # This is to handle the edge case that g(δ_N_ω) >> (ω * d_x̂_x̄) > 0 but
        # at the same time δ_N_ω < eps(). Since then g(δ_N_ω) = 0
        δ_N_ω = max(δ_N_ω, 1e-15) #
        λ = g(δ_N_ω) / (ω * d_x̂_x̄)
        Δs = nthroot(λ, ord) * state.Δs
        if state.last_step_failed
            Δs = min(Δs, state.Δs)
        end
    else
        if isfinite(model.ω)
            ω = max(ω, model.ω)
        end
        δ_N_ω = δ(options, ω, 0.25)
        ω_η = 0.5ω * Δx₀
        if δ_N_ω < ω_η
            λ = g(δ_N_ω) / g(ω_η)
            Δs = min(nthroot(λ, ord), 0.9) * state.Δs
        else
            Δs = 0.5 * state.Δs
        end
        # If we fail consecutively reduce step size more aggressively.
        if state.last_step_failed
            Δs = min(Δs, 0.25 * state.Δs)
        end
    end
    model.expected_Δx₀ = 2δ_N_ω / ω
    model.ω = ω
    Δs = min(Δs, options.max_step_size)
    if (length(state.segment) - state.s) < Δs
        state.Δs = length(state.segment) - state.s
    else
        state.Δs = Δs
        check_min_step_size!(tracker)
    end

    nothing
end

function simple_step_size_alg!(tracker::CoreTracker, result::CorrectorResult)
    @unpack state, options = tracker
    if isconverged(result)
        state.consecutive_successfull_steps += 1
        if state.consecutive_successfull_steps == 5
            Δs′ = 2 * state.Δs
            state.consecutive_successfull_steps = 0
        else
            Δs′ = state.Δs
        end
    else
        state.consecutive_successfull_steps = 0
        Δs′ = 0.5 * state.Δs
    end

    state.Δs = min(Δs′, length(state.segment) - state.s)

    if !isconverged(result)
        check_min_step_size!(tracker)
    end
end

function auto_scaling!(state::CoreTrackerState, options::CoreTrackerOptions)
    auto_scaling!(state.inner_product, state.x, options.auto_scaling_options)
end
function auto_scaling!(ip::WeightedIP, x::AbstractVector, opts::AutoScalingOptions)
    norm_x = ip(x)
    for i in 1:length(x)
        wᵢ = (abs(x[i]) + ip.weight[i]) / 2
        if wᵢ < opts.scale_min * norm_x
            wᵢ = opts.scale_min * norm_x
        elseif wᵢ > opts.scale_max * norm_x
            wᵢ = opts.scale_max * norm_x
        end
        ip.weight[i] = max(wᵢ, opts.scale_abs_min)
    end
    nothing
end
auto_scaling!(ip::EuclideanIP, x::AbstractVector, opts::AutoScalingOptions) = nothing

function check_terminated!(tracker::CoreTracker)
    if abs(tracker.state.s - length(tracker.state.segment)) < 2eps(length(tracker.state.segment))
        tracker.state.status = CoreTrackerStatus.success
    elseif iters(tracker) ≥ tracker.options.max_steps
        tracker.state.status = CoreTrackerStatus.terminated_maximal_iterations
    end
    nothing
end

function refine!(tracker::CoreTracker, accuracy=tracker.options.refinement_accuracy)
    if tracker.state.accuracy < accuracy
        return
    end
    result = correct!(tracker.state.x̄, tracker;
        accuracy=accuracy,
        max_iters=tracker.options.max_refinement_iters)
    if isconverged(result)
        tracker.state.x .= tracker.state.x̄
        tracker.state.accuracy = result.accuracy
    end
    nothing
end

function Base.iterate(tracker::CoreTracker, state=nothing)
    state === nothing && return tracker, 1

    if tracker.state.status == CoreTrackerStatus.tracking
        step!(tracker)
        check_terminated!(tracker)

        if tracker.state.status == CoreTrackerStatus.success
            refine!(tracker)
        end

        tracker, state + 1
    else
        nothing
    end
end

"""
    affine_tracking(tracker)

Returns `true` if the path tracker tracks in affine space. If `false` then then the path tracker
tracks in some affine chart of the projective space.
"""
affine_tracking(tracker::CoreTracker) = !isa(tracker.state.x, ProjectiveVectors.PVector)

#################
## Query State ##
#################
"""
     current_t(tracker::CoreTracker)

Current `t`.
"""
current_t(tracker::CoreTracker) = current_t(tracker.state)
current_t(state::CoreTrackerState) = state.segment[state.s]

"""
     current_Δt(tracker::CoreTracker)

Current step_size `Δt`.
"""
current_Δt(tracker::CoreTracker) = current_Δt(tracker.state)
current_Δt(state::CoreTrackerState) = state.segment[state.Δs] - state.segment.start

"""
     iters(tracker::CoreTracker)

Current number of iterations.
"""
iters(tracker::CoreTracker) = iters(tracker.state)
iters(state::CoreTrackerState) = state.accepted_steps + state.rejected_steps

"""
     status(tracker::CoreTracker)

Current status.
"""
status(tracker::CoreTracker) = status(tracker.state)
status(state::CoreTrackerState) = state.status

"""
    current_x(tracker::CoreTracker)

Return the current value of `x`.
"""
current_x(tracker::CoreTracker) = current_x(tracker.state)
current_x(state::CoreTrackerState) = state.x

"""
    LinearAlgebra.cond(tracker::CoreTracker)

Returns the currently computed approximation of the condition number of the
Jacobian.
"""
cond(tracker::CoreTracker) = LinearAlgebra.cond(tracker.state)
cond(state::CoreTrackerState) = state.jacobian.cond

"""
    digits_lost(tracker::CoreTracker)

Returns the currently computed approximation of the number of digits lost
during the linear system solving in Newton's method.
"""
digits_lost(tracker::CoreTracker) = digits_lost(tracker.state)
digits_lost(state::CoreTrackerState) = unpack(state.jacobian.digits_lost, 0.0)

"""
    inner(tracker::CoreTracker)

Returns the inner product used to compute distance during the path tracking.
"""
inner(tracker::CoreTracker) = inner(tracker.state)
inner(state::CoreTrackerState) = state.inner_product

"""
    options(tracker::CoreTracker)

Returns the options used in the tracker.
"""
options(tracker::CoreTracker) = tracker.options

##################
# Modify options #
##################
"""
     accuracy(tracker::CoreTracker)

Current accuracy.
"""
accuracy(tracker::CoreTracker) = tracker.options.accuracy

"""
     set_accuracy!(tracker::CoreTracker, accuracy, update_max_lost_digits=true)

Set the current accuracy to `accuracy`. If `update_max_lost_digits` is `true` then
the setting `max_lost_digits` will be updated to the default setting.
"""
function set_accuracy!(tracker::CoreTracker, accuracy, update_max_lost_digits::Bool=true)
    @unpack options = tracker
    options.accuracy = accuracy
    if update_max_lost_digits
        options.max_lost_digits = default_max_lost_digits(options.precision, accuracy)
    end
    tracker
end

"""
     refinement_accuracy(tracker::CoreTracker)

Current refinement accuracy.
"""
refinement_accuracy(tracker::CoreTracker) = tracker.options.refinement_accuracy

"""
     set_max_refinement_iters!(tracker::CoreTracker, accuracy)

Set the current refinement accuracy to `accuracy`.
"""
set_refinement_accuracy!(T::CoreTracker, accuracy) = T.options.refinement_accuracy = accuracy

"""
     max_refinement_iters(tracker::CoreTracker)

Current refinement max_steps.
"""
max_refinement_iters(T::CoreTracker) = T.options.max_refinement_iters

"""
     set_max_refinement_iters!(tracker::CoreTracker, n)

Set the current refinement max_steps to `n`.
"""
set_max_refinement_iters!(T::CoreTracker, n) = T.options.max_refinement_iters = n

"""
     max_corrector_iters(tracker::CoreTracker)

Current correction max_steps.
"""
max_corrector_iters(T::CoreTracker) = T.options.max_corrector_iters

"""
     set_max_corrector_iters!(tracker::CoreTracker, n)

Set the correction max_steps to `n`.
"""
set_max_corrector_iters!(T::CoreTracker, n) = T.options.max_corrector_iters = n

"""
     max_step_size (tracker::CoreTracker)

Current maximal step size.
"""
max_step_size(T::CoreTracker) = T.options.max_step_size

"""
     set_max_corrector_iters!(tracker::CoreTracker, Δs)

Set the maximal step size to `Δs`.
"""
set_max_step_size!(T::CoreTracker, Δs) = T.options.max_step_size  = Δs

################
# PathIterator #
################
struct PathIterator{Tracker<:CoreTracker}
    tracker::Tracker
    t_real::Bool
end
Base.IteratorSize(::Type{<:PathIterator}) = Base.SizeUnknown()
Base.IteratorEltype(::Type{<:PathIterator}) = Base.HasEltype()

"""
    iterator(tracker::CoreTracker, x₁, t₁=1.0, t₀=0.0; affine=true)

Prepare a tracker to make it usable as a (stateful) iterator. Use this if you want to inspect a specific
path. In each iteration the tuple `(x,t)` is returned.
If `affine == true` then `x` is the affine solution (internally we compute in projective space).

## Example

Assume you have `CoreTracker` `tracker` and you wan to track `x₁` from 1.0 to 0.25:
```julia
for (x,t) in iterator(tracker, x₁, 1.0, 0.25)
    println("x at t=\$t:")
    println(x)
end
```

Note that this is a stateful iterator. You can still introspect the state of the tracker.
For example to check whether the tracker was successfull
(and did not terminate early due to some problem) you can do
```julia
println("Success: ", status(tracker) == CoreTrackerStatus.success)
```
"""
function iterator(tracker::CoreTracker, x₁, t₁=1.0, t₀=0.0; kwargs...)
    setup!(tracker, x₁, t₁, t₀; kwargs...)
    PathIterator(tracker, typeof(t₁ - t₀) <: Real)
end

function current_x_t(iter::PathIterator)
    x = current_x(iter.tracker)
    t = current_t(iter.tracker)
    (copy(x), iter.t_real ? real(t) : t)
end

function Base.iterate(iter::PathIterator, state=nothing)
    state === nothing && return current_x_t(iter), 1
    iter.tracker.state.status != CoreTrackerStatus.tracking && return nothing

    step_done = false
    while !step_done && (iter.tracker.state.status == CoreTrackerStatus.tracking)

        step!(iter.tracker)
        check_terminated!(iter.tracker)

        if iter.tracker.state.status == CoreTrackerStatus.success
            refine!(iter.tracker)
        end

        step_done = !iter.tracker.state.last_step_failed
    end
    current_x_t(iter), state + 1
end

#############################
# Convencience constructors #
#############################
"""
    coretracker_startsolutions(args...; kwargs...)

Construct a [`CoreTracker`](@ref) and `startsolutions` in the same way `solve`
does it. This also takes the same input arguments as `solve`. This is convenient if you want
to investigate single paths.
"""
function coretracker_startsolutions(args...; kwargs...)
    supported, rest = splitkwargs(kwargs,problem_startsolutions_supported_keywords)
    prob, startsolutions = problem_startsolutions(args...; supported...)
    tracker = CoreTracker(prob, start_solution_sample(startsolutions); rest...)
    (tracker=tracker, startsolutions=startsolutions)
end

"""
    coretracker(args...; kwargs...)

Construct a [`CoreTracker`](@ref) in the same way `solve`
does it. This also takes the same input arguments as `solve` with the exception that you do not need to specify startsolutions.
This is convenient if you want to investigate single paths.

## Examples

### Obtain single solution
We want to construct a path tracker to track a parameterized system `f` with parameters `p`
from the parameters `a` to `b`.
```julia
tracker = coretracker(f, parameters=p, p₁=a, p₀=b)
```
You then can obtain a single solution at `b` by using
```julia
x_b = track(tracker, x_a).x
```

### Trace a path
To trace a path you can use the [`iterator`](@ref) method.

```julia
tracker = coretracker(f, parameters=p, p₁=a, p₀=b, max_step_size =0.01)
for (x, t) in iterator(tracker, x₁)
    @show (x,t)
end
```

If we want to guarantee smooth traces we can limit the maximal step size.
```julia
tracker = coretracker(f, parameters=p, p₁=a, p₀=b, max_step_size =0.01)
for (x, t) in iterator(tracker, x₁)
    @show (x,t)
end
```
"""
function coretracker(args...; kwargs...)
    tracker, _ = coretracker_startsolutions(args...; kwargs...)
    tracker
end


Array{T}(::UndefInitializer, ::Tuple{PosInfinity}) where T = throw(ArgumentError("Cannot create infinite Array"))
Array{T}(::UndefInitializer, ::Tuple{Integer, PosInfinity}) where {T,N} = throw(ArgumentError("Cannot create infinite Array"))
Array{T}(::UndefInitializer, ::Tuple{PosInfinity, Integer}) where {T,N} = throw(ArgumentError("Cannot create infinite Array"))
Matrix{T}(::UndefInitializer, ::Tuple{Integer, PosInfinity}) where T = throw(ArgumentError("Cannot create infinite Array"))
Matrix{T}(::UndefInitializer, ::Tuple{PosInfinity, Integer}) where T = throw(ArgumentError("Cannot create infinite Array"))
Array{T}(::UndefInitializer, ::Tuple{PosInfinity, PosInfinity}) where {T,N} = throw(ArgumentError("Cannot create infinite Array"))
Matrix{T}(::UndefInitializer, ::Tuple{PosInfinity, PosInfinity}) where T = throw(ArgumentError("Cannot create infinite Array"))

Array{T}(::UndefInitializer, ::PosInfinity) where T = throw(ArgumentError("Cannot create infinite Array"))

Array{T}(::UndefInitializer, ::PosInfinity, ::PosInfinity) where T = throw(ArgumentError("Cannot create infinite Array"))
Array{T}(::UndefInitializer, ::PosInfinity, ::Integer) where T = throw(ArgumentError("Cannot create infinite Array"))
Array{T}(::UndefInitializer, ::Integer, ::PosInfinity) where T = throw(ArgumentError("Cannot create infinite Array"))

Matrix{T}(::UndefInitializer, ::PosInfinity, ::PosInfinity) where T = throw(ArgumentError("Cannot create infinite Array"))
Matrix{T}(::UndefInitializer, ::PosInfinity, ::Integer) where T = throw(ArgumentError("Cannot create infinite Array"))
Matrix{T}(::UndefInitializer, ::Integer, ::PosInfinity) where T = throw(ArgumentError("Cannot create infinite Array"))

Vector{T}(::UndefInitializer, ::Tuple{PosInfinity}) where T = throw(ArgumentError("Cannot create infinite Array"))
Vector{T}(::UndefInitializer, ::PosInfinity) where T = throw(ArgumentError("Cannot create infinite Array"))

similar(A::AbstractArray, ::Type{T}, axes::Tuple{OneToInf{Int}}) where T = cache(Zeros{T}(axes))
similar(A::AbstractArray, ::Type{T}, axes::Tuple{OneToInf{Int},OneToInf{Int}}) where T = cache(Zeros{T}(axes))
similar(A::AbstractArray, ::Type{T}, dims::Tuple{PosInfinity}) where T = cache(Zeros{T}(dims))
similar(A::AbstractArray, ::Type{T}, dims::Tuple{PosInfinity,PosInfinity}) where T = cache(Zeros{T}(dims))
similar(A::AbstractArray, ::Type{T}, dims::Tuple{Integer,PosInfinity}) where T = cache(Zeros{T}(dims))
similar(A::AbstractArray, ::Type{T}, dims::Tuple{PosInfinity,Integer}) where T = cache(Zeros{T}(dims))

similar(::Type{<:AbstractArray{T}}, axes::Tuple{OneToInf{Int}}) where T = cache(Zeros{T}(axes))
similar(::Type{<:AbstractArray{T}}, axes::Tuple{OneToInf{Int},OneToInf{Int}}) where T = cache(Zeros{T}(axes))
similar(::Type{<:AbstractArray{T}}, axes::Tuple{OneToInf{Int},OneTo{Int}}) where T = cache(Zeros{T}(axes))
similar(::Type{<:AbstractArray{T}}, axes::Tuple{OneTo{Int},OneToInf{Int}}) where T = cache(Zeros{T}(axes))
similar(::Type{<:AbstractArray{T}}, dims::Tuple{PosInfinity}) where T = cache(Zeros{T}(dims))
similar(::Type{<:AbstractArray{T}}, dims::Tuple{PosInfinity,PosInfinity}) where T = cache(Zeros{T}(dims))
similar(::Type{<:AbstractArray{T}}, dims::Tuple{Integer,PosInfinity}) where T = cache(Zeros{T}(dims))
similar(::Type{<:AbstractArray{T}}, dims::Tuple{PosInfinity,Integer}) where T = cache(Zeros{T}(dims))

similar(arr::AbstractArray{T}, sz::Union{Infinity,Integer,AbstractUnitRange}...) where T = similar(arr, Base.to_shape(sz)...)
similar(arr::AbstractArray, ::Type{T}, sz::Union{Infinity,Integer,AbstractUnitRange}...) where T = similar(arr, T, Base.to_shape(sz)...)

for Typ in (:Zeros, :Ones)
    @eval begin
        @inline $Typ{T, N}(sz::Tuple{Vararg{Union{Infinity,Integer}, N}}) where {T, N} = $Typ{T,N}(oneto.(sz))
        @inline $Typ{T, N}(sz::Vararg{Union{Infinity,Integer}, N}) where {T, N} = $Typ{T,N}(sz)
        @inline $Typ{T}(sz::Vararg{Union{Infinity,Integer},N}) where {T, N} = $Typ{T, N}(sz)
        @inline $Typ{T}(sz::Infinity) where T = $Typ{T,1}(sz)
    end
end

Eye{T}(::Infinity) where T = Eye{T}(ℵ₀)
Eye{T}(::Infinity, ::Infinity) where T = Eye{T}(ℵ₀, ℵ₀)
Eye{T}(::Infinity, n::Integer) where T = Eye{T}(ℵ₀, n)
Eye{T}(m::Integer, ::Infinity) where T = Eye{T}(m, ℵ₀)
Eye(::Infinity) = Eye(ℵ₀)
Eye(::Infinity, ::Infinity) = Eye(ℵ₀, ℵ₀)
Eye(::Infinity, n::Integer) = Eye(ℵ₀, n)
Eye(m::Integer, ::Infinity) = Eye(m, ℵ₀)

@inline Fill(x, sz::Union{Infinity,Integer}...) = Fill(x, Base.to_shape(sz)...)
@inline Fill{T}(x, sz::Union{Infinity,Integer}...) where T = Fill{T}(x, Base.to_shape(sz)...)

zeros(sz::Union{Infinity, Integer, AbstractUnitRange}...) = zeros(Base.to_shape(sz)...)
zeros(::Type{T}, sz::Union{Infinity, Integer, AbstractUnitRange}...) where T = zeros(T, Base.to_shape(sz)...)
zeros(::Type{T}, ::Tuple{PosInfinity}) where T = cache(Zeros{T}(∞))
zeros(::Type{T}, nm::Tuple{Integer, PosInfinity}) where T = cache(Zeros{T}(nm...))
zeros(::Type{T}, nm::Tuple{PosInfinity, Integer}) where T = cache(Zeros{T}(nm...))
zeros(::Type{T}, nm::Tuple{PosInfinity, PosInfinity}) where T = cache(Zeros{T}(nm...))

ones(sz::Union{Infinity, Integer, AbstractUnitRange}...) = ones(Base.to_shape(sz)...)
ones(::Type{T}, ::Tuple{PosInfinity}) where T = cache(Ones{T}(∞))
ones(::Type{T}, nm::Tuple{Integer, PosInfinity}) where T = cache(Ones{T}(nm...))
ones(::Type{T}, nm::Tuple{PosInfinity, Integer}) where T = cache(Ones{T}(nm...))
ones(::Type{T}, nm::Tuple{PosInfinity, PosInfinity}) where T = cache(Ones{T}(nm...))

fill(x, sz::Union{Infinity, Integer, AbstractUnitRange}...) = fill(x, Base.to_shape(sz)...)
fill(x, ::Tuple{PosInfinity}) = cache(Fill(x,∞))
fill(x, nm::Tuple{Integer, PosInfinity}) = cache(Fill(x,nm...))
fill(x, nm::Tuple{PosInfinity, Integer}) = cache(Fill(x,nm...))
fill(x, nm::Tuple{PosInfinity, PosInfinity}) = cache(Fill(x,nm...))

# This gets called when infinit number of columns
axes_print_matrix_row(_, io, X, A, i, ::AbstractVector{<:PosInfinity}, sep) = nothing
print_matrix_row(io::IO, X::AbstractVecOrMat, A::Vector, i::Integer, cols::AbstractVector{<:PosInfinity}, sep::AbstractString) = nothing
Base.print_matrix_row(io::IO,
        X::Union{LayoutMatrix,
        LayoutVector,
        AbstractTriangular{<:Any,<:LayoutMatrix},
        AdjOrTrans{<:Any,<:LayoutMatrix},
        AdjOrTrans{<:Any,<:LayoutVector},
        HermOrSym{<:Any,<:LayoutMatrix},
        SubArray{<:Any,2,<:LayoutMatrix},
        Diagonal{<:Any,<:LayoutVector}}, A::Vector,
        i::Integer, cols::AbstractVector{<:PosInfinity}, sep::AbstractString) =
    axes_print_matrix_row(axes(X), io, X, A, i, cols, sep)
Base.print_matrix_row(io::IO,
        X::Union{AbstractFill{<:Any,1},
                 AbstractFill{<:Any,2},
                 Diagonal{<:Any,<:AbstractFill{<:Any,1}},
                 RectDiagonal,
                 AbstractTriangular{<:Any,<:AbstractFill{<:Any,2}}
                 }, A::Vector,
        i::Integer, cols::AbstractVector{<:PosInfinity}, sep::AbstractString) =
    axes_print_matrix_row(axes(X), io, X, A, i, cols, sep)

print_matrix_vdots(io::IO, vdots::AbstractString,
        A::Vector, sep::AbstractString, M::Integer, ::NotANumber, pad_right::Bool = true) = nothing

# Avoid infinite loops on maximum
Base.mapreduce_impl(f, op, A::AbstractArray, ifirst::Integer, ::PosInfinity) =
    throw(ArgumentError("Cannot call mapreduce on an infinite length $(typeof(A))"))

function show_delim_array(io::IO, itr::AbstractArray, op, delim, cl,
                          delim_one, i1, ::PosInfinity)
    print(io, op)
    l = 20
    if !show_circular(io, itr)
        recur_io = IOContext(io, :SHOWN_SET => itr)
        if !haskey(io, :compact)
          recur_io = IOContext(recur_io, :compact => true)
        end
        first = true
        i = i1
        if 20 >= i1
          while true
              if !isassigned(itr, i)
                  print(io, undef_ref_str)
              else
                  x = itr[i]
                  show(recur_io, x)
              end
              i += 1
              if i > l
                  print(io, delim)
                  print(io, ' ')
                  print(io, '…')
                  delim_one && first && print(io, delim)
                  break
              end
              first = false
              print(io, delim)
              print(io, ' ')
          end
        end
    end
    print(io, cl)
end

#####
# FillArrays
#####

# Lazy Broadacasting
for typ in (:Ones, :Zeros, :Fill)
    @eval begin
        BroadcastStyle(::Type{$typ{T,N,NTuple{N,<:OneToInf}}}) where {T,N} = LazyArrayStyle{N}()
        BroadcastStyle(::Type{$typ{T,2,<:Tuple{<:Any,<:OneToInf}}}) where {T} = LazyArrayStyle{2}()
        BroadcastStyle(::Type{$typ{T,2,<:Tuple{<:OneToInf,<:Any}}}) where {T} = LazyArrayStyle{2}()
    end
end

BroadcastStyle(::Type{<:Diagonal{T,<:AbstractFill{T,1,Tuple{OneToInf{I}}}}}) where {T,I} = LazyArrayStyle{2}()

## Support broadcast(*, ::AbstractFill, A)

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{Ones{T,1,Tuple{OneToInf{Int}}},AbstractArray{V,N}}}) where {N,T,V}
    a,b = bc.args
    @assert bc.axes == axes(b)
    convert(AbstractArray{promote_type(T,V),N}, b)
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{AbstractArray{T,N},Ones{V,1,Tuple{OneToInf{Int}}}}}) where {N,T,V}
    a,b = bc.args
    @assert bc.axes == axes(a)
    convert(AbstractArray{promote_type(T,V),N}, a)
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{AbstractFill{T,1,Tuple{OneToInf{Int}}},AbstractArray{V,N}}}) where {N,T<:Number,V}
    a,b = bc.args
    @assert bc.axes == axes(b)
    getindex_value(a) * b
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{AbstractArray{T,N},AbstractFill{V,1,Tuple{OneToInf{Int}}}}}) where {N,T,V<:Number}
    a,b = bc.args
    @assert bc.axes == axes(a)
    a * getindex_value(b)
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{AbstractFill{T,1,Tuple{OneToInf{Int}}},AbstractArray{V,N}}}) where {N,T,V}
    a,b = bc.args
    @assert bc.axes == axes(b)
    Ref(getindex_value(a)) .* b # Use broadcast in-case a is array-valued
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{AbstractArray{T,N},AbstractFill{V,1,Tuple{OneToInf{Int}}}}}) where {N,T,V}
    a,b = bc.args
    @assert bc.axes == axes(a)
    a .* Ref(getindex_value(b)) # Use broadcast in-case b is array-valued
end

# row Vector
function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{Adjoint{T,Ones{T,1,Tuple{OneToInf{Int}}}},AbstractMatrix{V}}}) where {T,V}
    a,b = bc.args
    @assert bc.axes == axes(b)
    convert(AbstractMatrix{promote_type(T,V)}, b)
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{AbstractMatrix{T},Adjoint{V,Ones{V,1,Tuple{OneToInf{Int}}}}}}) where {T,V}
    a,b = bc.args
    @assert bc.axes == axes(a)
    convert(AbstractMatrix{promote_type(T,V)}, a)
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{Adjoint{T,<:AbstractFill{T,1,Tuple{OneToInf{Int}}}},AbstractMatrix{V}}}) where {T,V}
    a,b = bc.args
    @assert bc.axes == axes(b)
    getindex_value(a') * b
end

function copy(bc::Broadcasted{<:BroadcastStyle,<:Any,typeof(*),<:Tuple{AbstractMatrix{T},Adjoint{V,<:AbstractFill{V,1,Tuple{OneToInf{Int}}}}}}) where {T,V}
    a,b = bc.args
    @assert bc.axes == axes(a)
    a * getindex_value(b')
end

#####
# Diagonal
#####

BroadcastStyle(::Type{<:Diagonal{<:Any,<:AbstractInfUnitRange}}) = LazyArrayStyle{2}()

#####
# Vcat length
#####

function getindex(f::Vcat{T,1}, k::PosInfinity) where T
    length(f) == ℵ₀ || throw(BoundsError(f,k))
    ∞
end

_gettail(k, a::Number, b...) = k ≤ 1 ? tuple(a, b...) : _gettail(k - length(a), b...)
_gettail(k, a, b...) = k ≤ length(a) ? tuple(a[k:end], b...) : _gettail(k - length(a), b...)
_vcat(a) = a
_vcat(a, b, c...) = Vcat(a, b, c...)
getindex(A::Vcat, r::InfUnitRange) = Base.invoke(getindex, Tuple{AbstractArray, Any}, A, r)
_unsafe_getindex(::IndexLinear, A::Vcat, r::InfUnitRange) = _vcat(_gettail(first(r), A.args...)...)

getindex(A::AbstractVector, r::InfRanges{<:Integer}) = layout_getindex(A, r)
getindex(A::LayoutVector, r::InfRanges{<:Integer}) = layout_getindex(A, r)
getindex(A::CachedVector, r::InfRanges{<:Integer}) = layout_getindex(A, r)
getindex(A::AbstractFill{<:Any,1}, r::InfRanges{<:Integer}) = FillArrays.fillsimilar(A, length(r))

Base.checkindex(::Type{Bool}, inds::AbstractUnitRange, I::AbstractFill) = Base.checkindex(Bool, inds, getindex_value(I))
LazyArrays.cache_getindex(::InfiniteCardinal{0}, A::AbstractVector, I, J...) = layout_getindex(A, I, J...)


__precompile__(true)
module ParticleSwarmOptimizer

include("TestFunctions.jl")
include("helper.jl")
include("neighbourhood.jl")
include("pso.jl")

export TestFunctions

end # module


const outputname = joinpath(@__DIR__, "currency-data.jl")

# First, check if currency-data.jl already exists
isfile(outputname) && exit()

using JSON3

const src = "https://pkgstore.datahub.io/core/country-codes/country-codes_json/data/471a2e653140ecdd7243cdcacfd66608/country-codes_json.json"

const inputname = joinpath(@__DIR__, "country-codes.json")

const currency_list = Dict{String,Tuple{Int,Int,String}}()

const SymCurr = Symbol("ISO4217-currency_alphabetic_code")
const SymUnit = Symbol("ISO4217-currency_minor_unit")
const SymName = Symbol("ISO4217-currency_name")
const SymCode = Symbol("ISO4217-currency_numeric_code")

function genfile(io)
    for country in country_list
        (abbrlist = country[SymCurr]) === nothing && continue
        (unitlist = country[SymUnit]) === nothing && continue
        (namelist = country[SymName]) === nothing && continue
        (codelist = country[SymCode]) === nothing && continue
        currencies = split(abbrlist, ',')
        units = split(string(unitlist), ',')
        names = split(namelist, ',')
        codes = split(string(codelist), ',')

        for (curr, unit, code, name) in zip(currencies, units, codes, names)
            length(curr) != 3 && continue
            haskey(currency_list, curr) && continue
            currency_list[curr] = (parse(Int, unit), parse(Int, code), string(name))
        end
    end
    println(io, "const _currency_data = Dict(")
    for (curr, val) in currency_list
        println(io, "    :$curr => (Currency{:$curr}, $(val[1]), $(lpad(val[2], 4)), \"$(val[3])\"),")
    end
    println(io, ")\n")
end

# Only download the file from datahub.io if not already present
if !isfile(inputname)
    println("Downloading currency data: ", src)
    download(src, inputname)
end

const country_list = open(io -> JSON3.read(io), inputname)

open(io -> genfile(io), outputname, "w")


using BenchmarkTools
using Augmentor, Images, TestImages

const SUITE = BenchmarkGroup()

pattern = testpattern()
affpattern = Augmentor.prepareaffine(pattern)
camera = testimage("cameraman")
cameras = similar(camera, size(camera)..., 2)
copy!(view(cameras,:,:,1), camera)
copy!(view(cameras,:,:,2), camera)

# compile the gauss filter (takes a couple seconds)
Augmentor.applyeager(ElasticDistortion(10), pattern)

shortname(op) = string(typeof(op).name.name)

# missing: CombineChannels, CacheImageInto
SUITE["applyeager"] = BenchmarkGroup(["eager", "operations"])
for op in [
        NoOp(),
        FlipX(), FlipY(),
        ShearX(5), ShearY(5),
        Either((Rotate90(), Rotate270()), (1,0)),
        Rotate90(), Rotate180(), Rotate270(),
        Rotate(45),
        Crop(1:100,1:100),
        CropNative(1:100,1:100),
        CropSize(100,100),
        CropRatio(1), RCropRatio(1),
        Scale(1.1), Zoom(1.1),
        Resize(100,100),
        ConvertEltype(Gray),
        CacheImage(),
        ElasticDistortion(10),
        SplitChannels(),
        Reshape(400,300),
        PermuteDims(2,1),
    ]
    SUITE["applyeager"][shortname(op)] = @benchmarkable Augmentor.applyeager($op, $pattern)
end

SUITE["applylazy"] = BenchmarkGroup(["lazy", "operations"])
for op in [
        NoOp(),
        FlipX(), FlipY(),
        ShearX(5), ShearY(5),
        Either((Rotate90(), Rotate270()), (1,0)),
        Rotate90(), Rotate180(), Rotate270(),
        Rotate(45),
        Crop(1:100,1:100),
        CropNative(1:100,1:100),
        CropSize(100,100),
        CropRatio(1), RCropRatio(1),
        Scale(1.1), Zoom(1.1),
        Resize(100,100),
        ConvertEltype(Gray),
        ElasticDistortion(10),
        SplitChannels(),
        Reshape(400,300),
        PermuteDims(2,1),
    ]
    SUITE["applylazy"][shortname(op)] = @benchmarkable Augmentor.plain_array(Augmentor.applylazy($op, $pattern))
end

SUITE["applyaffine"] = BenchmarkGroup(["affine", "lazy", "operations"])
for op in [
        NoOp(),
        FlipX(), FlipY(),
        ShearX(5), ShearY(5),
        Either((Rotate90(), Rotate270()), (1,0)),
        Rotate90(), Rotate180(), Rotate270(),
        Rotate(45),
        Scale(1.1),
    ]
    SUITE["applyaffine"][shortname(op)] = @benchmarkable Augmentor.plain_array(Augmentor.applyaffine($op, $affpattern))
end

SUITE["applyaffineview"] = BenchmarkGroup(["affine", "lazy", "operations"])
for op in [
        NoOp(),
        FlipX(), FlipY(),
        ShearX(5), ShearY(5),
        Either((Rotate90(), Rotate270()), (1,0)),
        Rotate90(), Rotate180(), Rotate270(),
        Rotate(45),
        Crop(1:100,1:100),
        CropNative(1:100,1:100),
        CropSize(100,100),
        CropRatio(1), RCropRatio(1),
        Scale(1.1), Zoom(1.1),
        Resize(100,100),
    ]
    SUITE["applyaffineview"][shortname(op)] = @benchmarkable Augmentor.plain_array(Augmentor.applyaffineview($op, $affpattern))
end

grp = SUITE["augment"] = BenchmarkGroup(["-"])
pl = ShearX(10)
grp["affine1"] = @benchmarkable augment($pattern, $pl)
pl = ShearX(10) |> ShearX(-10)
grp["affine2"] = @benchmarkable augment($pattern, $pl)
pl = ShearX(10) |> ShearX(-10) |> ShearY(10) |> ShearY(-10)
grp["affine3"] = @benchmarkable augment($pattern, $pl)
pl = Rotate180() |> Rotate90(1)
grp["lazyrotate"] = @benchmarkable augment($pattern, $pl)
pl = Rotate180() |> Crop(5:200,100:400) |> Rotate90(1) |> Crop(1:250,1:150)
grp["lazycrop"] = @benchmarkable augment($pattern, $pl)
pl = Resize(100,100) |> Resize(200,400) |> Resize(20,20) |> Resize(100,100)
grp["resize"] = @benchmarkable augment($pattern, $pl)

grp = SUITE["augment!"] = BenchmarkGroup(["-"])
pl = Rotate180() |> Rotate90()
out = similar(pattern, 400, 300)
grp["lazyrotate"] = @benchmarkable augment!($out, $pattern, $pl)
pl = Resize(100,100) |> Resize(200,400) |> Resize(20,20) |> Resize(100,100)
out = similar(pattern, 100, 100)
grp["resize"] = @benchmarkable augment!($out, $pattern, $pl)


function load_genelist(genelist, description="entest")
    url = "http://maayanlab.cloud/Enrichr/addList"
    genestr = join(genelist, "\n")
    payload = Dict("list" => genestr, "description" => description)
    r = requests.post(url, files=payload)
    !r.ok && error("Error loading list")
    JSON.parse(r.text)
end

function view_genelist(config)
    url = string("http://maayanlab.cloud/Enrichr/view?userListId=", config["userListId"])
    r = requests.get(url)
    !r.ok && error("Error fetching results")
    JSON.parse(r.text)
end

function enrichment(config, geneset)
    url = "http://maayanlab.cloud/Enrichr/enrich"
    querystring = string("?userListId=", config["userListId"], "&backgroundType=", geneset)
    r = requests.get(string(url, querystring))
    !r.ok && error("Error fetching results")
    data = JSON.parse(r.text)

    df = DataFrame(GeneSet=String[], Rank=Int[], Term=String[], pvalue=Float64[], zscore=Float64[], Combined=Float64[], Genes=Vector{String}[], padj=Float64[])

    for record in data[geneset]
        push!(df, ([geneset ; record[1:end-2]]))
    end
    df
end


m = PoolModel("ss1")
fp = dirname(@__FILE__)

m <= Setting(:dataroot, "$(fp)/../reference/")
m <= Setting(:data_vintage, "190822")
observables = OrderedDict{Symbol,Observable}()
identity_transform_805 = function(levels)
    return levels[!,:p805]
end
identity_transform_904 = function(levels)
    return levels[!,:p904]
end
observables[:Model904] = Observable(:Model904, [:p904__wrongorigmatlab],
                                    identity_transform_904,
                                    identity_transform_904,
                                    "Model 904 predictive density",
                                    "Model 904 conditional predictive density scores")
observables[:Model805] = Observable(:Model805, [:p805__wrongorigmatlab],
                                    identity_transform_805,
                                    identity_transform_805,
                                    "Model 805 predictive density",
                                    "Model 805 conditional predictive density scores")

@testset "Test various forms of data loading" begin
    m.observable_mappings = observables
    @info "The following warnings are expected"
    df1 = load_data(m; try_disk = false)
    df1 = load_data(m)
    df2 = CSV.read(joinpath(dataroot(m), "raw/wrongorigmatlab_190822.csv"))
    df2[!,:date] = Vector{Dates.Date}(df2[!,:date])
    df2[!,:p904] = Vector{Float64}(df2[!,:p904])
    df2[!,:p805] = Vector{Float64}(df2[!,:p805])

    @test df1[!,:Model904] == df2[!,:p904]
    @test df1[!,:Model805] == df2[!,:p805]
end

nothing


##
## --------------------------------------------------
##
# Macros to register POLY{T},  POLY{αs..., T}
#
# We use `Vector{T}` with N =  d+1 to store  a degree d  polynomial - no trailing zeros permitted.
##
macro register0(name, parent)
    poly = esc(name)
    parent_type = esc(parent)
    quote
        struct $poly{T,N} <: $parent_type{T,N}
            coeffs::Vector{T}
            var::Symbol
            function $poly{T,N}(coeffs::Vector{T}, var::Polynomials.SymbolLike=:x) where {T, N}
                M = length(coeffs)
                (M  != N || (N > 0 &&iszero(coeffs[end]))) &&  throw(ArgumentError("wrong  size"))
                new{T,N}(coeffs, Symbol(var))

            end
            function $poly{T,N}(coeffs::NTuple{N,T}, var::Polynomials.SymbolLike=:x) where {T, N}
                M = length(coeffs)
                (M  != N || (N > 0 &&iszero(coeffs[end]))) &&  throw(ArgumentError("wrong  size"))
                new{T,N}(collect(coeffs), Symbol(var))
            end

            function $poly{T}(coeffs::Vector{S},  var::Polynomials.SymbolLike=:x) where {T,S}
                N = findlast(!iszero, coeffs)
                if N ==  nothing
                    new{T,0}(T[], Symbol(var))
                else
                    cs = T.(coeffs[1:N])
                    new{T,N}(cs,  Symbol(var))
                end
            end

            function $poly(coeffs::Vector{S},  var::Polynomials.SymbolLike=:x) where {S}
                $poly{S}(coeffs, var)
            end

        end

        Base.length(p::$poly{T,N}) where {T,N} = N
        (p::$poly)(x::S) where  {S} = eval_cop(typeof(p), p.coeffs, x)

        Polynomials.@register $poly

        # work around N parameter in promote(p,q) usage in defaults
        Base.:*(p::$poly{T}, q::$poly{T}) where {T}  = ⊗(p,q)
        Base.:+(p::$poly{T}, q::$poly{T}) where {T}  = ⊕(p,q)
        Base.divrem(p::$poly{T}, q::$poly{T}) where {T}  = _divrem(p,q)
    end
end

# register macro for polynomial families with parameters
# would be nice to consolidate, but ...
macro registerN(name,  parent, params...)
    poly = esc(name)
    parent_type = esc(parent)
    αs = tuple(esc.(params)...)
    quote
        struct $poly{$(αs...), T,N} <: $parent_type{$(αs...), T,N}
            coeffs::Vector{T}
            var::Symbol
            function $poly{$(αs...), T,N}(coeffs::Vector{T}, var::Polynomials.SymbolLike=:x) where {$(αs...), T, N}
                M = length(coeffs)
                (M  != N || (N > 0 &&iszero(coeffs[end]))) &&  throw(ArgumentError("wrong  size"))
                new{$(αs...), T,N}(coeffs, Symbol(var))
            end

            function $poly{$(αs...),T,N}(coeffs::NTuple{N,T}, var::Polynomials.SymbolLike=:x) where {$(αs...), T, N}
                M = length(coeffs)
                (M  != N || (N > 0 &&iszero(coeffs[end]))) &&  throw(ArgumentError("wrong  size"))
                new{$(αs...),T,N}(collect(coeffs), Symbol(var))
            end

            function $poly{$(αs...),T}(coeffs::Vector{S},  var::Polynomials.SymbolLike=:x) where {$(αs...),T,S}
                N = findlast(!iszero, coeffs)
                if N ==  nothing
                    new{$(αs...),T,0}(T[], Symbol(var))
                else
                    new{$(αs...),T,N}(T.(coeffs[1:N]), Symbol(var))
                end
            end

            function $poly{$(αs...)}(coeffs::Vector{T},  var::Polynomials.SymbolLike=:x) where {$(αs...), T}
                $poly{$(αs...),T}(coeffs, var)
            end

        end

        Base.length(ch::$poly{$(αs...),T,N}) where {$(αs...),T,N} = N
        (ch::$poly)(x::S) where  {S} = eval_cop(typeof(ch),  ch.coeffs, x)

        Base.convert(::Type{P}, q::Q) where {$(αs...),T, P<:$poly{$(αs...),T}, Q <: $poly{$(αs...),T}} = q
        Base.convert(::Type{$poly{$(αs...)}}, q::Q) where {$(αs...),T, Q <: $poly{$(αs...),T}} = q
        Base.promote(p1::P, p2::Q) where {$(αs...),T, P<:$poly{$(αs...),T}, Q <: $poly{$(αs...),T}} = p1,p2
        Base.promote_rule(::Type{<:$poly{$(αs...),T}}, ::Type{<:$poly{$(αs...),S}}) where {$(αs...),T,S} =
            $poly{$(αs...),promote_type(T, S)}
        Base.promote_rule(::Type{<:$poly{$(αs...),T}}, ::Type{S}) where {$(αs...),T,S<:Number} =
            $poly{$(αs...),promote_type(T,S)}

        $poly{$(αs...),T}(n::Number, var::Polynomials.SymbolLike = :x) where {$(αs...),T} =
            n * one($poly{$(αs...),T}, var)
        $poly{$(αs...)}(n::S, var::Polynomials.SymbolLike = :x)        where {$(αs...), S<:Number} =
            $poly{$(αs...),S}(n,var)
        $poly{$(αs...),T}(var::Polynomials.SymbolLike=:x)              where {$(αs...), T} =
            variable($poly{$(αs...),T}, var)
        $poly{$(αs...)}(var::Polynomials.SymbolLike=:x)                where {$(αs...)} =
            variable($poly{$(αs...)}, var)

        # work around N parameter in promote
        Base.:*(p1::$poly{$(αs...),T}, p2::$poly{$(αs...),T}) where {$(αs...),T}  = ⊗(p1,p2)
        Base.:+(p1::$poly{$(αs...),T}, p2::$poly{$(αs...),T}) where {$(αs...),T}  = ⊕(p1,p2)
        Base.divrem(p1::$poly{$(αs...),T}, p2::$poly{$(αs...),T}) where {$(αs...),T}  = _divrem(p1,p2)

    end
end

## Delegation macros
"""
    ϟ(P) = Q; ϟ(P{T}) where {T} =   Q{T}

Used to delegate methods of P to Q.  ([slash]upkoppa[tab])
"""
ϟ(P::Type{<:Polynomials.AbstractPolynomial}) = throw(ArgumentError("No default for  delegation"))

## Make monic version
macro register_monic(name)

    monic=esc(name)

    quote
        SpecialPolynomials.ismonic(::Type{<:$monic}) = true
        SpecialPolynomials.abcde(P::Type{<:$monic})  = abcde(ϟ(P))
        SpecialPolynomials.basis_symbol(P::Type{<:$monic})  = basis_symbol(ϟ(P)) * "̃"
        SpecialPolynomials.weight_function(P::Type{<:$monic}) = weight_function(ϟ(P))
        Polynomials.domain(P::Type{<:$monic}) = domain(ϟ(P))
        SpecialPolynomials.B̃n(P::Type{<:$monic}, n::Int) =  B̃n(ϟ(P),n)
        SpecialPolynomials.B̃n(P::Type{<:$monic}, v::Val{N}) where {N} =  B̃n(ϟ(P),v)
        SpecialPolynomials.C̃n(P::Type{<:$monic}, n::Int) =  C̃n(ϟ(P),n)
        SpecialPolynomials.C̃n(P::Type{<:$monic}, v::Val{N}) where {N} =  C̃n(ϟ(P),v)
        SpecialPolynomials.ẫn(P::Type{<:$monic}, v::Val{N}) where {N} =  ẫn(ϟ(P),v)
        SpecialPolynomials.b̂̃n(P::Type{<:$monic}, v::Val{N}) where {N} =  b̂̃n(ϟ(P),v)
        SpecialPolynomials.ĉ̃n(P::Type{<:$monic}, v::Val{N}) where {N} =  ĉ̃n(ϟ(P),v)
    end
end

## Make ortho
macro register_orthonormal(name)

    orthonormal=esc(name)

    quote
        SpecialPolynomials.isorthonormal(::Type{<:$orthonormal}) = true
        SpecialPolynomials.abcde(P::Type{<:$orthonormal})  = abcde(ϟ(P))
        SpecialPolynomials.basis_symbol(P::Type{<:$orthonormal})  = basis_symbol(ϟ(P)) * "̃"
        SpecialPolynomials.weight_function(P::Type{<:$orthonormal}) = weight_function(ϟ(P))
        Polynomials.domain(P::Type{<:$orthonormal}) = domain(ϟ(P))
        SpecialPolynomials.k0(::Type{P}) where{P <: $orthonormal} = k0(ϟ(P)) / sqrt(norm2(ϟ(P),0))
        SpecialPolynomials.k1k0(::Type{P},  n::Int) where{P <: $orthonormal} = k1k0(ϟ(P),n) /  ω₁₀(ϟ(P), n)
        SpecialPolynomials.ω₁₀(P::Type{<:$orthonormal}, n::Int) = one(eltype(P))

        SpecialPolynomials.B̃n(P::Type{<:$orthonormal}, n::Int) =  B̃n(ϟ(P),n)
        SpecialPolynomials.B̃n(P::Type{<:$orthonormal}, v::Val{N}) where {N} =  B̃n(ϟ(P),v)
        SpecialPolynomials.C̃n(P::Type{<:$orthonormal}, n::Int) =  C̃n(ϟ(P),n)
        SpecialPolynomials.C̃n(P::Type{<:$orthonormal}, v::Val{N}) where {N} =  C̃n(ϟ(P),v)
        SpecialPolynomials.ẫn(P::Type{<:$orthonormal}, v::Val{N}) where {N} =  ẫn(ϟ(P),v)
        SpecialPolynomials.b̂̃n(P::Type{<:$orthonormal}, v::Val{N}) where {N} =  b̂̃n(ϟ(P),v)
        SpecialPolynomials.ĉ̃n(P::Type{<:$orthonormal}, v::Val{N}) where {N} =  ĉ̃n(ϟ(P),v)
    end
end

# P̃ = P(αx+β), so P̃'' = α^2P''(α x+ β); P̃' = αP'(αx+β)
macro register_shifted(name, alpha, beta)
    shifted = esc(name)
    quote
        function SpecialPolynomials.abcde(P::Type{<:$shifted})
            α,β = $alpha, $beta
            oP = one(eltype(P))
            a,b,c,d,e = abcde(ϟ(P))
            ã = oP*a
            b̃ = (oP * 2a*β)/α + (oP*b)/α
            c̃ = (oP * (a*β^2 +b*β + c))/α^2
            d̃ = oP *  d
            ẽ = (oP * (d*β + e))/α
            NamedTuple{(:a,:b,:c,:d,:e)}((ã, b̃, c̃, d̃, ẽ))
        end

        SpecialPolynomials.basis_symbol(P::Type{<:$shifted})  = basis_symbol(ϟ(P)) * "ˢ"
        function  Polynomials.domain(P::Type{<:$shifted})
            α,β = $alpha, $beta
            a, b = extrema(ϟ(P))
            Polynomials.Interval((a- β)/α, (b - β)/α)
        end
        function SpecialPolynomials.k1k0(P::Type{<:$shifted}, n::Int)
            α,β = $alpha, $beta
            α * k1k0(ϟ(P), n)
        end

    end
end

# Weight Functions
macro register_weight_function(W, P, ω)
    W′ = esc(W)
    P′ = esc(P)
    ω′ = esc(ω)

    quote
        SpecialPolynomials.:ϟ(::Type{<:$W′}) =  $P′
        SpecialPolynomials.weight_function(::Type{<:$W′}) =  $ω′
        (ch::$W′)(x)  = clenshaw_eval(ch, x)
    end
end

macro register_discrete_weight_function(W, xs, ws)
    W′ = esc(W)
    xs′ = esc(xs)
    ws′ = esc(ws)

    quote
        SpecialPolynomials.xs_ws(::Type{<:$W′}) = ($xs′, $ws′)
        (p::$W′)(x)  = clenshaw_eval(p, x)
    end
end


function get_plus_inside(obj::BoundaryAttribute)
    return jcall(obj, "getPlusInside", SubHyperplane, ())
end

function get_plus_outside(obj::BoundaryAttribute)
    return jcall(obj, "getPlusOutside", SubHyperplane, ())
end

function get_splitters(obj::BoundaryAttribute)
    return jcall(obj, "getSplitters", NodesSet, ())
end


module TypedTables

using Unicode
using Tables
using SplitApplyCombine

using Base: @propagate_inbounds, @pure, OneTo, Fix2
import Tables.columns, Tables.rows

export @Compute, @Select
export Table, FlexTable, columns, rows, columnnames, showtable

# Resultant element type of given column arrays
@generated function _eltypes(a::NamedTuple{names, T}) where {names, T <: Tuple{Vararg{AbstractArray}}}
    Ts = []
    for V in T.parameters
        push!(Ts, eltype(V))
    end
    return NamedTuple{names, Tuple{Ts...}}
end

_ndims(a::NamedTuple{<:Any, T}) where {T} = _ndims(T)
_ndims(::Type{<:Tuple{Vararg{AbstractArray{<:Any, n}}}}) where {n} = n

# The following code causes newer versions of Julia to hang in precompilation
# and the workaround should not be needed after JuliaLang/julia#30577 was
# merged.
if VERSION < v"1.2.0-DEV.291"
    # Workaround for JuliaLang/julia#29970
    let
        for n in 1:32
            Ts = [Symbol("T$i") for i in 1:n]
            xs = [:(x[$i]) for i in 1:n]
            NT = :(Core.NamedTuple{names, Tuple{$(Ts...)}})
            eval(quote
                $NT(x::Tuple{$(Ts...)}) where {names, $(Ts...)} = $(Expr(:new, NT, xs...))
            end)
        end
    end
end

include("properties.jl")
include("Table.jl")
include("FlexTable.jl")
include("columnops.jl")
include("show.jl")

end # module