#
# This file is a part of MolecularGraph.jl
# Licensed under the MIT License http://opensource.org/licenses/MIT
#

export
    atommatch, bondmatch,
    exactatommatch, exactbondmatch,
    structmatches,
    exactmatches, hasexactmatch,
    substructmatches, hassubstructmatch,
    nodeinducedmatches, hasnodeinducedmatch,
    edgeinducedmatches, hasedgeinducedmatch,
    maxcommonsubstruct,
    connectedmcis, connectedmces,
    disconnectedmcis, disconnectedmces,
    tcmcis, tcmces

"""
    atommatch(mol1::UndirectedGraph, mol2::UndirectedGraph) -> Function

Return a default atom attribute comparator between two atoms.
"""
function atommatch(mol1::UndirectedGraph, mol2::UndirectedGraph)
    sym1 = atomsymbol(mol1)
    sym2 = atomsymbol(mol2)
    pi1 = pielectron(mol1)
    pi2 = pielectron(mol2)
    return (a1, a2) -> sym1[a1] == sym2[a2] && pi1[a1] == pi2[a2]
end

"""
    bondmatch(mol1::UndirectedGraph, mol2::UndirectedGraph) -> Function

Return a default bond attribute comparator between two bonds.
"""
function bondmatch(mol1::UndirectedGraph, mol2::UndirectedGraph)
    # TODO: need bond attribute matching?
    return (b1, b2) -> true
end

function querymatchtree(
        query::QueryFormula, mol::UndirectedGraph, matcher, i; recursive=Dict())
    if query.key === :any
        return true
    elseif query.key === :and
        return all(querymatchtree(q, mol, matcher, i, recursive=recursive) for q in query.value)
    elseif query.key === :or
        return any(querymatchtree(q, mol, matcher, i, recursive=recursive) for q in query.value)
    elseif query.key === :not
        return !querymatchtree(query.value, mol, matcher, i, recursive=recursive)
    elseif query.key === :stereo
        # TODO: stereo not implemented yet
        return true
    elseif query.key === :recursive
        if haskey(recursive, query.value)
            return i in recursive[query.value]
        else
            qm = smartstomol(query.value)
            return subgraph_is_monomorphic(
                mol, qm,
                nodematcher=(a, qa) -> querymatchtree(
                    nodeattr(qm, qa).query, mol, matcher, a),
                edgematcher=(b, qb) -> querymatchtree(
                    edgeattr(qm, qb).query, mol, bmatcher, b),
                mandatory=Dict(i => 1))
        end
    else
        return matcher[query.key][i] == query.value
    end
end

"""
    atommatch(mol::UndirectedGraph, querymol::QueryMol) -> Function

Return a default atom attribute comparator that returns true if the atom satisfies all the queryatom conditions.
"""
function atommatch(mol::UndirectedGraph, qmol::QueryMol)
    matcher = Dict(
        :atomsymbol => atomsymbol(mol),
        :isaromatic => isaromatic(mol),
        :charge => charge(mol),
        :mass => getproperty.(nodeattrs(mol), :mass),
        :stereo => getproperty.(nodeattrs(mol), :stereo),
        :connectivity => connectivity(mol),
        :nodedegree => nodedegree(mol),
        :valence => valence(mol),
        :hydrogenconnected => hydrogenconnected(mol),
        :smallestsssr => smallestsssr(mol),
        :sssrcount => sssrcount(mol)
    )
    bmatcher = Dict(
        :bondorder => bondorder(mol),
        :isringbond => isringbond(mol),
        :isaromaticbond => isaromaticbond(mol),
        :stereo => getproperty.(edgeattrs(mol), :stereo)
    )
    # precalc recursive
    recursive = Dict()
    for q in 1:nodecount(qmol)
        for fml in findallformula(nodeattr(qmol, q).query, :recursive)
            qm = smartstomol(fml)
            recursive[fml] = Set()
            for n in 1:nodecount(mol)
                if subgraph_is_monomorphic(
                        mol, qm,
                        nodematcher=(a, qa) -> querymatchtree(
                            nodeattr(qm, qa).query, mol, matcher, a),
                        edgematcher=(b, qb) -> querymatchtree(
                            edgeattr(qm, qb).query, mol, bmatcher, b),
                        mandatory=Dict(n => 1))
                    push!(recursive[fml], n)
                end
            end
        end
    end
    return (a, qa) -> querymatchtree(
        nodeattr(qmol, qa).query, mol, matcher, a, recursive=recursive)
end

"""
    bondmatch(mol::UndirectedGraph, querymol::QueryMol) -> Function

Return a default bond attribute comparator that returns true if the bond satisfies all the querybond conditions.
"""
function bondmatch(mol::UndirectedGraph, querymol::QueryMol)
    matcher = Dict(
        :bondorder => bondorder(mol),
        :isringbond => isringbond(mol),
        :isaromaticbond => isaromaticbond(mol),
        :stereo => getproperty.(edgeattrs(mol), :stereo)
    )
    return (b, qb) -> querymatchtree(
        edgeattr(querymol, qb).query, mol, matcher, b)
end

"""
    exactatommatch(qmol1::QueryMol, qmol2::QueryMol) -> Function

Return a default atom attribute comparator between two atom queries.
"""
function exactatommatch(qmol1::QueryMol, qmol2::QueryMol)
    return function (a1, a2)
        return nodeattr(qmol1, a1).query == nodeattr(qmol2, a2).query
    end
end

"""
    exactbondmatch(qmol1::QueryMol, qmol2::QueryMol) -> Function

Return a default bond attribute comparator between two bond queries.
"""
function exactbondmatch(qmol1::QueryMol, qmol2::QueryMol)
    return function (b1, b2)
        return edgeattr(qmol1, b1).query == edgeattr(qmol2, b2).query
    end
end

"""
    atommatch(qmol1::QueryMol, qmol2::QueryMol) -> Function

Return an atom attribute comparator that returns true if a2 contains a1.
"""
function atommatch(qmol1::QueryMol, qmol2::QueryMol)
    return function (a1, a2)
        return issubset(nodeattr(qmol1, a1).query, nodeattr(qmol2, a2).query)
    end
end

"""
    bondmatch(qmol1::QueryMol, qmol2::QueryMol) -> Function

Return a bond attribute comparator that returns true if b2 contains b1.
"""
function bondmatch(qmol1::QueryMol, qmol2::QueryMol)
    return function (b1, b2)
        return issubset(edgeattr(qmol1, b1).query, edgeattr(qmol2, b2).query)
    end
end

function fastidentityfilter(mol1::UndirectedGraph, mol2::UndirectedGraph)
    if nodecount(mol1) != nodecount(mol2)
        return false
    elseif edgecount(mol1) != edgecount(mol2)
        return false
    elseif circuitrank(mol1) != circuitrank(mol2)
        return false
    end
    return true
end

function fastsubstrfilter(mol1::UndirectedGraph, mol2::UndirectedGraph)
    if nodecount(mol1) < nodecount(mol2)
        return false
    elseif edgecount(mol1) < edgecount(mol2)
        return false
    elseif circuitrank(mol1) < circuitrank(mol2)
        return false
    end
    return true
end

"""
    structmatches(
        mol::UndirectedGraph, query::UndirectedGraph,
        matchtype::Symbol; kwargs...) -> Iterator

Return a lazy iterator that generate all isomorphism mappings between `mol` and `query`.

`matchtype` should be one of the followings
  - `:exact`: mol is isomorphic to query
  - `:substruct`: a subgraph of mol is monomorphic to query (typical substructure search)
  - `:nodeinduced`: a node-induced subgraph of mol is isomorphic to query
  - `:edgeinduced`: an edge-induced subgraph of mol is isomorphic to query`

# options

- `prefilter::Bool`: if true, apply simple prefilter by graph size and topology to skip vf2 calculation (dafault: true)
- `fastsingleton::Bool`: if true, skip vf2 if the query is single node or edge that can be matched by simple iteration (dafault: false)
- `atommatcher::Function`: a function for semantic atom attribute matching (default: `MolecularGraph.atommatch`)
- `bondmatcher::Function`: a function for semantic bond attribute matching (default: `MolecularGraph.edgematch`)
- `mandatory::Dict{Int,Int}`: mandatory node mapping (or edge mapping if matchtype=:edgeinduced)
- `forbidden::Dict{Int,Int}`: forbidden node mapping (or edge mapping if matchtype=:edgeinduced)
- `timeout::Union{Int,Nothing}`: if specified, abort vf2 calculation when the time reached and return empty iterator (default: 10 seconds).

Note that null mol and null query never match (e.g. isstructmatch(smilestomol(""), smilestomol("")) is false)
"""
function structmatches(
        mol::UndirectedGraph, query::UndirectedGraph, matchtype;
        prefilter=true, fastsingleton=false,
        atommatcher=atommatch, bondmatcher=bondmatch, kwargs...)
    # Null molecule filter
    nodecount(mol) == 0 && return ()
    nodecount(query) == 0 && return ()
    matchtype === :edgeinduced && edgecount(mol) == 0 && return ()
    matchtype === :edgeinduced && edgecount(query) == 0 && return ()

    # Prefilter by graph size and topology
    if prefilter
        if matchtype === :exact
            fastidentityfilter(mol, query) || return ()
        else
            fastsubstrfilter(mol, query) || return ()
        end
    end

    # Matcher functions
    afunc = atommatcher(mol, query)
    bfunc = bondmatcher(mol, query)

    # Skip substruct calculation if the query is a sigle node/edge
    # This slightly speeds up SMARTS queries and hence functional group analysis
    # Note that mandatory and forbidden mapping would be ignored
    if matchtype === :substruct && fastsingleton
        if nodecount(query) == 1
            res = Dict{Int,Int}[]
            q = pop!(nodeset(query))
            for n in nodeset(mol)
                afunc(n, q) && push!(res, Dict(n => q))
            end
            return res
        elseif nodecount(query) == 2 && edgecount(query) == 1
            res = Dict{Int,Int}[]
            q = pop!(edgeset(query))
            (qu, qv) = getedge(query, q)
            for e in edgeset(mol)
                bfunc(e, q) || continue
                (u, v) = getedge(mol, e)
                if afunc(u, qu) && afunc(v, qv)
                    push!(res, Dict(u => qu, v => qv))
                elseif afunc(u, qv) && afunc(v, qu)
                    push!(res, Dict(u => qv, v => qu))
                end
            end
            return res
        end
    end

    # Isomorphism
    if matchtype === :exact
        return isomorphisms(mol, query, nodematcher=afunc, edgematcher=bfunc; kwargs...)
    elseif matchtype === :substruct
        return subgraph_monomorphisms(mol, query, nodematcher=afunc, edgematcher=bfunc; kwargs...)
    elseif matchtype === :nodeinduced
        return nodesubgraph_isomorphisms(mol, query, nodematcher=afunc, edgematcher=bfunc; kwargs...)
    elseif matchtype === :edgeinduced
        return edgesubgraph_isomorphisms(mol, query, nodematcher=afunc, edgematcher=bfunc; kwargs...)
    end
end

"""
    exactmatches(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return a lazy iterator that generate node mappings between `mol` and `query` if these are exactly same.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
exactmatches(mol1, mol2; kwargs...
    ) = structmatches(mol1, mol2, :exact; kwargs...)

"""
    hasexactmatch(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return whether `mol` and `query` have exactly the same structure.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
hasexactmatch(mol1, mol2; kwargs...
    ) = !isempty(exactmatches(mol1, mol2; kwargs...))

"""
    substructmatches(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return a lazy iterator that generate node mappings between `mol` and `query` if `mol` has `query` as a substructure.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
substructmatches(mol1, mol2; kwargs...
    ) = structmatches(mol1, mol2, :substruct; kwargs...)

"""
    hassubstructmatch(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return whether `mol` has `query` as a substructure.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
hassubstructmatch(mol1, mol2; kwargs...
    ) = !isempty(substructmatches(mol1, mol2; kwargs...))

"""
    nodeinducedmatches(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return a lazy iterator that generate node mappings between `mol` and `query` if a node-induced subgraph of `mol` graph is isomorphic to `query` graph.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
nodeinducedmatches(mol1, mol2; kwargs...
    ) = structmatches(mol1, mol2, :nodeinduced; kwargs...)

"""
    hasnodeinducedmatch(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return whether a node-induced subgraph of `mol` graph is isomorphic to `query` graph.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
hasnodeinducedmatch(mol1, mol2; kwargs...
    ) = !isempty(nodeinducedmatches(mol1, mol2; kwargs...))

"""
    edgeinducedmatches(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return a lazy iterator that generate **edge** mappings between `mol` and `query` if a edge-induced subgraph of `mol` graph is isomorphic to `query` graph.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
edgeinducedmatches(mol1, mol2; kwargs...
    ) = structmatches(mol1, mol2, :edgeinduced; kwargs...)

"""
    hasedgeinducedmatch(
        mol::UndirectedGraph, query::UndirectedGraph; kwargs...) -> Iterator

Return whether a edge-induced subgraph of `mol` graph is isomorphic to `query` graph.
See [`MolecularGraph.structmatches`](@ref) for available options.
"""
hasedgeinducedmatch(mol1, mol2; kwargs...
    ) = !isempty(edgeinducedmatches(mol1, mol2; kwargs...))

"""
    maxcommonsubstruct(mol1::UndirectedGraph, mol2::UndirectedGraph, matchtype; kwargs...) -> MaxCommonSubgraphResult

Compute maximum common substructure (MCS) of mol1 and mol2.

## Keyword arguments

- matchtype(Symbol): :nodeinduced or :edgeinduced (default=:edgeinduced).
- connected(Bool): if true, apply connected MCS constraint (default=false).
- topological(Bool): if true, apply topological constraint (default=false).
- diameter(Int): distance cutoff for topological constraint.
- tolerance(Int): distance mismatch tolerance for topological constraint.
- timeout(Int): abort calculation and return suboptimal results if the execution
time has reached the given value (default=60, in seconds).
- targetsize(Int): abort calculation and return suboptimal result so far if the
given mcs size achieved.

# References

1. Kawabata, T. (2011). Build-Up Algorithm for Atomic Correspondence between
Chemical Structures. Journal of Chemical Information and Modeling, 51(8),
1775–1787. https://doi.org/10.1021/ci2001023
1. https://www.jstage.jst.go.jp/article/ciqs/2017/0/2017_P4/_article/-char/en
"""
function maxcommonsubstruct(mol1::UndirectedGraph, mol2::UndirectedGraph, matchtype; kwargs...)
    afunc = atommatch(mol1, mol2)
    bfunc = bondmatch(mol1, mol2)
    return maxcommonsubgraph(mol1, mol2, matchtype, nodematcher=afunc, edgematcher=bfunc; kwargs...)
end

disconnectedmcis(mol1, mol2; kwargs...) = maxcommonsubstruct(mol1, mol2, :nodeinduced; kwargs...)

disconnectedmces(mol1, mol2; kwargs...) = maxcommonsubstruct(mol1, mol2, :edgeinduced; kwargs...)

connectedmcis(mol1, mol2; kwargs...) = maxcommonsubstruct(mol1, mol2, :nodeinduced, connected=true; kwargs...)

connectedmces(mol1, mol2; kwargs...) = maxcommonsubstruct(mol1, mol2, :edgeinduced, connected=true; kwargs...)

tcmcis(mol1, mol2; kwargs...) = maxcommonsubstruct(mol1, mol2, :nodeinduced, topological=true; kwargs...)

tcmces(mol1, mol2; kwargs...) = maxcommonsubstruct(mol1, mol2, :edgeinduced, topological=true; kwargs...)

"""
    nmap = emaptonmap(emap, mol, query)

Convert an edge-based mapping, of the form returned by [`edgesubgraphmatches`](@ref), into
a map between nodes. Commonly, `nmap[i]` is a length-1 vector `[j]`, where `i=>j` is the mapping
from `nodeattr(query, i)` to `nodeattr(mol, j)`. In cases where the mapping is ambiguous,
`nmap[i]` may be multivalued.
"""
function emaptonmap(emap, mol::UndirectedGraph, query::QueryMol)
    nmol, nq = nodecount(mol), nodecount(query)
    nq <= nmol || throw(ArgumentError("query must be a substructure of mol"))
    # Each node in the query edges can map to either of two nodes in mol
    qnodeoptions = [Tuple{Int,Int}[] for _ = 1:nq]
    for (edgeidmol, edgeidq) in emap
        edgemol, edgeq = getedge(mol, edgeidmol), getedge(query, edgeidq)
        for nq in edgeq
            push!(qnodeoptions[nq], edgemol)
        end
    end
    # For nodes connected to two or more other nodes, intersection results in a unique answer
    assignment = [intersect(nodeops...) for nodeops in qnodeoptions]
    # For the singly-connected nodes, assign them by eliminating ones already taken
    taken = falses(nmol)
    for a in assignment
        if length(a) == 1
            taken[a[1]] = true
        end
    end
    for a in assignment
        if length(a) > 1
            deleteat!(a, findall(taken[a]))
        end
    end
    return assignment
end


module SimpleUpdate

using TensorOperations, TensorNetworkTensors, LinearAlgebra, Parameters
export SimpleUpdateProblem, simpleupdate,
       update!, update,
       extract_weight!, extract_weights!,
       energy

include("simpleupdateproblem.jl")

export tfisinghamiltonian, tfisingpropagator
include("ising.jl")
end # module


a1=solveQ([:x,:y,:z],[[:x は :y を 作る],[:z は :y です]],魔法会議)
f1=statements([:x,:y,:z],[[:x は :y を 作る],[:z は :y です]],a1)
u1=unique(f1)


export prunefiles, upload_par, read_resp, yyyyjjj2date, date2yyyyjjj, nancorr
export s3_get_autocorr, XML_download, indexpath, scedcpath, size_check, update_resp_t!
export sync_resp, get_gaps, scedc_stream, mseed_stream, seisio_stream
export fast_mad, fast_mad2d

function prunefiles(filelist::AbstractArray; minfraction = 0.25, maxfraction = 2., minsize=20000)
    if length(filelist) == 0
	    return []
    end

    files = size_check(filelist,minfraction=minfraction,maxfraction=maxfraction, minsize=minsize)

    # get individual stations
    stations = unique([replace(basename(f)[1:7],"_"=>"") for f in files])
    infiles = []
    for jj in eachindex(stations)
	    ind = findall(occursin.(stations[jj],files))
        if length(ind) != 3
	        continue
        end
        push!(infiles,files[ind])
    end
	return infiles
end

function size_check(filelist::AbstractArray; minfraction = 0.25, maxfraction = 2., minsize=20000)
	files = deepcopy(filelist)
    fsizes = [filesize(f) for f in files]

    # check for minimum and maximum sizes relative to median
    ind1 = findall((fsizes .< median(fsizes) * minfraction) .| (fsizes .> median(fsizes) * maxfraction))
    ind2 = findall(fsizes .< minsize)
    deleteat!(files,unique([ind1;ind2]))
    return files
end

function yyyyjjj2date(yearday::String)
    @assert occursin(r"[1-2][0-9][0-9][0-9][0-3][0-9][0-9]",yearday)
    yint = parse(Int,yearday[1:4])
    dint = parse(Int,yearday[5:end])
    @assert dint <= 366 "Input day must be less than or equal to 366"
    return DateTime(yint) + Day(dint-1)
end

function date2yyyyjjj(d::TimeType)
    return "$(year(d))_$(lpad(dayofyear(d),3,"0"))"
end

function upload_par(aws::AWSConfig,output_bucket::String,s3file::String,ec2file::String)
    println("Uploading file $ec2file")
    s3_put(aws,output_bucket,s3file,read(ec2file))
end
upload_par(a...) = upload_par(global_aws_config(region="us-west-2"),a...)

function read_resp(file::String,XMLDIR::String)
    s = yyyyjjj2date(file[end-9:end-3])
	t = s + Day(1)
	s = Dates.format(s, "yyyy-mm-dd HH:MM:SS")
	t = Dates.format(t, "yyyy-mm-dd HH:MM:SS")
	net = basename(file)[1:2]
	sta = split(basename(file),"_")[1][3:end]
	instpath = joinpath(XMLDIR,net * '_' * sta * ".xml" )
    return read_meta("sxml",instpath,s=s,t=t)
end

function XML_download(aws,XMLDIR)
    if !isdir(XMLDIR)
        mkpath(XMLDIR)
    end
    req = collect(s3_list_objects(aws,"scedc-pds","FDSNstationXML/CI/"))
    xmlin = [r["Key"] for r in req]
    xmlout = joinpath.(XMLDIR,basename.(xmlin))
    for ii = 1:length(xmlin)
        s3_get_file(aws,"scedc-pds",xmlin[ii],xmlout[ii])
    end
    return nothing
end
XML_download(XMLDIR) = XML_download(global_aws_config(region="us-west-2"),XMLDIR)

function indexpath(d::Date)
    days = (d - Date(Year(d))).value + 1
    n = ndigits(days)
	jstr = ('0' ^ (3 - n)) * string(days)
	ystr = string(Year(d).value)
    outstring = "continuous_waveforms/index/csv/year="
    outstring *= ystr * "/year_doy="
    outstring *= ystr * '_' * jstr
    outstring *= "/$(ystr)_$(jstr)_waveform_index.csv"
    return outstring
end

"""
  scedcpath(filename)
Convert filename to scedc-pds path.
"""
function scedcpath(filename::String)
    year = filename[14:17]
    day = filename[18:20]
    return "continuous_waveforms/" * year * '/' * year * '_' * day * '/' * filename
end

"""
  update_resp_t!(S)

Add time matrices for instrument response stored in SeisData
"""
function update_resp_t!(S::SeisData)
    for ii = 1:S.n
        S.t[ii] = [1 S.misc[ii]["startDate"];0 S.misc[ii]["endDate"]]
    end
    return nothing
end

"""
  sync_resp(S,s,t)

Prune SeisData such that response are within statrttime `s`
"""
function sync_resp(S::SeisData,s::Int64,t::Int64)
    todelete = []
    for ii = 1:S.n
        if S.t[ii][1,2] > t
            append!(todelete,ii)
        elseif S.t[ii][2,2] < s
            append!(todelete,ii)
        end
    end
    ind = setdiff(1:S.n,todelete)
    return S[ind]
end

function get_gaps(S::SeisData)
    ngaps = zeros(Int,S.n)
    for ii = 1:S.n
        ngaps[ii] = size(S.t[ii],1) - 2
    end
    return ngaps
end

function nancorr(d::DateTime, fs::Real, maxlag::Real)
    C = CorrData()
    C.t = [d2u(d)]
    C.corr = zeros(Float32,convert(Int,2 * fs * maxlag) + 1,1)
    C.corr .= NaN
    return C
end

function s3_get_autocorr(aws::AWSConfig,bucket::String,path::String)
    s = IOBuffer(s3_get(aws,bucket,path))
    seekstart(s)
    return deserialize(s)
end

function mseed_stream(
    aws::AWSConfig,
    bucket::String,
    mseedfiles::AbstractArray,
)
    S = SeisData()
	for ii = 1:length(mseedfiles)
		S += SCEDC.s3_get_seed(
            aws,
			bucket,
			mseedfiles[ii],
			false,
			false,
			false,
			false,
			false,
			false,
			false,
			false,
			false,
			0,
		)
	end
    return S
end
scedc_stream(mseedfiles::AbstractArray) = mseed_stream(global_aws_config(), "scedc-pds",mseedfiles)

function seisio_stream(
    aws::AWSConfig,
    bucket::String,
    files;
    v=0,
)

    A = []
    sbuf  = Array{UInt8, 1}(undef, 65535)
    chans = Int64[]
    ver   = SeisIO.vSeisIO
    L     = zero(Int64)
    if isa(files, String)
        files = [files]
    end

    for path in files
        io  = IOBuffer(s3_get(aws, bucket, path))

        # All SeisIO files begin with "SEISIO"
        if SeisIO.fastread(io, 6) != UInt8[0x53, 0x45, 0x49, 0x53, 0x49, 0x4f]
            @warn string("Skipped ", path, ": not a SeisIO file!")
            close(io)
        end

        ver = SeisIO.fastread(io, Float32)
        L   = SeisIO.fastread(io, Int64)
        C   = read!(io, Array{UInt32,1}(undef, L))
        B   = read!(io, Array{UInt64,1}(undef, L))

        # DND -- faster to avoid seek
        (v > 0) && println("Reading ", L, " objects from ", f)
        (v > 1) && println("C = ", C)
        for n = 1:L
            (v > 1) && println("Reading object with code ", repr(getindex(C, n)), " (", n, "/", L, ")")
            R = SeisIO.read_rec(io, ver, getindex(C, n), getindex(B, n == L ? n : n+1))
            push!(A, R)
            (v > 1) && println("Read ", typeof(getindex(A, n)), " object (", n, "/", L, ")")
        end

        close(io)
    end
    if size(A,1) == 1
        return A[1]
    else
        return SeisData(A...)
    end
end

"""
    fast_mad(A)
Approximate Median Absolute Deviation of array `A`.
MAD = median(|Xi- median(A)|)
"""
function fast_mad(A::AbstractArray,B::Int=1000)
    return median_approx(abs.(A .- median_approx(A,B)),B)
end

function fast_mad2d(A::AbstractArray,B::Int=1000)
    T = eltype(A)
    N = size(A,2)
    mads = zeros(T,1,N)
    for ii in 1:N
        mads[ii] = fast_mad(A[:,ii],B)
    end
    return mads
end

function median_bins(A::AbstractArray, B)
    T = eltype(A)
    N = length(A)
    μ = mean(A)
    σ = std(A)
    minval = μ - σ
    maxval = μ + σ

    leftbin = 1
    bins = zeros(T,B)
    binwidth = 2 * σ / B

    for ii in 1:N
        if A[ii] <= minval
            leftbin += 1
        elseif A[ii] < maxval
            cur = clamp(ceil(Int,(A[ii] - minval) / binwidth),1,B)
            bins[cur] += 1
        end
    end
    return μ, σ, leftbin, bins
end

function median_approx(A::AbstractArray,B)
    μ, σ, leftbin, bins = median_bins(A, B)
    NA = length(A)
    mid = (NA + 1) / 2
    count = leftbin
    lastii = length(bins)
    for (ii, bincount) in enumerate(bins)
        count += bincount
        if count >= mid
            lastii = ii
            break
        end
    end

    binwidth = 2 * σ / B
    med = μ - σ + binwidth * (lastii + 0.5)
    return med
end


abstract type GloVe{LANG} <: EmbeddingSystem{LANG} end

const glove_max_size = 4_000_000

function init(::Type{GloVe})
    vectors = [("glove.6B",
                "Trained on 6B tokens (Wikipedia 2014, Gigaword 5), 400K vocab, uncased. Includes 50d, 100d, 200d, & 300d vectors. 822 MB download.",
                "617afb2fe6cbd085c235baf7a465b96f4112bd7f7ccb2b2cbd649fed9cbcf2fb",
                ["50d", "100d", "200d", "300d"]),
               ("glove.42B.300d",
                "Trained on 42B tokens (Common Crawl), 1.9M vocab, uncased, Includes 300d vectors. 1.75 GB download.",
                "03d5d7fa28e58762ace4b85fb71fe86a345ef0b5ff39f5390c14869da0fc1970",
               []),
               ("glove.840B.300d",
                "Trained on 840B tokens (Common Crawl), 2.2M vocab, cased. Includes 300d vectors. 2.03 GB download.",
                "c06db255e65095393609f19a4cfca20bf3a71e20cc53e892aafa490347e3849f",
                []),
               ("glove.twitter.27B",
                "Trained on 2B tweets, 27B tokens, 1.2M vocab, uncased, 25d, 50d, 100d, & 200d vectors, 1.42 GB download.",
                "792af52f795d1a32c9842a3240f5f3fe5e941a8ff6df5eb0f9d668092ebc019c",
                ["25d", "50d", "100d", "200d"])]
    for (depname, description, sha, dims) in vectors
        register(DataDep(depname,
                         """
                         Pretrained GloVe word embeddings.
                         Website: https://nlp.stanford.edu/projects/glove/
                         Author: Jeffrey Pennington, Richard Socher, Christopher D. Manning
                         Year: 2014
                         Licence: Open Data Commons Public Domain Dedication and License (PDDL)

                         Paper:
                             Jeffrey Pennington, Richard Socher, Christopher D. Manning. GloVe: Global Vectors for Word Representation. In Proceedings of EMNLP, 2014.

                         $description
                         """,
                         "https://nlp.stanford.edu/data/$(depname).zip",
                         sha,
                         post_fetch_method = unpack))
        if length(dims) >= 1
            append!(language_files(GloVe{:en}), ["$(depname)/$(depname).$(dim).txt" for dim in dims])
        else
            push!(language_files(GloVe{:en}), "$depname/$depname.txt")
        end
    end
end

function _load_embeddings(::Type{<:GloVe}, embedding_file, max_vocab_size, keep_words)
    local LL, indexed_words, index
    if length(keep_words) > 0
        max_vocab_size = length(keep_words)
    end
    indexed_words = Vector{String}()
    LL = Vector{Vector{Float32}}()
    open(embedding_file) do f
        index = 1
        for line in eachline(f)
            xs = split(line, ' ')
            word = xs[1]
            if length(keep_words) == 0 || (word in keep_words)
                index > max_vocab_size && break
                push!(indexed_words, word)
                try
                    push!(LL, parse.(Float32, @view(xs[2:end])))
                catch err
                    err isa ArgumentError || rethrow()
                    @warn "Could not parse word vector" index word exception=err
                end
                index += 1
            end
        end
    end
    return reduce(hcat, LL), indexed_words
end


EXAMPLES_DIR = "../examples/"

function run_example(replace_strings, example_name, module_suffix="")
    example_filepath = joinpath(EXAMPLES_DIR, example_name * ".jl")
    txt = read(example_filepath, String)

    for strs in replace_strings
        txt = replace(txt, strs[1] => strs[2])
    end

    test_script_filepath = example_name * "_example_test.jl"

    open(test_script_filepath, "w") do f
        write(f, "module Test_$example_name" * "_$module_suffix\n")
        write(f, txt)
        write(f, "\nend # module")
    end

    try
        include(test_script_filepath)
    catch err
        @error sprint(showerror, err)
        rm(test_script_filepath)
        return false
    end

    rm(test_script_filepath)
    return true
end

@testset "Examples" begin
    println("Testing examples...")

    for arch in archs
        @testset "Wind and convection mixing example [$(typeof(arch))]" begin
            println("  Testing wind and convection-driving mixing example [$(typeof(arch))]")

            replace_strings = [ ("Nz = 48", "Nz = 16"),
                                ("while model.clock.time < end_time", "while model.clock.iteration < 1"),
                                ("time_step!(model, 10, wizard.Δt)", "time_step!(model, 1, wizard.Δt)"),
                              ]

            arch == GPU() && push!(replace_strings, ("architecture = CPU()", "architecture = GPU()"))

            @test_skip run_example(replace_strings, "ocean_wind_mixing_and_convection", string(typeof(arch)))
            rm("ocean_wind_mixing_and_convection.jld2", force=true)
        end
    end

    @testset "Ocean convection with plankton example" begin
        println("  Testing ocean convection with plankton example")

        replace_strings = [ ("Nz = 128", "Nz = 16"),
                           ("end_time = 1day", "end_time=1.0"),
                           ("time_step!(model, 10", "time_step!(model, 1")
                          ]

        @test_skip run_example(replace_strings, "ocean_convection_with_plankton")
    end

    @testset "Simple diffusion example" begin
        println("  Testing simple diffusion example")

        replace_strings = [ ("N = (1, 1, 128)", "N = (1, 1, 16)"),
                            ("Nt = 1000", "Nt = 2")
                          ]

        @test_skip run_example(replace_strings, "simple_diffusion")
    end

    @testset "Internal wave example" begin
        println("  Testing internal wave example")

        replace_strings = [ ("Nx = 128", "Nx = 16"),
                            ("i = 1:10", "i = 1:1"),
                            ("Nt = 200", "Nt = 2"),
                          ]

        @test_skip run_example(replace_strings, "internal_wave")
    end

    @testset "Two-dimensional turbulence example" begin
        println("  Testing two-dimensional turbulence example")

        replace_strings = [ ("N=(128, 128, 1)", "N=(16, 16, 1)"),
                            ("i = 1:10", "i = 1:1"),
                            ("Nt = 100", "Nt = 2")
                          ]

        @test_skip run_example(replace_strings, "two_dimensional_turbulence")
    end

end


"""
$(TYPEDEF)

An abstract type the subtypes of which are associated with functions method
overloaded for for new extensions. An instance of the `DefaultExt <:ExtensionType`
structure to the `Optimizer` in the `ext_type` field.
"""
abstract type ExtensionType end
struct DefaultExt <: ExtensionType end
MOIU.map_indices(::Function, x::ExtensionType) = x
MOIU.map_indices(::Function, x::DefaultExt) = x


############################
# Circuits
############################

type Circuit{N} <: AbstractQuCircuit{N}
    gates::Array{GateUnit,1}
end

Circuit(num::Integer,gates::Array{GateUnit,1}) = Circuit{num}(gates)
Circuit(num::Integer)=Circuit{num}(Array(GateUnit,0))

# TODO
# function show(io::IO,circuit::Circuit)
# end

############################
# Circuit Constructor
############################

max(a::Int) = a

# NOTE
# Should we open AbstractGateUnit as API for users?

function addgate!{N,M}(circuit::AbstractQuCircuit{N},gate::AbstractGateUnit{M})
  push!(circuit.gates,gate)
  sort!(circuit,gates,alg=QuickSort,by=x->x.pos[1])
end

# For gate units
function addgate!{T,N,M}(circuit::AbstractQuCircuit{N},gate::Gate{T,M},pos::Vector{Int})
    # Bounds check
    @assert length(pos)==M+1 "number of qubits do not match"
    @assert maximum(pos[2:end])<=N "bit's id out of range, maximum is $N"

    addgate!(circuit,GateUnit(gate,pos))
end

addgate!{T,N,M}(circuit::AbstractQuCircuit{N},gate::Gate{T,M},pos::Int...) = addgate!(circuit,gate,collect(pos))

# for controled gates
function addgate!{T,N,M}(circuit::AbstractQuCircuit{N},gate::Gate{T,M},pos::Vector{Int},ctrl::Vector{Int})
    # Bounds check
    @assert length(pos)+length(ctrl) == M+1
    @assert max(maximum(pos[2:end]),maximum(ctrl)) <= N "bit's id out of range, maximum is $N"

    addgate!(circuit,CtrlGateUnit(gate,pos,ctrl))
end

function removegate!{N}(circuit::AbstractQuCircuit{N},pos::Vector{Int})
    for i in eachindnex(circuits.gates)
        if circuits.gates.pos == pos
            deleteat!(circuits.gates,i)
        end
    end
end


module ReducedBasisMethods

    using HDF5
    using HDF5: H5DataStore
    using Particles
    using TypedTables

    include("utils.jl")

    include("parameter.jl")

    export Parameter, hassamples

    include("parametersampler.jl")

    export ParameterSampler, CartesianParameterSampler, sample

    include("parameterspace.jl")

    export ParameterSpace

    include("regression.jl")

    export get_regression_αβ

    include("trainingset.jl")

    export TrainingSet

    include("time_marching.jl")

    export IntegratorParameters, IntegratorCache, ReducedIntegratorCache
    export integrate_vp, reduced_integrate_vp

    include("eigen.jl")

    export sorteigen

    include("projection.jl")

    export get_Ψ

    include("deim.jl")

    export deim_get_Π

    include("h5routines.jl")

    export h5save, h5load, read_sampling_parameters

end


"""
    n"Fe"

Implements compile time parsing of strings to produce Element, SubShell, AtomicSubShell,
Transition or CharXRay objects.  The only oddity is that to get SubShell(\"K\") you
must enter n\"K1\" to differentiate the sub-shell from potassium.
Examples:

    n"Fe" == elements[26]
    n"K" == elements[19]
    n"K1" == subshell("K")
    n"L3" == subshell("L3")
    n"Fe L3" == AtomicSubShell(elements[26],subshell("L3"))
    n"L3-M5" == Transition(SubShell("L3"),"SubShell("M5"))
    n"Fe L3-M5" == CharXRay(elements[26],Transition(SubShell("L3"),"SubShell("M5"))
"""
macro n_str(str)
    parsex(str)
end

"""
    enx"Fe K"
    enx"Fe L3-M5"

Implements compile time parsing of strings to produce the energy associated with AtomicSubShell,
or CharXRay objects.  The advantage of enx"Fe K" over `energy(n"Fe K")` is that the evaluation occurs
entirely at compile time with the former while the later still needs to call `energy(...)` at runtime.

Examples:

    enx"Fe L3" == energy(AtomicSubShell(element(26),subshell("L3")))
    enx"Fe L3-M5" == energy(CharXRay(element(26),Transition(SubShell("L3"),"SubShell("M5")))
"""

macro enx_str(str)
    energy(parsex(str))
end

Base.parse(::Type{Element}, str::AbstractString) = element(str)

"""
    element(str::AbstractString)

Parses a string to determine if the string represents an Element by atomic number, symbol or name.
"""
function element(str::AbstractString)::Element
    elm = get(PeriodicTable.elements, str,
            get(PeriodicTable.elements, Symbol(str), missing))
    if ismissing(elm)
        try
            elm = PeriodicTable.elements[parse(Int, str)]
        catch
            error("Unable to parse ", str, " as an Element.")
        end
    end
    return elm
end

"""
    subshell(name::AbstractString)::SubShell

Returns a SubShell structure from a string of the form "K", "L1", ...., "O11"
"""
function subshell(name::AbstractString)
    i = findfirst(ss->repr(ss)==name, allsubshells)
    return !isnothing(i) ? allsubshells[i] : error("Unknown sub-shell $(name)")
end

Base.parse(::Type{SubShell}, name::AbstractString) = subshell(name)

"""
    Base.parse(::Type{SubShell}, name::AbstractString)
    subshell(name::AbstractString)

Parse an SubShell from a string of the form "K", "L1", "L2", ...

    Base.parse(::Type{AtomicSubShell}, str::AbstractString)::AtomicSubShell
    atomicsubshell(str::AbstractString)::AtomicSubShell

Parse an AtomicSubShell from a string of the form \"Fe K\" or \"U M5\".
"""
function Base.parse(::Type{AtomicSubShell}, str::AbstractString)::AtomicSubShell
    sp = split(str, " ")
    if length(sp) == 2
        sh = subshell(sp[2])
        elm = element(sp[1])
        return AtomicSubShell(z(elm), sh)
    end
    error("Cannot parse $str as an AtomicSubShell like \"Fe L3\"")
end

"""
    atomicsubshell(str::AbstractString)::AtomicSubShell

Parse an AtomicSubShell from a string of the form \"Fe K\" or \"U M5\".
"""
atomicsubshell(str::AbstractString) = Base.parse(AtomicSubShell, str)

"""
    characteristic(str::AbstractString)::CharXRay

Create a CharXRay structure from a string like \"Fe K-L3\" or \"U L3-M5\".
"""
function characteristic(str::AbstractString)::CharXRay
    sp1 = split(str, " ")
    if length(sp1) == 2
        elm = element(sp1[1])
        if !ismissing(elm)
            sp2 = split(sp1[2], "-")
            if length(sp2) == 2
                inner = subshell(sp2[1])
                outer = subshell(sp2[2])
                if (!ismissing(inner)) && (!ismissing(outer))
                    return CharXRay(z(elm), transition(inner, outer))
                end
            end
        end
    end
    error("Unable to parse $str as a characteristic X-ray.")
end

Base.parse(::Type{CharXRay}, str::AbstractString)::CharXRay = characteristic(str)

"""
    parsex(str::AbstractString)::Union{Element, SubShell, AtomicSubShell, Transition, CharXRay}

Implements compile time parsing of strings to produce Element, SubShell, AtomicSubShell,
Transition or CharXRay objects. The only oddity is that to get SubShell(\"K\") you
must enter n\"K1\" to differentiate the sub-shell from potassium. `parsex(...)` provides
the implementation for `n"..."`.

Examples:

  * n"Fe" => Element
  * n"L3" => SubShell
  * n"K" => Element
  * n"K1" => SubShell ("K" would be the element potassium)
  * n"Fe K" => AtomicSubShell ("K" not ambiguous here...)
  * n"Fe L3" => AtomicSubShell
  * n"L3-M5" => Transition
  * n"Fe L3-M5" => CharXRay.
"""
function parsex(
    str::AbstractString,
)::Union{Element,SubShell,AtomicSubShell,Transition,CharXRay}
    function _impl_(str::AbstractString)
        sp1 = split(str, " ")
        if length(sp1) == 1  # Could be an Element, an SubShell or a Transition
            if length(split(sp1[1], "-")) == 2 # A transition like "L3-M5"
                try
                    return parse(Transition, str)
                catch
                    return nothing
                end
            else # An Element or a SubShell like "Fe" or "L3"
                try # Try and Element first
                    return parse(Element, str)
                catch
                    # Ignore and try to parse as a sub-shell
                end
                # If it isn't an element, it must be a SubShell
                # Note that since "K" is an element we use "K1" to denote the SubShell
                try
                    return str == "K1" ? subshell("K") : parse(SubShell, str)
                catch
                    return nothing
                end
            end
        else # Could be a AtomicSubShell or a CharXRay
            sp2 = split(sp1[2], "-")
            try
                if length(sp2) == 1  # Like "Fe L3"
                    return parse(AtomicSubShell, str)
                else  # Like "Fe L3-M5"
                    return parse(CharXRay, str)
                end
            catch
                return nothing
            end
        end
    end
    res = _impl_(str)
    isnothing(res) && throw(ArgumentError("Unable to parse \"$str\" as an Element, a SubShell, a Transition, an AtomicSubShell or a CharXRay."))
    return res
end


using Turing, Test
import Random

dir = splitdir(splitdir(pathof(Turing))[1])[1]
include(dir*"/test/test_utils/AllUtils.jl")

@stage_testset "dynamichmc" "dynamichmc.jl" begin
    import DynamicHMC
    Random.seed!(100)
    chn = sample(gdemo_default, DynamicNUTS(), 5000);
    check_numerical(chn, [:s, :m], [49/24, 7/6], atol=0.2)
end


"""

`leftjoin(base, src, join_axis...)`

Left join a `LabeledArray` into another `LabeledArray`.
Note that the left array (base) can be multidimensional. The function creates a dictionary from the right array (`src`).

##### Arguments

* `base` : the left `LabeledArray`.
* `src` : the right `LabeledArray`.
* `join_axes...` can be one of the following forms:
    * integers for the directions in `src` along which to join. In this case, the keys is the `base` side are found by matching the field names in the join directions in `src` with those in `base`.
    * a list of integer=>integer or integer=>vector of arrays, each of the same shape as `base`.

Ultimately, `join_axes...` produces pairs of direction in `src` => vector of arrays, each of the shape of `base`. If the value in key=>value is an integer, the axis along that direction in `base` is taken, after broadcast. The field values are combined into a vector of arrays. If the right hand side is missing (i.e. just an integer), the field names in the axis along the integer direction are used to create an array for `base`.

##### Return

A left joined `LabeledArray`. The join is performed as follows: Given an `i=>arr` form as an element in `join_axes`, the keys in `i`th direction in `src` are used as keys and `arr` are used the keys in the `base` side to left join. The values will be the sliced subarrays for each value in the `join_axes`. Note that `join_axis...` chooses multiple axes for keys.
The output number of dimensions is `ndims(base) + ndims(src) - length(join_axes)`.
Note that when `join_axis` is empty, the result is the tensor product of `base` and `src` from `tensorprod`.

##### Examples

```julia
julia> b = larr(k=[:x :x :y;:z :u :v], axis1=[:x,:y], axis2=darr(r=[:x, :y, :z]))
2 x 3 LabeledArray

r |x |y |z
--+--+--+--
  |k |k |k
--+--+--+--
x |x |x |y
y |z |u |v

julia> s = larr(axis1=darr(k=[:x,:y,:z,:m,:n,:p]), b=[1,2,3,4,5,6])
6 LabeledArray

k |b
--+--
x |1
y |2
z |3
m |4
n |5
p |6

julia> leftjoin(b, s, 1)
2 x 3 LabeledArray

r |x   |y   |z
--+----+----+----
  |k b |k b |k b
--+----+----+----
x |x 1 |x 1 |y 2
y |z 3 |u   |v

julia> leftjoin(b, s, 1=>1)
2 x 3 LabeledArray

r |x   |y   |z
--+----+----+----
  |k b |k b |k b
--+----+----+----
x |x 1 |x 1 |y 1
y |z 2 |u 2 |v 2

julia> leftjoin(b, s, 1=>Any[nalift([:x :z :n;:y :m :p])])
2 x 3 LabeledArray

r |x   |y   |z
--+----+----+----
  |k b |k b |k b
--+----+----+----
x |x 1 |x 3 |y 5
y |z 2 |u 4 |v 6
```

"""
function leftjoin end

leftjoin{N}(base::LabeledArray{TypeVar(:V),N}, src::LabeledArray, join_axes::Int...) = begin
  # needs to find corresponding fields in base, so src axes for joining should be DictArray.
  for axis_index in join_axes
    @assert isa(src.axes[axis_index], DictArray)
  end
  base_key_arrays = AbstractArray{TypeVar(:V),N}[]
  for axis_index in join_axes
    for field in src.axes[axis_index].data.keys
      push!(base_key_arrays, selectfield(base, field))
    end
  end
  perform_leftjoin(base, src, (base_key_arrays...), join_axes)
end

leftjoin(base::LabeledArray, src::LabeledArray) = tensorprod(base, src)

leftjoin(base::DictArray, src::LabeledArray, join_axes::Int...) = leftjoin(LabeledArray(base), src, join_axes...)
leftjoin(base::DictArray, src::LabeledArray) = leftjoin(LabeledArray(base), src)
leftjoin(base::DictArray, src::LabeledArray, join_pairs::Pair{Int}...) = leftjoin(LabeledArray(base), src, join_pairs...)

leftjoin{N}(base::LabeledArray{TypeVar(:V),N}, src::LabeledArray, join_pairs::Pair{Int}...) = begin
  join_axes = Int[join_pair[1] for join_pair in join_pairs]
  axis_labeled_src = LabeledArray(src.data, ntuple(length(src.axes)) do d
    srcaxesd = src.axes[d]
    if in(d, join_axes) && !isa(srcaxesd, DictArray)
      create_dictarray_nocheck(create_ldict_nocheck(:dummy => srcaxesd))
    else
      srcaxesd
    end
  end)
  base_key_arrays = AbstractArray{TypeVar(:V),N}[]
  for (axis_index, axis_values0) in join_pairs
    axis_values = lift_axis_values0_helper1(axis_values0, base, src.axes[axis_index])
    push!(base_key_arrays, axis_values...)
  end
  perform_leftjoin(base, axis_labeled_src, (base_key_arrays...), (join_axes...))
end

lift_axis_values0_helper1(axis_values0::Integer, base, srcaxis) = lift_axis_values0_helper2(BroadcastAxis(base.axes[axis_values0], base, axis_values0), srcaxis)
lift_axis_values0_helper1(axis_values0::DictArray, base, srcaxis) = values(axis_values0)
lift_axis_values0_helper1(axis_values0, base, srcaxis) = axis_values0

lift_axis_values0_helper2(arr::DictArray, srcaxis::DictArray) = begin
  if keys(srcaxis) != keys(arr)
    throw(ArgumentError("field names do not match: $(keys(srcaxis)) vs $(keys(arr))"))
  end
  values(arr)
end
lift_axis_values0_helper2(arr::DictArray, srcaxis) = values(arr)
lift_axis_values0_helper2(arr, srcaxis) = Any[arr]

@generated perform_leftjoin{N,M,L,K}(base::LabeledArray{TypeVar(:T),N},
               src::LabeledArray{TypeVar(:U),M},
               base_key_arrays::NTuple{L,AbstractArray{TypeVar(:V),N}},
               src_key_axis_indices::NTuple{K,Int},
               concat_function::Function=default_concat_array_function) = begin
  rest_dims = M - K
  quote
    @assert $L == sum(i->length(src.axes[i].data), src_key_axis_indices)
    rest_src_axis_indices = filter(i->!(i in src_key_axis_indices), 1:$M) #length(src.axes))
    new_src_indices_ordering = [rest_src_axis_indices...;src_key_axis_indices...]
    permuted_src = permutedims_if_necessary(src, (new_src_indices_ordering...))
    src_axis_index_offset = length(rest_src_axis_indices)
    bymaps = ntuple($K) do i
      axis = permuted_src.axes[i+src_axis_index_offset] #src_key_axis_indices[i]]
      create_join_bymap(axis, Tuple{[eltype(v) for v in axis.data.values]...})
    end
    offset = 1
    bymaps_tuple_positions = ntuple($K) do i
      r = length(src.axes[i+src_axis_index_offset].data)
      offset0 = offset
      offset += r
      offset0
    end
    base_to_src_loc_map = similar(base_key_arrays[1], NTuple{K,Int})
    create_base_to_src_loc_map_all!(base_to_src_loc_map, base_key_arrays, ntuple(d->0,length(base_key_arrays)), bymaps_tuple_positions, bymaps)
    extra_dims = ntuple(d->length(permuted_src.axes[d]), $rest_dims)
    extra_axes = ntuple(d->permuted_src.axes[d], $rest_dims)
    src_mapped_to_base = LabeledArray(similar(src.data, (size(base_to_src_loc_map)...,extra_dims...)), (base.axes...,extra_axes...))
    srctobasedata = src_mapped_to_base.data
    permutedsrcdata = permuted_src.data

    fill_srctobasedata_join_helper!(srctobasedata, base_to_src_loc_map, permutedsrcdata)

    lifted_base = expand_dims(base, (), extra_axes)
    concat_function(lifted_base, src_mapped_to_base)
  end
end

"""

`innerjoin(base, src, join_axis...)`

Inner join an LabeledArray into another LabeledArray. `innerjoin` is different from `leftjoin` in that only elements in the left array that have the corresponding elements in the right array will be kept. Otherwise, the elements will be set to null. If the entire elements along some direction are null, they will be all removed in the output.
Note that the left array (base) can be multidimensional. The function creates a dictionary from the right array (`src`).

##### Arguments

* `base` : the left `LabeledArray`.
* `src` : the right `LabeledArray`.
* `join_axes...` can be one of the following forms:
    * integers for the directions in `src` along which to join.
    * a list of integer=>integer or integer=>vector of arrays, each array of the same shape as `base`.

Ultimately, `join_axes...` produces pairs of direction in `src` => vector of arrays, each of the shape of `base`. If the value in key=>value is an integer, the axis along that direction in `base` is taken, after broadcast. The field values are combined into a vector of arrays. If the right hand side is missing (i.e. just an integer), the field names in the axis along the integer direction are used to create an array for `base`.

##### Return

An inner joined `LabeledArray`. The join is performed as follows: Given an `i=>arr` form as an element in `join_axes`, the keys in `i`th direction in `src` are used as keys and `arr` are used the keys in the `base` side to inner join. The values will be the sliced subarrays for each value in the `join_axes`. Note that `join_axis...` chooses multiple axes for keys.
The output number of dimensions is `ndims(base) + ndims(src) - length(join_axes)`.
Note that when `join_axis` is empty, the result is the tensor product of `base` and `src` from `tensorprod`.

##### Examples

```julia
julia> b = larr(k=[:x :x :y;:z :u :v], axis1=[:x,:u], axis2=darr(r=[:x, :y, :z]))
2 x 3 LabeledArray

r |x |y |z
--+--+--+--
  |k |k |k
--+--+--+--
x |x |x |y
u |z |u |v

julia> s = larr(axis1=darr(k=[:x,:y,:z,:m,:n,:p]), b=[1,2,3,4,5,6])
6 LabeledArray

k |b
--+--
x |1
y |2
z |3
m |4
n |5
p |6

julia> innerjoin(b, s, 1)
2 x 3 LabeledArray

r |x   |y   |z
--+----+----+----
  |k b |k b |k b
--+----+----+----
x |x 1 |x 1 |y 2
u |z 3 |u   |v

julia> innerjoin(b, s, 1=>1)
1 x 3 LabeledArray

r |x   |y   |z
--+----+----+----
  |k b |k b |k b
--+----+----+----
x |x 1 |x 1 |y 1

julia> innerjoin(b, s, 1=>Any[nalift([:o :x :x;:q :r :y])])
2 x 2 LabeledArray

r |y   |z
--+----+----
  |k b |k b
--+----+----
x |x 1 |y 1
u |u   |v 2
```

"""
function innerjoin end

innerjoin{N}(base::LabeledArray{TypeVar(:V),N}, src::LabeledArray, join_axes::Int...) = begin
  # needs to find corresponding fields in base, so src axes for joining should be DictArray.
  for axis_index in join_axes
    @assert isa(src.axes[axis_index], DictArray)
  end
  base_key_arrays = AbstractArray{TypeVar(:V),N}[]
  for axis_index in join_axes
    for field in src.axes[axis_index].data.keys
      push!(base_key_arrays, selectfield(base, field))
    end
  end
  perform_innerjoin(base, src, (base_key_arrays...), join_axes)
end
innerjoin(base::LabeledArray, src::LabeledArray) = tensorprod(base, src)

innerjoin(base::DictArray, src::LabeledArray, join_axes::Int...) = innerjoin(LabeledArray(base), src, join_axes...)
innerjoin(base::DictArray, src::LabeledArray) = innerjoin(LabeledArray(base), src)
innerjoin(base::DictArray, src::LabeledArray, join_pairs::Pair{Int}...) = innerjoin(LabeledArray(base), src, join_pairs...)

innerjoin{N}(base::LabeledArray{TypeVar(:V),N}, src::LabeledArray, join_pairs::Pair{Int}...) = begin
  join_axes = Int[join_pair[1] for join_pair in join_pairs]
  axis_labeled_src = LabeledArray(src.data, ntuple(length(src.axes)) do d
    srcaxesd = src.axes[d]
    if in(d, join_axes) && !isa(srcaxesd, DictArray)
      create_dictarray_nocheck(create_ldict_nocheck(:dummy => srcaxesd))
    else
      srcaxesd
    end
  end)
  base_key_arrays = AbstractArray{TypeVar(:V),N}[]
  for (axis_index, axis_values0) in join_pairs
    axis_values = lift_axis_values0_helper1(axis_values0, base, src.axes[axis_index])
    push!(base_key_arrays, axis_values...)
  end
  perform_innerjoin(base, axis_labeled_src, (base_key_arrays...), (join_axes...))
end

@generated perform_innerjoin{N,M,L,K}(base::LabeledArray{TypeVar(:T),N},
               src::LabeledArray{TypeVar(:U),M},
               base_key_arrays::NTuple{L,AbstractArray{TypeVar(:V),N}},
               src_key_axis_indices::NTuple{K,Int},
               concat_function::Function=default_concat_array_function) = begin
  rest_dims = M - K
  quote
    @assert $L == sum(i->length(src.axes[i].data), src_key_axis_indices)
    rest_src_axis_indices = filter(i->!(i in src_key_axis_indices), 1:$M) #length(src.axes))
    new_src_indices_ordering = [rest_src_axis_indices...;src_key_axis_indices...]
    permuted_src = permutedims(src, (new_src_indices_ordering...))
    src_axis_index_offset = length(rest_src_axis_indices)
    bymaps = ntuple($K) do i
      axis = permuted_src.axes[i+src_axis_index_offset] #src_key_axis_indices[i]]
      create_join_bymap(axis, Tuple{[eltype(v) for v in axis.data.values]...})
    end
    offset = 1
    bymaps_tuple_positions = ntuple($K) do i
      r = length(src.axes[i+src_axis_index_offset].data)
      offset0 = offset
      offset += r
      offset0
    end
    base_to_src_loc_map_all = similar(base_key_arrays[1], NTuple{K,Int})
    create_base_to_src_loc_map_all!(base_to_src_loc_map_all, base_key_arrays, ntuple(d->0,length(base_key_arrays)), bymaps_tuple_positions, bymaps)
    base_axes_to_display = ntuple(d->falses(size(base,d)), $N)
    join_helper_populate_base_axes_to_display!(base_axes_to_display, base_to_src_loc_map_all)
    base_to_src_loc_map = base_to_src_loc_map_all[base_axes_to_display...]
    extra_dims = ntuple(d->length(permuted_src.axes[d]), $rest_dims)
    extra_axes = ntuple(d->permuted_src.axes[d], $rest_dims)
    src_mapped_to_base = LabeledArray(similar(src.data, (size(base_to_src_loc_map)...,extra_dims...)),
                                      ([axis[base_axes_to_display[i]] for (i,axis) in enumerate(base.axes)]...,extra_axes...))
    srctobasedata = src_mapped_to_base.data
    permutedsrcdata = permuted_src.data
    fill_srctobasedata_join_helper!(srctobasedata, base_to_src_loc_map, permutedsrcdata)

    lifted_base = expand_dims(base[base_axes_to_display...], (), extra_axes)
    concat_function(lifted_base, src_mapped_to_base)
  end
end

@generated join_helper_populate_base_axes_to_display!{T,K,N}(base_axes_to_display::T,
                                                             base_to_src_loc_map_all::AbstractArray{NTuple{K,Int},N}) = quote
  @nloops $N i base_to_src_loc_map_all begin
    if @nref($N,base_to_src_loc_map_all,i)[1] > 0
      @nexprs $N j->(base_axes_to_display[j][i_j] = true)
    end
  end
end

@generated fill_srctobasedata_join_helper!{N,K,MKN,M}(srctobasedata::AbstractArray{TypeVar(:KK),MKN},
                                                      base_to_src_loc_map::AbstractArray{NTuple{K,Int},N},
                                                      permutedsrcdata::AbstractArray{TypeVar(:TT),M}) = begin
  rest_dims = M - K
  @assert(MKN == M-K+N)
  quote
    @nloops $N i base_to_src_loc_map begin
      src_coords = @nref($N,base_to_src_loc_map,i)
      if src_coords[1] == 0
        setna!(srctobasedata, @ntuple($N,i)..., @ntuple($rest_dims, d->Colon())...)
      else
        setindex_nocheck!(srctobasedata, @nref($M,permutedsrcdata,d->d<=$rest_dims ? Colon() : src_coords[d-$rest_dims]), @ntuple($N,i)...,@ntuple($rest_dims,d->Colon())...)
      end
    end
  end
end

fill_srctobasedata_join_helper!{N,K,MKN,M}(srctobasedata::DictArray{TypeVar(:KK),MKN},
                                           base_to_src_loc_map::AbstractArray{NTuple{K,Int},N},
                                           permutedsrcdata::DictArray{TypeVar(:TT),M}) = begin
  for i in eachindex(srctobasedata.data.values, permutedsrcdata.data.values)
    fill_srctobasedata_join_helper!(srctobasedata.data.values[i], base_to_src_loc_map, permutedsrcdata.data.values[i])
  end
end

create_join_bymap{T}(axis::DictArray, ::Type{T}) = begin
  result = Dict{T,Int}()
  for i in 1:length(axis)
    result[getindexvalue(axis,i)] = i
  end
  result
end

@generated create_base_to_src_loc_map_all!{N,L,V1,A}(result::AbstractArray{NTuple{1,Int},N},
                                      base_key_arrays::A, #NTuple{L}, #::NTuple{L,AbstractArray{TypeVar(:V),N}},
                                      ::NTuple{L,Int},
                                      bymaps_tuple_positions::NTuple{1,Int},
                                      bymaps::Tuple{Dict{V1,Int}}) = quote
  zerocoords = (0,) # K=1
  bymaps1::Dict{V1,Int} = bymaps[1]
  for i in eachindex(result, base_key_arrays...)
    k::V1 = @ntuple $L d->base_key_arrays[d][i]
    result[i] = if haskey(bymaps1, k)
      (bymaps1[k],)
    else
      zerocoords
    end
  end
end

@generated create_base_to_src_loc_map_all!{K,N,L,A}(result::AbstractArray{NTuple{K,Int},N},
                                    base_key_arrays::A, #::NTuple{L}, #::NTuple{L,AbstractArray{TypeVar(:V),N}},
                                    ::NTuple{L,Int},
                                    bymaps_tuple_positions::NTuple{K,Int},
                                    bymaps) = quote
  zerocoords = ntuple(k->0, $K)
  for i in eachindex(result, base_key_arrays...)
    base_indices = @ntuple $L d->base_key_arrays[d][i]
    na_flag = false
    src_coords = ntuple($K) do i
      k = base_indices[bymaps_tuple_positions[i]:(i==$K ? L:bymaps_tuple_positions[i+1]-1)]
      bymapsi = bymaps[i]
      if haskey(bymapsi, k)
        bymapsi[k]
      else
        na_flag = true
        0
      end
    end
    result[i] = if na_flag
      zerocoords
    else
      src_coords
    end
  end
end

default_concat_array_function{N}(x::LabeledArray{TypeVar(:T),N}, y::LabeledArray{TypeVar(:T),N}) = begin
  tracker = Any[allfieldnames(x)...;allfieldnames(y)...] # need Array, not AbstractArrayWrapper, hence ...
  xdata = if isa(x.data, DictArray)
    x.data
  else
    create_dictarray_nocheck(create_ldict_nocheck(create_additional_fieldname(x, tracker) => x.data))
  end
  ydata = if isa(y.data, DictArray)
    y.data
  else
    create_dictarray_nocheck(create_ldict_nocheck(create_additional_fieldname(y, tracker) => y.data))
  end
  data = DictArray(LDict([xdata.data.keys;ydata.data.keys], [xdata.data.values;ydata.data.values]))
  axes = x.axes
  LabeledArray(data, axes)
end


using NCDatasets
using Dates
using TOML
using CFTime
using Random
using UnPack

tomlpath = joinpath(@__DIR__, "sbm_config.toml")
parsed_toml = TOML.parsefile(tomlpath)
config = Wflow.Config(tomlpath)

@testset "configuration file" begin
    @test parsed_toml isa Dict{String,Any}
    @test config isa Wflow.Config
    @test Dict(config) == parsed_toml
    @test pathof(config) == tomlpath
    @test dirname(config) == dirname(tomlpath)

    # test if the values are parsed as expected
    @test config.casename == "testcase"
    @test config.starttime === DateTime(2000)
    @test config.endtime === DateTime(2000, 2)
    @test config.output.path == "data/output_moselle.nc"
    @test config.output isa Wflow.Config
    @test collect(keys(config.output)) == ["lateral", "vertical", "path"]
end

@testset "checkdims" begin
    @test_throws AssertionError Wflow.checkdims(("z", "lat", "time"))
    @test_throws AssertionError Wflow.checkdims(("z", "lat"))
    @test Wflow.checkdims(("lon", "lat", "time")) == ("lon", "lat", "time")
    @test Wflow.checkdims(("time", "lon", "lat")) == ("time", "lon", "lat")
    @test Wflow.checkdims(("lat", "lon", "time")) == ("lat", "lon", "time")
    @test Wflow.checkdims(("time", "lat", "lon")) == ("time", "lat", "lon")
end

@testset "timecycles" begin
    @test Wflow.timecycles([Date(2020, 4, 21), Date(2020, 10, 21)]) == [(4, 21), (10, 21)]
    @test_throws ErrorException Wflow.timecycles([Date(2020, 4, 21), Date(2021, 10, 21)])
    @test_throws ErrorException Wflow.timecycles(collect(1:400))
    @test Wflow.timecycles(collect(1:12)) == collect(zip(1:12, fill(1, 12)))
    @test Wflow.timecycles(collect(1:366)) ==
          monthday.(Date(2000, 1, 1):Day(1):Date(2000, 12, 31))

    @test Wflow.monthday_passed((1,1), (1,1))  # same day
    @test Wflow.monthday_passed((1,2), (1,1))  # day later
    @test Wflow.monthday_passed((2,1), (1,1))  # month later
    @test Wflow.monthday_passed((2,2), (1,1))  # month and day later
    @test !Wflow.monthday_passed((2,1), (2,2))  # day before
    @test !Wflow.monthday_passed((1,2), (2,2))  # month before
    @test !Wflow.monthday_passed((1,1), (2,2))  # day and month before
end

# test reading and setting of warm states (reinit=false)
# modify existing config and initialize model with warm states
@test config.model.reinit
config["model"]["reinit"] = false
@test !config.model.reinit
model = Wflow.initialize_sbm_model(config)

@unpack vertical, clock, reader, writer = model

@testset "output and state names" begin
    ncdims = ("lon", "lat", "layer", "time")
    @test dimnames(writer.dataset["ustorelayerdepth"]) == ncdims
    ncvars = [k for k in keys(writer.dataset) if !in(k, ncdims)]
    @test "snow" in ncvars
    @test "q_river" in ncvars
    @test "q_land" in ncvars
    @test length(writer.state_parameters) == 14
end

@testset "warm states" begin
    @test Wflow.param(model, "lateral.river.reservoir.volume")[2] ≈ 2.7393418e7
    @test Wflow.param(model, "vertical.satwaterdepth")[41120] ≈ 201.51429748535156
    @test Wflow.param(model, "vertical.snow")[41120] ≈ 4.21874475479126
    @test Wflow.param(model, "vertical.tsoil")[41120] ≈ -1.9285825490951538
    @test Wflow.param(model, "vertical.ustorelayerdepth")[1][1] ≈ 16.73013687133789
    @test Wflow.param(model, "vertical.snowwater")[41120] ≈ 0.42188167572021484
    @test Wflow.param(model, "vertical.canopystorage")[1] ≈ 0.0
    @test Wflow.param(model, "lateral.subsurface.ssf")[39308] ≈ 1.1614302208e11
    @test Wflow.param(model, "lateral.river.q")[5501] ≈ 111.46229553222656
    @test Wflow.param(model, "lateral.river.h")[5501] ≈ 8.555977821350098
    @test Wflow.param(model, "lateral.land.q")[39626] ≈ 1.0575231313705444
    @test Wflow.param(model, "lateral.land.h")[39626] ≈ 0.03456519544124603
end

@testset "reducer" begin
    V = [6, 5, 4, 1]
    @test Wflow.reducerfunction("maximum")(V) == 6
    @test Wflow.reducerfunction("mean")(V) == 4
    @test Wflow.reducerfunction("median")(V) == 4.5
    @test Wflow.reducerfunction("first")(V) == 6
    @test Wflow.reducerfunction("last")(V) == 1
    @test_throws ErrorException Wflow.reducerfunction("other")
end

@testset "network" begin
    @unpack network = model
    @unpack indices, reverse_indices = model.network.land
    # test if the reverse index reverses the index
    linear_index = 100
    cartesian_index = indices[linear_index]
    @test cartesian_index === CartesianIndex(115, 6)
    @test reverse_indices[cartesian_index] === linear_index
end

Wflow.close_files(model, delete_output = false)

@testset "NetCDF creation" begin
    path = Base.Filesystem.tempname()
    _ = Wflow.create_tracked_netcdf(path)
    # safe to open the same path twice
    ds = Wflow.create_tracked_netcdf(path)
    close(ds)  # path is removed on process exit
end


"""
    struct CSF

Represents a single _configuration state function_ (CSF) adapted to angular momentum
symmetries. Depending on the type parameters, it can represent both non-relativistic CSFs in
[``LS``-coupling](@ref) and relativistic CSFs in [``jj``-coupling](@ref).

A CSF is defined by the following information:

* The configuration, i.e. an ordered list of subshells together with their occupations.
* A list of intermediate coupling terms (including the seniority quantum number to label
  states in degenerate subspaces) for each subshell in the configuration.
* A list of coupling terms for the coupling tree. The coupling is assumed to be done by,
  first, coupling two orbitals together, then coupling that to the next orbital, and so on.

An instance of a [`CSF`](@ref) object does not specify the ``J_z`` or ``L_z``/``S_z``
quantum number(s).

# Constructors

    CSF(configuration::Configuration, subshell_terms::Vector, terms::Vector)

Constructs an instance of a [`CSF`](@ref) from the information provided. The arguments have
different requirements depending on whether it is `configuration` is based on relativistic
or non-relativistic orbitals.

* If the configuration is based on [`Orbital`](@ref)s, `subshell_terms` must be a list of
  [`IntermediateTerm`](@ref)s and `terms` a list of [`Term`](@ref)s.
* If it is a configuration of [`RelativisticOrbital`](@ref)s, both `subshell_terms` and
  `terms` should both be a list of half-integer values.
"""
struct CSF{O<:AbstractOrbital, T<:Union{Term,HalfInteger}, S}
    config::Configuration{<:O}
    subshell_terms::Vector{IntermediateTerm{T,S}}
    terms::Vector{T}

    function CSF(config::Configuration{O},
                 subshell_terms::Vector{IntermediateTerm{T,S}},
                 terms::Vector{T}) where {O<:Union{<:Orbital,<:RelativisticOrbital},
                                          T<:Union{Term,HalfInt}, S}
        length(subshell_terms) == length(peel(config)) ||
            throw(ArgumentError("Need to provide $(length(peel(config))) subshell terms for $(config)"))
        length(terms) == length(peel(config)) ||
            throw(ArgumentError("Need to provide $(length(peel(config))) terms for $(config)"))

        for (i,(orb,occ,_)) in enumerate(peel(config))
            st = subshell_terms[i]
            assert_unique_classification(orb, occ, st) ||
                throw(ArgumentError("$(st) is not a unique classification of $(orb)$(to_superscript(occ))"))
        end

        new{O,T,S}(config, subshell_terms, terms)
    end

    CSF(config, subshell_terms::Vector{<:IntermediateTerm}, terms::Vector{<:Real}) =
        CSF(config, subshell_terms, convert.(HalfInt, terms))
end

"""
    const NonRelativisticCSF = CSF{<:Orbital,Term}
"""
const NonRelativisticCSF = CSF{<:Orbital,Term}

"""
    const RelativisticCSF = CSF{<:RelativisticOrbital,HalfInt}
"""
const RelativisticCSF = CSF{<:RelativisticOrbital,HalfInt}

Base.:(==)(a::CSF{O,T}, b::CSF{O,T}) where {O,T} =
    (a.config == b.config) && (a.subshell_terms == b.subshell_terms) && (a.terms == b.terms)

# We possibly want to sort on configuration as well
Base.isless(a::CSF, b::CSF) = last(a.terms) < last(b.terms)

num_electrons(csf::CSF) = num_electrons(csf.config)

"""
    orbitals(csf::CSF{O}) -> Vector

Access the underlying list of orbitals

```jldoctest
julia> csf = first(csfs(rc"1s2 2p-2"))
1s²   2p-²
     0     0
      0     0+

julia> orbitals(csf)
2-element Vector{RelativisticOrbital{Int64}}:
 1s
 2p-
```
"""
orbitals(csf::CSF) = orbitals(csf.config)

"""
    csfs(::Configuration) -> Vector{CSF}
    csfs(::Vector{Configuration}) -> Vector{CSF}

Generate all [`CSF`](@ref)s corresponding to a particular configuration or a set of
configurations.
"""
function csfs end

function csfs(config::Configuration)
    map(allchoices(intermediate_terms(peel(config)))) do subshell_terms
        map(intermediate_couplings(subshell_terms)) do coupled_terms
            CSF(config, subshell_terms, coupled_terms[2:end])
        end
    end |> c -> vcat(c...) |> sort
end

csfs(configs::Vector{Configuration{O}}) where O = vcat(map(csfs, configs)...)

Base.length(csf::CSF) = length(peel(csf.config))
Base.getindex(csf::CSF, i::Integer) =
    (peel(csf.config)[i],csf.subshell_terms[i],csf.terms[i])

Base.iterate(csf::CSF, (el, i)=(length(csf)>0 ? csf[1] : nothing,1)) =
    i > length(csf) ? nothing : (el, (csf[i==length(csf) ? i : i+1],i+1))

function Base.show(io::IO, csf::CSF)
    c = core(csf.config)
    p = peel(csf.config)
    nc = length(c)
    if nc > 0
        show(io, c)
        write(io, " ")
    end

    for (i,((orb,occ,state),st,t)) in enumerate(csf)
        show(io, orb)
        occ > 1 && write(io, to_superscript(occ))
        write(io, "($(st)|$(t))")
        i != lastindex(p) && write(io, " ")
    end
    print(io, iseven(parity(csf.config)) ? "+" : "-")
end

function Base.show(io::IO, ::MIME"text/plain", csf::RelativisticCSF)
    c = core(csf.config)
    nc = length(c)
    cw = length(string(c))

    w = 0
    p = peel(csf.config)
    for (orb,occ,state) in p
        w = max(w, length(string(orb))+length(digits(occ)))
    end

    w += 1

    cfmt = FormatExpr("{1:$(cw)s} ")
    ofmt = FormatExpr("{1:<$(w+1)s}")
    ifmt = FormatExpr("{1:>$(w+1)d}")
    rfmt = FormatExpr("{1:>$(w-1)d}/2")

    nc > 0 && printfmt(io, cfmt, c)
    for (orb,occ,state) in p
        printfmt(io, ofmt, join([string(orb), occ > 1 ? to_superscript(occ) : ""], ""))
    end
    println(io)

    nc > 0 && printfmt(io, cfmt, "")
    for it in csf.subshell_terms
        j = it.term
        if denominator(j) == 1
            printfmt(io, ifmt, numerator(j))
        else
            printfmt(io, rfmt, numerator(j))
        end
    end
    println(io)

    nc > 0 && printfmt(io, cfmt, "")
    print(io, " ")
    for j in csf.terms
        if denominator(j) == 1
            printfmt(io, ifmt, numerator(j))
        else
            printfmt(io, rfmt, numerator(j))
        end
    end
    print(io, iseven(parity(csf.config)) ? "+" : "-")
end

export CSF, NonRelativisticCSF, RelativisticCSF, csfs


# This file is a part of Julia. License is MIT: https://julialang.org/license

module TypesTest

import Dates
using Test
using ExtendedDates

# Create "test" check manually
y = Year(1)
s = SemesterDate
q = Quarter(1)
m = Month(1)
w = Week(1)
d = Day(1)
h = Hour(1)
mi = Minute(1)
s = Second(1)
ms = Millisecond(1)
@testset "SimpleDate construction by parts" begin
    test = YearDate(ExtendedDates.UTY(2012))
    @test YearDate(2013) == test

    test = DayDate(ExtendedDates.UTD(Dates.value(Dates.Date(2013))))
    @test DayDate(2013) == test
    @test DayDate(2013, 1) == test
    @test DayDate(2013, 1, 1) == test

    @test DayDate(y) == DayDate(1)
    @test DayDate(y, m) == DayDate(1, 1)
    @test DayDate(y, m, d) == DayDate(1, 1, 1)
    @test DayDate(Day(10), Month(2), y) == DayDate(1, 2, 10)

    test = WeekDate(ExtendedDates.UTW(ExtendedDates.WEEKTABLE[2012]["cum_weeks"] + 1))
    @test WeekDate(2013) == test
    @test WeekDate(2013, 1) == test

    @test WeekDate(y) == WeekDate(1)
    @test WeekDate(y, w) == WeekDate(1, 1)
    @test WeekDate(Week(10), y) == WeekDate(1, 10)

    test = MonthDate(ExtendedDates.UTM(2012 * 12 + 1))
    @test MonthDate(2013) == test
    @test MonthDate(2013, 1) == test

    @test MonthDate(y) == MonthDate(1)
    @test MonthDate(y, m) == MonthDate(1, 1)
    @test MonthDate(Month(10), y) == MonthDate(1, 10)

    test = QuarterDate(ExtendedDates.UTQ(2012 * 4 + 1))
    @test QuarterDate(2013) == test
    @test QuarterDate(2013, 1) == test

    @test QuarterDate(y) == QuarterDate(1)
    @test QuarterDate(y, q) == QuarterDate(1, 1)
    @test QuarterDate(Quarter(3), y) == QuarterDate(1, 3)

    test = SemesterDate(ExtendedDates.UTS(2012 * 2 + 1))
    @test SemesterDate(2013) == test
    @test SemesterDate(2013, 1) == test

    @test SemesterDate(y) == SemesterDate(1)
    @test SemesterDate(y, s) == SemesterDate(1, 1)
    @test SemesterDate(Semester(2), y) == SemesterDate(1, 2)
end

@testset "various input types for Date/DateTime" begin
    test = DayDate(1, 1, 1)
    @test DayDate(Int8(1), Int8(1), Int8(1)) == test
    @test DayDate(UInt8(1), UInt8(1), UInt8(1)) == test
    @test DayDate(Int16(1), Int16(1), Int16(1)) == test
    @test DayDate(UInt8(1), UInt8(1), UInt8(1)) == test
    @test DayDate(Int32(1), Int32(1), Int32(1)) == test
    @test DayDate(UInt32(1), UInt32(1), UInt32(1)) == test
    @test DayDate(Int64(1), Int64(1), Int64(1)) == test
    @test DayDate('\x01', '\x01', '\x01') == test
    @test DayDate(true, true, true) == test
    @test_throws ArgumentError DayDate(false, true, false)
    @test DayDate(false, true, true) == test - Day(366)
    @test_throws ArgumentError DayDate(true, true, false)
    @test DayDate(UInt64(1), UInt64(1), UInt64(1)) == test
    @test DayDate(0, UInt64(12), UInt64(30)) == test - Day(2)
    @test DayDate(Int128(1), Int128(1), Int128(1)) == test
    @test_throws InexactError DayDate(
        170141183460469231731687303715884105727,
        Int128(1),
        Int128(1),
    )
    @test DayDate(UInt128(1), UInt128(1), UInt128(1)) == test
    @test DayDate(big(1), big(1), big(1)) == test
    @test DayDate(big(1), big(1), big(1)) == test
    # Potentially won't work if can't losslessly convert to Int64
    @test DayDate(BigFloat(1), BigFloat(1), BigFloat(1)) == test
    @test DayDate(complex(1), complex(1), complex(1)) == test
    @test DayDate(Float64(1), Float64(1), Float64(1)) == test
    @test DayDate(Float32(1), Float32(1), Float32(1)) == test
    @test DayDate(Float16(1), Float16(1), Float16(1)) == test
    @test DayDate(Rational(1), Rational(1), Rational(1)) == test
    @test_throws InexactError DayDate(BigFloat(1.2), BigFloat(1), BigFloat(1))
    @test_throws InexactError DayDate(1 + im, complex(1), complex(1))
    @test_throws InexactError DayDate(1.2, 1.0, 1.0)
    @test_throws InexactError DayDate(1.2f0, 1.0f0, 1.0f0)
    @test_throws InexactError DayDate(3 // 4, Rational(1), Rational(1)) == test

    test = WeekDate(1, 1)
    @test WeekDate(Int8(1), Int8(1)) == test
    @test WeekDate(UInt8(1), UInt8(1)) == test
    @test WeekDate(Int16(1), Int16(1)) == test
    @test WeekDate(UInt8(1), UInt8(1)) == test
    @test WeekDate(Int32(1), Int32(1)) == test
    @test WeekDate(UInt32(1), UInt32(1)) == test
    @test WeekDate(Int64(1), Int64(1)) == test
    @test WeekDate('\x01', '\x01') == test
    @test WeekDate(true, true) == test
    @test_throws ArgumentError WeekDate(0)
    @test_throws ArgumentError WeekDate(false, false)
    @test_throws ArgumentError WeekDate(true, false)
    @test WeekDate(UInt64(1), UInt64(1)) == test
    @test WeekDate(Int128(1), Int128(1)) == test
    @test_throws InexactError WeekDate(170141183460469231731687303715884105727, Int128(1))
    @test WeekDate(UInt128(1), UInt128(1)) == test
    @test WeekDate(big(1), big(1)) == test
    @test WeekDate(big(1), big(1)) == test
    # Potentially won't work if can't losslessly convert to Int64
    @test WeekDate(BigFloat(1), BigFloat(1)) == test
    @test WeekDate(complex(1), complex(1)) == test
    @test WeekDate(Float64(1), Float64(1)) == test
    @test WeekDate(Float32(1), Float32(1)) == test
    @test WeekDate(Float16(1), Float16(1)) == test
    @test WeekDate(Rational(1), Rational(1)) == test
    @test_throws InexactError WeekDate(BigFloat(1.2), BigFloat(1))
    @test_throws InexactError WeekDate(1 + im, complex(1))
    @test_throws InexactError WeekDate(1.2, 1.0)
    @test_throws InexactError WeekDate(1.2f0, 1.0f0)
    @test_throws InexactError WeekDate(3 // 4, Rational(1)) == test

    test = MonthDate(1, 1)
    @test MonthDate(Int8(1), Int8(1)) == test
    @test MonthDate(UInt8(1), UInt8(1)) == test
    @test MonthDate(Int16(1), Int16(1)) == test
    @test MonthDate(UInt8(1), UInt8(1)) == test
    @test MonthDate(Int32(1), Int32(1)) == test
    @test MonthDate(UInt32(1), UInt32(1)) == test
    @test MonthDate(Int64(1), Int64(1)) == test
    @test MonthDate('\x01', '\x01') == test
    @test MonthDate(true, true) == test
    @test_throws ArgumentError MonthDate(false, false)
    @test MonthDate(false, true) == test - Dates.Month(12)
    @test_throws ArgumentError MonthDate(true, false)
    @test MonthDate(UInt64(1), UInt64(1)) == test
    @test MonthDate(0, UInt64(12)) == test - Dates.Month(1)
    @test MonthDate(Int128(1), Int128(1)) == test
    @test_throws InexactError MonthDate(170141183460469231731687303715884105727, Int128(1))
    @test MonthDate(UInt128(1), UInt128(1)) == test
    @test MonthDate(big(1), big(1)) == test
    @test MonthDate(big(1), big(1)) == test
    # Potentially won't work if can't losslessly convert to Int64
    @test MonthDate(BigFloat(1), BigFloat(1)) == test
    @test MonthDate(complex(1), complex(1)) == test
    @test MonthDate(Float64(1), Float64(1)) == test
    @test MonthDate(Float32(1), Float32(1)) == test
    @test MonthDate(Float16(1), Float16(1)) == test
    @test MonthDate(Rational(1), Rational(1)) == test
    @test_throws InexactError MonthDate(BigFloat(1.2), BigFloat(1))
    @test_throws InexactError MonthDate(1 + im, complex(1))
    @test_throws InexactError MonthDate(1.2, 1.0)
    @test_throws InexactError MonthDate(1.2f0, 1.0f0)
    @test_throws InexactError MonthDate(3 // 4, Rational(1)) == test

    test = QuarterDate(1, 1)
    @test QuarterDate(Int8(1), Int8(1)) == test
    @test QuarterDate(UInt8(1), UInt8(1)) == test
    @test QuarterDate(Int16(1), Int16(1)) == test
    @test QuarterDate(UInt8(1), UInt8(1)) == test
    @test QuarterDate(Int32(1), Int32(1)) == test
    @test QuarterDate(UInt32(1), UInt32(1)) == test
    @test QuarterDate(Int64(1), Int64(1)) == test
    @test QuarterDate('\x01', '\x01') == test
    @test QuarterDate(true, true) == test
    @test_throws ArgumentError QuarterDate(false, false)
    @test QuarterDate(false, true) == test - Dates.Quarter(4)
    @test_throws ArgumentError QuarterDate(true, false)
    @test QuarterDate(UInt64(1), UInt64(1)) == test
    @test QuarterDate(0, UInt64(4)) == test - Dates.Quarter(1)
    @test QuarterDate(Int128(1), Int128(1)) == test
    @test_throws InexactError QuarterDate(
        170141183460469231731687303715884105727,
        Int128(1),
    )
    @test QuarterDate(UInt128(1), UInt128(1)) == test
    @test QuarterDate(big(1), big(1)) == test
    @test QuarterDate(big(1), big(1)) == test
    # Potentially won't work if can't losslessly convert to Int64
    @test QuarterDate(BigFloat(1), BigFloat(1)) == test
    @test QuarterDate(complex(1), complex(1)) == test
    @test QuarterDate(Float64(1), Float64(1)) == test
    @test QuarterDate(Float32(1), Float32(1)) == test
    @test QuarterDate(Float16(1), Float16(1)) == test
    @test QuarterDate(Rational(1), Rational(1)) == test
    @test_throws InexactError QuarterDate(BigFloat(1.2), BigFloat(1))
    @test_throws InexactError QuarterDate(1 + im, complex(1))
    @test_throws InexactError QuarterDate(1.2, 1.0)
    @test_throws InexactError QuarterDate(1.2f0, 1.0f0)
    @test_throws InexactError QuarterDate(3 // 4, Rational(1)) == test

    test = SemesterDate(1, 1)
    @test SemesterDate(Int8(1), Int8(1)) == test
    @test SemesterDate(UInt8(1), UInt8(1)) == test
    @test SemesterDate(Int16(1), Int16(1)) == test
    @test SemesterDate(UInt8(1), UInt8(1)) == test
    @test SemesterDate(Int32(1), Int32(1)) == test
    @test SemesterDate(UInt32(1), UInt32(1)) == test
    @test SemesterDate(Int64(1), Int64(1)) == test
    @test SemesterDate('\x01', '\x01') == test
    @test SemesterDate(true, true) == test
    @test_throws ArgumentError SemesterDate(false, false)
    @test SemesterDate(false, true) == test - Semester(2)
    @test_throws ArgumentError SemesterDate(true, false)
    @test SemesterDate(UInt64(1), UInt64(1)) == test
    @test SemesterDate(0, UInt64(2)) == test - Semester(1)
    @test SemesterDate(Int128(1), Int128(1)) == test
    @test_throws InexactError SemesterDate(
        170141183460469231731687303715884105727,
        Int128(1),
    )
    @test SemesterDate(UInt128(1), UInt128(1)) == test
    @test SemesterDate(big(1), big(1)) == test
    @test SemesterDate(big(1), big(1)) == test
    # Potentially won't work if can't losslessly convert to Int64
    @test SemesterDate(BigFloat(1), BigFloat(1)) == test
    @test SemesterDate(complex(1), complex(1)) == test
    @test SemesterDate(Float64(1), Float64(1)) == test
    @test SemesterDate(Float32(1), Float32(1)) == test
    @test SemesterDate(Float16(1), Float16(1)) == test
    @test SemesterDate(Rational(1), Rational(1)) == test
    @test_throws InexactError SemesterDate(BigFloat(1.2), BigFloat(1))
    @test_throws InexactError SemesterDate(1 + im, complex(1))
    @test_throws InexactError SemesterDate(1.2, 1.0)
    @test_throws InexactError SemesterDate(1.2f0, 1.0f0)
    @test_throws InexactError SemesterDate(3 // 4, Rational(1)) == test

    test = YearDate(1)
    @test YearDate(Int8(1)) == test
    @test YearDate(UInt8(1)) == test
    @test YearDate(Int16(1)) == test
    @test YearDate(UInt8(1)) == test
    @test YearDate(Int32(1)) == test
    @test YearDate(UInt32(1)) == test
    @test YearDate(Int64(1)) == test
    @test YearDate('\x01') == test
    @test YearDate(true) == test
    @test YearDate(false) == test - Year(1)
    @test YearDate(UInt64(1)) == test
    @test YearDate(0) == test - Year(1)
    @test YearDate(Int128(1)) == test
    @test_throws InexactError YearDate(170141183460469231731687303715884105727)
    @test YearDate(UInt128(1)) == test
    @test YearDate(big(1)) == test
    @test YearDate(big(1)) == test
    # Potentially won't work if can't losslessly convert to Int64
    @test YearDate(BigFloat(1)) == test
    @test YearDate(complex(1)) == test
    @test YearDate(Float64(1)) == test
    @test YearDate(Float32(1)) == test
    @test YearDate(Float16(1)) == test
    @test YearDate(Rational(1)) == test
    @test_throws InexactError YearDate(BigFloat(1.2))
    @test_throws InexactError YearDate(1 + im)
    @test_throws InexactError YearDate(1.2)
    @test_throws InexactError YearDate(1.2f0)
    @test_throws InexactError YearDate(3 // 4) == test

    test = UndatedDate(1)
    @test UndatedDate(Int8(1)) == test
    @test UndatedDate(UInt8(1)) == test
    @test UndatedDate(Int16(1)) == test
    @test UndatedDate(UInt8(1)) == test
    @test UndatedDate(Int32(1)) == test
    @test UndatedDate(UInt32(1)) == test
    @test UndatedDate(Int64(1)) == test
    @test UndatedDate('\x01') == test
    @test UndatedDate(true) == test
    @test UndatedDate(false) == test - Undated(1)
    @test UndatedDate(UInt64(1)) == test
    @test UndatedDate(0) == test - Undated(1)
    @test UndatedDate(Int128(1)) == test
    @test_throws InexactError UndatedDate(170141183460469231731687303715884105727)
    @test UndatedDate(UInt128(1)) == test
    @test UndatedDate(big(1)) == test
    @test UndatedDate(big(1)) == test
    # Potentially won't work if can't losslessly convert to Int64
    @test UndatedDate(BigFloat(1)) == test
    @test UndatedDate(complex(1)) == test
    @test UndatedDate(Float64(1)) == test
    @test UndatedDate(Float32(1)) == test
    @test UndatedDate(Float16(1)) == test
    @test UndatedDate(Rational(1)) == test
    @test_throws InexactError UndatedDate(BigFloat(1.2))
    @test_throws InexactError UndatedDate(1 + im)
    @test_throws InexactError UndatedDate(1.2)
    @test_throws InexactError UndatedDate(1.2f0)
    @test_throws InexactError UndatedDate(3 // 4) == test

    # Semesters, Quarters, Months, Weeks Days must be in range
    @test_throws ArgumentError DayDate(2013, 0, 1)
    @test_throws ArgumentError DayDate(2013, 13, 1)
    @test_throws ArgumentError DayDate(2013, 1, 0)
    @test_throws ArgumentError DayDate(2013, 1, 32)

    @test_throws ArgumentError WeekDate(2013, 0)
    @test_throws ArgumentError WeekDate(2013, 54)

    @test_throws ArgumentError MonthDate(2013, 0)
    @test_throws ArgumentError MonthDate(2013, 13)

    @test_throws ArgumentError QuarterDate(2013, 0)
    @test_throws ArgumentError QuarterDate(2013, 5)

    @test_throws ArgumentError SemesterDate(2013, 0)
    @test_throws ArgumentError SemesterDate(2013, 3)

end
a = Dates.DateTime(2000)
b = Dates.Date(2000)
c = Dates.Time(0)
@testset "DateTime traits" begin
    for (T1, T2) in zip(
        (:DayDate, :WeekDate, :MonthDate, :QuarterDate, :YearDate),
        (:Day, :Week, :Month, :Quarter, :Year),
    )
        @eval begin
            @test ExtendedDates.calendar($T1(2000)) == Dates.ISOCalendar
            @test eps($T1) == Dates.$T2(1)
            @test zero($T1) == Dates.$T2(0)
            @test isfinite($T1)
        end
    end
end
@testset "Date-DateTime conversion/promotion" begin
    for T in (
        :DayDate,
        :WeekDate,
        :MonthDate,
        :QuarterDate,
        :SemesterDate,
        :YearDate,
        :UndatedDate,
    )
        @eval begin
            a = $T(2000)
            b = $T(2000)
            @test $T(a) == a
            @test $T(a) == b
            @test $T(b) == b
            @test a == b
            @test a == a
            @test b == a
            @test b == b
            @test !(a < b)
            @test !(b < a)
            b = $T(2001)
            @test b > a
            @test a < b
            @test a != b
        end
    end
end

@testset "min and max" begin
    for T in (
        :DayDate,
        :WeekDate,
        :MonthDate,
        :QuarterDate,
        :SemesterDate,
        :YearDate,
        :UndatedDate,
    )
        @eval begin
            a = $T(2000)
            b = $T(2001)
            @test min(a, b) == a
            @test min(b, a) == a
            @test min(a) == a
            @test max(a, b) == b
            @test max(b, a) == b
            @test max(b) == b
            @test minmax(a, b) == (a, b)
            @test minmax(b, a) == (a, b)
            @test minmax(a) == (a, a)
        end
    end

    for T in (:DayDate, :WeekDate, :MonthDate, :QuarterDate, :SemesterDate)
        @eval begin
            a = $T(2000, 1)
            b = $T(2000, 2)
            @test min(a, b) == a
            @test min(b, a) == a
            @test min(a) == a
            @test max(a, b) == b
            @test max(b, a) == b
            @test max(b) == b
            @test minmax(a, b) == (a, b)
            @test minmax(b, a) == (a, b)
            @test minmax(a) == (a, a)
        end
    end

    @test DayDate(2000, 1, 1) < DayDate(2000, 1, 2)

end

@testset "simpleperiod" begin
    @test ExtendedDates.simpleperiod(FYear, 2000) == YearDate(2000)
    @test ExtendedDates.simpleperiod(FSemester, 2000, 2) == SemesterDate(2000, 2)
    @test ExtendedDates.simpleperiod(FQuarter, 2000, 4) == QuarterDate(2000, 4)
    @test ExtendedDates.simpleperiod(FMonth, 2000, 11) == MonthDate(2000, 11)
    @test ExtendedDates.simpleperiod(FWeek, 2000, 52) == WeekDate(2000, 52)
    @test ExtendedDates.simpleperiod(FDay, 2004, 2, 29) == DayDate(2004, 2, 29)
    @test ExtendedDates.simpleperiod(FUndated, 2000) == UndatedDate(2000)
end

end


# This application is licensed under the MIT License.
#
# Copyright (c) 2020 Gustavo A. Hoffmann
#
# https://github.com/gusthoff/chaos-music
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

# using Pkg; Pkg.add("DynamicalSystems")
# using Pkg; Pkg.add("MusicXML")
# using Pkg; Pkg.add("Plots")

using DynamicalSystems
using MusicXML
@importMX # imports all the MusicXML types
# using Plots

##########################################################################
# Create logistic map
##########################################################################

lg = Systems.logistic()
tr = trajectory(lg, 99)

# tr = rand(Float64, 100)

# plot(1:101, tr)

##########################################################################
# MusicXML file: general definition for piano part
#
# Based on example code from MusicXML file
##########################################################################

midiinstrument_piano = MidiInstrument(channel= 1,
                                      program =1,
                                      volume = 100,
                                      pan = 0,
                                      id = "P1-I1")
scorepart_piano = ScorePart(name = "Piano",
                            midiinstrument = midiinstrument_piano,
                            id = "P1")

partlist = PartList(scoreparts = [scorepart_piano])

attributes1_piano = Attributes(
   time = Time(beats = 4, beattype = 4), # 4/4
   divisions = 4, # we want to use 16th notes at minimum
   clef = [Clef(number = 1, sign = "G", line = 2),
           Clef(number = 2, sign = "F", line = 4)], # Piano clefs
   staves = 2, # Piano staves
   key = Key(fifths = 0, mode = "major"), # no accidentals, major key
)

##########################################################################
# Generate notes based on logistic map
##########################################################################

notes = ["C", "D", "E", "F", "G", "A", "B", "C"]

measures_piano = Measure[]
measure1_notes_piano = Note[]

first = true

for (i, j) in enumerate(tr)
    curr_note = notes[round(Int, j*length(notes) + 0.5)]
    push!(measure1_notes_piano,
          Note(pitch = Pitch(step = curr_note, alter = 0, octave = 4),
               duration = 4))

    # Add measure if complete
    if i % 4 == 0
        if first
            push!(measures_piano,
                  Measure(attributes = attributes1_piano,
                          notes = measure1_notes_piano))
            global first = false
        else
            push!(measures_piano,
                  Measure(notes = measure1_notes_piano))
        end
        global measure1_notes_piano = Note[]
    end
end

# Add remaining notes to measure

if length(measure1_notes_piano) > 0
    if first
        push!(measures_piano,
              Measure(attributes = attributes1_piano,
                      notes = measure1_notes_piano))
        global first = false
    else
        push!(measures_piano,
              Measure(notes = measure1_notes_piano))
    end
end

##########################################################################
# MusicXML file: write output file
##########################################################################

part_piano = Part(measures = measures_piano, id = "P1")

score = ScorePartwise(partlist = partlist, parts = [part_piano],)

writemusicxml("myscore.musicxml", score)


using Base.Iterators: partition
using Flux
using Flux.Optimise: update!
using Flux.Losses: mse, mae
using Images
using Statistics
using Parameters: @with_kw
using Random
using Printf
using Zygote
using CycleGAN: Discriminator, Generator

size_img = 28

@with_kw struct HyperParams
    batch_size::Int = 5
    epochs::Int = 100
    verbose_freq::Int = 800
    size_dataset::Int = 1000
    lr_dscr_A::Float64 = 0.00005
    lr_gen_A::Float64 = 0.00005
    lr_dscr_B::Float64 = 0.00005
    lr_gen_B::Float64 = 0.00005
end

function convertI2Float(img)
    img_resize = imresize(float.(img), (size_img,size_img))
    if length(size(img_resize)) == 2
        img_resize = RGB.(img_resize)
    end
	return permutedims(channelview(img_resize), (3,2,1))
end

function load_images(path::String, size::Int)
	images= zeros(Float32,size_img,size_img,3,size)
	for (index, img) in enumerate(readdir(path, join = true))
		images[:,:,:,index] = convertI2Float(load(img))
        if index == size
            break
        end
	end
	return images
end

function load_data(hparams)
    # Load folder dataset
    images_A = load_images("horse2zebra/trainA/", hparams.size_dataset)
    images_B = load_images("horse2zebra/trainB/", hparams.size_dataset)
    data = [ (images_A[:,:,:, r], images_B[:,:,:, r]) |> gpu for r in partition(1:hparams.size_dataset, hparams.batch_size)]
    return data
end

function create_output_image(gen, image)
    @eval Flux.istraining() = false
    fake_image = cpu(gen(image))
    @eval Flux.istraining() = true
    image_array = permutedims(dropdims(fake_image; dims=4), (3,2,1))
    image_array = colorview(RGB, image_array)
    return clamp01nan.(image_array)
end

function train()
    hparams = HyperParams()

    data = load_data(hparams)

    #test images
    test_images_A=zeros(Float32,size_img,size_img,3,1)
    test_images_B=zeros(Float32,size_img,size_img,3,1)
    test_images_A[:,:,:,1] = convertI2Float(load("horse2zebra/testA/n02381460_1000.jpg"))
    test_images_B[:,:,:,1] = convertI2Float(load("horse2zebra/testB/n02391049_100.jpg"))

    # Discriminator
    dscr_A = Discriminator()
    dscr_B = Discriminator()

    # Generator
    gen_A =  Generator(3, 64) |> gpu
    gen_B =  Generator(3, 64) |> gpu

    # Optimizers
    opt_dscr = ADAM(hparams.lr_dscr_A, (0.5,0.99))
    opt_gen = ADAM(hparams.lr_gen_A, (0.5,0.99))

    cyclegGAN = Cyclegan(gen_A, gen_B, dscr_A, dscr_B)

    isdir("output")||mkdir("output")

    # Training
    train_steps = 0
    for ep in 1:hparams.epochs
        @info "Epoch $ep"
        for (x,y) in data
                # Update discriminator and generator
            loss = train_gan(cyclegGAN, x, y, opt_gen, opt_dscr)
            if train_steps % hparams.verbose_freq == 0
                @info("Train step $(ep), Discriminator loss = $(loss["D_loss"]), Generator loss = $(loss["G_loss"])")
                # Save generated fake image
                output_image_A = create_output_image(cyclegGAN.gen_AB, test_images_B |> gpu)
                output_image_B = create_output_image(cyclegGAN.gen_BA, test_images_A |> gpu)
                save(@sprintf("output/cgan_A_steps_%06d.png", train_steps), output_image_A)
                save(@sprintf("output/cgan_B_steps_%06d.png", train_steps), output_image_B)
            end
        train_steps += 1
        end
    end
    @info("Finish  training")
    output_image_A = create_output_image(cyclegGAN.gen_AB, test_images_B |> gpu)
    output_image_B = create_output_image(cyclegGAN.gen_BA, test_images_A |> gpu)
    save("output/cgan_A_steps_final.png", output_image_A)
    save("output/cgan_B_steps_final.png", output_image_B)
end

cd(@__DIR__)
train()


struct STLGeometry <: CSG.Geometry
  tree::Leaf{Tuple{T,String,Nothing}} where T<:DiscreteModel
end

function STLGeometry(stl::DiscreteModel;name="stl")
  tree = Leaf( ( stl, name, nothing ) )
  STLGeometry( tree )
end

function STLGeometry(filename::String;name="stl")
  stlmodel = _to_stl_model(filename)
  STLGeometry(stlmodel,name=name)
end

get_tree(geo::STLGeometry) = geo.tree

get_stl(geo::STLGeometry) = geo.tree.data[1]

function compatible_geometries(a::STLGeometry,b::STLGeometry)
  a,b
end

function similar_geometry(a::STLGeometry,tree::Leaf)
  STLGeometry(tree)
end

get_bounding_box(a::STLGeometry) = get_bounding_box( get_stl(a) )

struct STLCutter <: Interfaces.Cutter
  options::Dict{Symbol,Any}
  STLCutter(;options...) = new(options)
end

function cut(cutter::STLCutter,background::DiscreteModel,geom::STLGeometry)
  data,bgf_to_ioc = _cut_stl(background,geom;cutter.options...)
  EmbeddedDiscretization(background, data..., geom), bgf_to_ioc
end

function cut(background::DiscreteModel,geom::STLGeometry)
  cutter = STLCutter()
  cut(cutter,background,geom)
end

function _cut_stl(model::DiscreteModel,geom::STLGeometry;kwargs...)
  subcell_grid, subface_grid, labels = subtriangulate(model,geom;kwargs...)

  inout_dict = Dict{Int8,Int8}(
    FACE_IN => IN, FACE_OUT => OUT, FACE_CUT => CUT, UNSET => OUT )

  cell_to_io = [ replace( labels.cell_to_io, inout_dict... ) ]
  cell_to_bgcell = labels.cell_to_bgcell
  cell_to_points = get_cell_node_ids( subcell_grid )
  point_to_coords = get_node_coordinates( subcell_grid )
  point_to_rcoords = send_to_ref_space(model,cell_to_bgcell,subcell_grid)
  data = cell_to_points,cell_to_bgcell,point_to_coords,point_to_rcoords
  subcells = SubCellData(data...)

  face_to_bgcell = labels.face_to_bgcell
  face_to_points = get_cell_node_ids( subface_grid )
  point_to_coords = get_node_coordinates( subface_grid )
  point_to_rcoords = send_to_ref_space(model,face_to_bgcell,subface_grid)
  face_to_normal = _normals(geom,labels.face_to_stlface)
  data = face_to_points,face_to_normal,face_to_bgcell,point_to_coords,point_to_rcoords
  subfacets = SubFacetData(data...)

  face_to_io = [ fill(Int8(INTERFACE),num_cells(subface_grid)) ]

  bgface_to_ioc = replace( labels.bgface_to_ioc, inout_dict... )
  bgcell_to_ioc = [ replace( labels.bgcell_to_ioc, inout_dict... ) ]

  oid_to_ls = Dict{UInt,Int}( objectid( get_stl(geom) ) => 1  )
  (bgcell_to_ioc,subcells,cell_to_io,subfacets,face_to_io,oid_to_ls),bgface_to_ioc
end

function send_to_ref_space(grid::Grid,cell_to_bgcell::Vector,subgrid::Grid)
  send_to_ref_space(
    grid,
    cell_to_bgcell,
    get_cell_node_ids(subgrid),
    get_node_coordinates(subgrid))
end

function send_to_ref_space(
  grid::Grid,
  cell_to_bgcell::Vector,
  cell_to_points::AbstractVector,
  points::Vector)

  rpoints = zero(points)
  cache = array_cache(cell_to_points)
  bgcache = array_cache(get_cell_node_ids(grid))
  for i in 1:length(cell_to_points)
    for p in getindex!(cache,cell_to_points,i)
      rpoints[p] = send_to_ref_space!(bgcache,grid,cell_to_bgcell[i],points[p])
    end
  end
  rpoints
end

function send_to_ref_space!(cache,grid::Grid,cell::Integer,point::Point)
  cell_nodes = getindex!(cache,get_cell_node_ids(grid),cell)
  node_coordinates = get_node_coordinates(grid)
  pmin = node_coordinates[cell_nodes[1]]
  pmax = node_coordinates[cell_nodes[end]]
  Point(Tuple(point-pmin)./Tuple(pmax-pmin))
end

function _normals(geom::STLGeometry,face_to_stlface)
  stl = get_stl(geom)
  cache = get_cell_cache(stl)
  N = typeof(normal(get_cell!(cache,stl,1)))
  normals = zeros(N,length(face_to_stlface))
  for (i,stlf) in enumerate( face_to_stlface )
    normals[i] = normal(get_cell!(cache,stl,stlf))
  end
  normals
end

_to_stl_model(geo::STLGeometry) =  get_stl(geo)

surface(geo::STLGeometry) = surface(get_grid(get_stl(geo)))