function _move_loop!(
    N::Integer,
    positions::AbstractArray,
    forces::AbstractArray,
    pot::PairwisePotential,
    energies::AbstractArray,
    interval::Integer,
    params;
    rdf = nothing,
    zfactor = nothing,
    msd = nothing,
)
    # * Main loop
    for i = 1:N
        # Always re-compute random numbers
        rng_matrix!(params.random_matrix, params.rng_list)

        # We check for the MeanSquaredDisplacement first as it needs special treatment
        # with the periodic boundary conditions
        if !isnothing(msd)
            ermak!(
                positions,
                forces,
                params.τ,
                params.boxl,
                params.random_matrix;
                pbc = false,
            )
            total_energy = energy_force!(positions, forces, params, pot)
            # We save positions in time here as needed by the MeanSquaredDisplacement
            if i % msd.interval == 0
                record_positions!(msd, positions, params.τ)
            end
        else
            # If it happens that we don't need the MeanSquaredDisplacement, then we proceed
            # moving the particles around
            ermak!(positions, forces, params.τ, params.boxl, params.random_matrix)

            # We check to see if we need to compute physical observables, if not
            # we just move particles around and check their energy
            if isnothing(rdf) && isnothing(zfactor)
                total_energy = energy_force!(positions, forces, params, pot)

                # We now check to see if the PairDistributionFunction is needed, if so
                # we step in and compute it
            elseif !isnothing(rdf)
                total_energy = energy_force!(positions, forces, params, pot; gofr = rdf)
                rdf.naverage += 1
                # If it so happens that the CompressibilityFactor needs to be computed
                # as well, we check to see if the object is passed and compute it
                if !isnothing(zfactor)
                    total_energy =
                        energy_force!(positions, forces, params, pot; zfactor = zfactor)
                    zfactor.naverage += 1
                end

                # If nothing but the CompressibilityFactor is needed, we compute it here
            elseif !isnothing(zfactor)
                total_energy =
                    energy_force!(positions, forces, params, pot; zfactor = zfactor)
                zfactor.naverage += 1
            end
        end

        # Finally, we check the interval and print the total energy
        # as well as save it
        if i % interval == 0
            @show total_energy
            idx = Int(i / interval)
            energies[idx] = total_energy
        end
    end

    # Lastly, we update the energy for the full system
    total_energy = energy_force!(positions, forces, params, pot)

    return total_energy
end

function _prepare(s::SimulationSystem, N::Integer, interval::Integer)
    # Allocate information for random values
    (_, rng_list) = _create_rngs(s.dims; seed = s.params.seed)
    random_matrix = zeros(typeof(s.boxl), size(s.system.positions))

    # Create an energy array to store values
    energy_size = Int(N / interval)

    # 2 dimensions because we only need the energy value and the movement number
    energies = zeros(energy_size, 2)

    # A collection of multiple parameters, it makes it easier to move it around
    # functions
    params = (
        N = s.params.N,
        rc = s.rc,
        boxl = s.boxl,
        rc2 = s.rc * s.rc,
        random_matrix = random_matrix,
        rng_list = rng_list,
        τ = s.params.τ,
    )

    return energies, params
end

function move!(N::Integer, s::SimulationSystem, pot::PairwisePotential; interval = 1000)
    # Retrieve system information
    @unpack positions, forces = s.system

    # Prepare all the arrays
    (energies, params) = _prepare(s, N, interval)

    # And then we move all the particles
    total_energy = _move_loop!(N, positions, forces, pot, energies, interval, params)
    # Lastly, we assign the system's energy
    s.energy = total_energy
end

function move!(
    N::Integer,
    s::SimulationSystem,
    pot::PairwisePotential,
    grobject::PairDistributionFunction;
    interval = 1000,
)
    # Retrieve system information
    @unpack positions, forces = s.system
    (energies, params) = _prepare(s, N, interval)

    # Pass in the PairDistributionFunction object and move particles
    total_energy =
        _move_loop!(N, positions, forces, pot, energies, interval, params; rdf = grobject)
    s.energy = total_energy

    # After all the statistics have been collected, we normalize the
    # PairDistributionFunction object
    grobject.naverage = N
    compute_rdf!(grobject, s)
end

function move!(
    N::Integer,
    s::SimulationSystem,
    pot::PairwisePotential,
    grobject::PairDistributionFunction,
    zfactor::ZFactor;
    interval = 1000,
)
    # Retrieve system information
    @unpack positions, forces = s.system
    (energies, params) = _prepare(s, N, interval)

    # Pass in the PairDistributionFunction and CompressibilityFactor objects
    total_energy = _move_loop!(
        N,
        positions,
        forces,
        pot,
        energies,
        interval,
        params;
        rdf = grobject,
        zfactor = zfactor,
    )

    # We compute the full CompressibilityFactor here
    zfactor.zval = compute_z(zfactor, s.params.N)

    # After all the statistics have been collected, we normalize the
    # PairDistributionFunction object
    grobject.naverage = N
    compute_rdf!(grobject, s)

    # Assign the full system's energy
    s.energy = total_energy
end

function move!(
    N::Integer,
    s::SimulationSystem,
    pot::PairwisePotential,
    zfactor::ZFactor;
    interval = 1000,
)
    # Retrieve system information
    @unpack positions, forces = s.system
    (energies, params) = _prepare(s, N, interval)
    total_energy = _move_loop!(
        N,
        positions,
        forces,
        pot,
        energies,
        interval,
        params;
        zfactor = zfactor,
    )
    # We compute the full CompressibilityFactor here
    zfactor.zval = compute_z(zfactor, s.params.N)

    # Assign the full system's energy
    s.energy = total_energy
end

function move!(
    N::Integer,
    s::SimulationSystem,
    pot::PairwisePotential,
    msd::MeanSquaredDisplacement;
    interval = 1000,
)
    # Retrieve system information
    @unpack positions, forces = s.system
    (energies, params) = _prepare(s, N, interval)
    _move_loop!(N, positions, forces, pot, energies, interval, params; msd = msd)

    # Compute and normalize the MeanSquaredDisplacement with all the recorded positions
    difusion!(msd)
end

function move!(
    N::Integer,
    s::SimulationSystem,
    pot::PairwisePotential,
    msd::MeanSquaredDisplacement,
    sft::SelfScatteringFunction;
    interval = 1000,
)
    # Retrieve system information
    @unpack positions, forces = s.system
    (energies, params) = _prepare(s, N, interval)
    _move_loop!(N, positions, forces, pot, energies, interval, params; msd = msd)

    # Copy the displacement vector from the MeanSquaredDisplacement to the
    # SelfScatteringFunction
    sft.dft[:, 1] = copy(msd.wt[:, 1])

    # Compute and normalize the MeanSquaredDisplacement and the SelfScatteringFunction
    # with all the recorded positions
    difusion!(msd, sft)
end


module day5

using ..aoc19: run_program!

readInput() = parse.(Int,split(readline(joinpath(@__DIR__, "input.txt")), ','))

function part1(data, userInput = 1)
    output = Channel{Int}(Inf)
    input = Channel{Int}(Inf)
    put!(input, userInput)
    run_program!(copy(data), input, output)
    val = take!(output)
    while val == 0
        val = take!(output)
    end
    val
end

part2(input) = part1(input, 5)

end


module CbiomesProcessing

using YAML, FortranFiles, JLD, Dates
using Distributed, DistributedArrays
using SparseArrays, MeshArrays
using MITgcmTools

include("utilities.jl")
include("fileloops.jl")
include("examples.jl")

export StartWorkers, TaskDriver
export cbioproc_example1, cbioproc_example2

#export AverageYear

end # module


module SpinMC

include("UnitCell.jl")
export UnitCell, addInteraction!, setInteractionOnsite!, setField!, addBasisSite!
include("InteractionMatrix.jl")
include("Lattice.jl")
export Lattice, size, length, getSpin, setSpin!, getSitePosition

include("Observables.jl")
export Observables
include("Spin.jl")
export getEnergy, getMagnetization, getCorrelation

include("MonteCarlo.jl")
export MonteCarlo, run!

include("Helper.jl")
include("IO.jl")
export writeMonteCarlo, readMonteCarlo

using Reexport
@reexport using BinningAnalysis

end


function combine_decomposition!(db::SQLite.DB, id::Int, name::AbstractString, scenario::Union{Int, AbstractString},
                                clique_path::AbstractString, cliquetree_path::AbstractString, nb_added_edge_dec::Int,
                                id_1::Int, graph_path_1::AbstractString,
                                id_2::Int, graph_path_2::AbstractString;
                                how::AbstractString, extension_alg::AbstractString, rng)
    println("Combining decompositions ($name, $scenario) (in1=$id_1, in2=$id_2)")

    # Retrieve the graphs
    g1 = loadgraph(graph_path_1)
    g2 = loadgraph(graph_path_2)

    # Combine
    g3 = combine_graph(g1, g2; how=how, extension_alg=extension_alg)

    # Extract features
    clique = maximal_cliques(g3)
    cliquetree, nb_lc = get_cliquetree_array(clique)
    features = get_features_graph(g3)
    features["nb_lc"] = nb_lc
    features["nb_added_edge_dec"] = nb_added_edge_dec + (ne(g3) - ne(g1))
    merge!(features, get_clique_features(clique))

    # Save cliques, cliquetree and graph
    cliques_path = dirname(clique_path)
    cliquetrees_path = dirname(cliquetree_path)
    graphs_path = dirname(graph_path_1)
    uuid = uuid1(rng)
    clique_path = joinpath(cliques_path, "$(name)_$(scenario)_$(uuid)_clique.csv")
    cliquetree_path = joinpath(cliquetrees_path, "$(name)_$(scenario)_$(uuid)_cliquetree.csv")
    graph_path = joinpath(graphs_path, "$(name)_$(scenario)_$(uuid)_graph.lgz")
    save_clique(clique, clique_path)
    save_cliquetree(cliquetree, cliquetree_path)
    savegraph(graph_path, g3)

    # Other columns
    date = Dates.now()

    features = Dict(Symbol(k) => v for (k, v) in features)
    insert_decomposition!(db, id, uuid, name, scenario, "combine", extension_alg, date, clique_path, cliquetree_path, graph_path; features...)

    # Insert in merge table
    out_id = DBInterface.execute(db, "SELECT id FROM decompositions WHERE uuid = '$uuid'") |> DataFrame
    out_id = out_id[1, :id]
    insert_combination!(db, id_1, id_2, out_id, how, extension_alg)
end

function combine_decomposition_dfrow!(db::SQLite.DB, row; how, extension_alg, rng)
    combine_decomposition!(db, row["origin_id"], row["origin_name"], row["origin_scenario"],
                           row["clique_path"], row["cliquetree_path"], row["nb_added_edge_dec"],
                           row["id"], row["graph_path"],
                           row["id_1"], row["graph_path_1"];
                           how=how, extension_alg=extension_alg, rng=rng)
end

function combine_decompositions!(db::SQLite.DB; how::AbstractString, extension_alg::AbstractString,
                                 min_nv=typemin(Int), max_nv=typemax(Int), subset=nothing, exclude=["combine"], rng=MersenneTwister(42))
    query = """
    SELECT d1.origin_id, d1.id, d1.origin_name, d1.origin_scenario, d1.graph_path, d1.clique_path, d1.cliquetree_path, d1.nb_added_edge_dec, d1.extension_alg,
           d2.id, d2.origin_name, d2.origin_scenario, d2.graph_path, d2.extension_alg
    FROM decompositions AS d1 CROSS JOIN decompositions AS d2
    WHERE d1.origin_name = d2.origin_name AND d1.origin_scenario = d2.origin_scenario AND d1.id > d2.id
    """
    if !isnothing(subset)
        query *= " AND d1.origin_id IN ($(join(subset, ',')))"
    end
    for e in exclude
        query *= " AND d1.extension_alg <> '$e' AND d2.extension_alg <> '$e'"
    end
    results = DBInterface.execute(db, query) |> DataFrame
    println("Combine config: how=$how, extension_alg=$extension_alg, min_nv=$min_nv, max_nv=$max_nv, exlucde=$exclude, rng=$rng")
    println("subset\n$subset\nend subset")
    combine_function!(row) = combine_decomposition_dfrow!(db, row; how=how, extension_alg=extension_alg, rng=rng)
    combine_function!.(eachrow(results[!, ["origin_id", "origin_name", "origin_scenario", "clique_path", "cliquetree_path", "nb_added_edge_dec",
                                           "id", "graph_path", "id_1", "graph_path_1"]]))
end


using ProtoBuf

type MyType
  x::Array{Float64,1}
  d::Dict{Int64,Int64}
  MyType() = new()
  MyType(a::Array{Float64,1},b::Dict{Int64,Int64}) = new(a,b)
end

type CompType
  mt::MyType
  str::ASCIIString
  CompType() = new()
  CompType(a::MyType,s::ASCIIString) = new(a,s)
end

iob  = PipeBuffer()

mt = MyType(rand(3),Dict{Int,Int}(1 => 10, 2 => 20))
s = ASCIIString("Hello world.")
ct = CompType(mt,s)

writeproto(iob, ct)
dd = readproto(iob, CompType())

@time writeproto(iob, ct)
@time dd = readproto(iob, CompType())


using AutoHashEquals

"""
    Compound <: RDate

Apply a list of rdates in order
"""
@auto_hash_equals struct Compound <: RDate
    parts::Vector{RDate}
end

apply(rdate::Compound, date::Dates.Date, cal_mgr::CalendarManager) = Base.foldl((x,y) -> apply(y, x, cal_mgr), rdate.parts, init=date)
multiply(rdate::Compound, count::Integer) = Compound(map(x -> multiply(x, count), rdate.parts))
combine(left::RDate, right::RDate) = Compound([left,right])
Base.:+(left::RDate, right::RDate) = combine(left, right)
Base.:-(left::RDate, right::RDate) = combine(left, -right)
Base.:-(rdate::Compound) = Compound(map(-, rdate.parts))
Base.:+(x::Compound, y::RDate) = Compound(vcat(x.parts, y))
Base.:+(x::RDate, y::Compound) = Compound(vcat(x, y.parts))

function Base.show(io::IO, rdate::Compound)
    for (i,part) in enumerate(rdate.parts)
        if i > 1 print(io, "+") end
        show(io, part)
    end
end

"""
    Repeat <: RDate

Repeat the application of an rdate n times.
"""
@auto_hash_equals struct Repeat <: RDate
    count::Int64
    part::RDate

    function Repeat(count::Int64, part::RDate)
        count > 0 || error("Repeat must use a positive count, not $count")
        new(count, part)
    end

    Repeat(part::RDate) = new(1, part)
end

function apply(rdate::Repeat, date::Dates.Date, cal_mgr::CalendarManager)
    for _ in 1:rdate.count
        date = apply(rdate.part, date, cal_mgr)
    end
    date
end

multiply(rdate::Repeat, count::Integer) = Repeat(count*rdate.count, rdate.part)
Base.:-(rdate::Repeat) = Repeat(rdate.count, -rdate.part)
Base.show(io::IO, rdate::Repeat) = (print(io, "$(rdate.count)*Repeat("), show(io, rdate.part), print(io,")"))

"""
    CalendarAdj(calendars, rdate::RDate, rounding::HolidayRoundingConvention)

Apply a calendar adjustment to an underlying rdate, applying an appropriate convention if
our final date falls on a holiday.

The calendars can only use alphanumeric characters, plus `/`, `-` and ` `.

In the string-form, you can apply a calendar adjustment using the `@` character and
provide `|` separated calendar names to apply it on. The convention is finally provided in
square brackets using its string-form name.

### Examples
```julia-repl
julia> cal_mgr = SimpleCalendarManager()
julia> setcalendar!(cal_mgr, "WEEKEND", WeekendCalendar())
julia> apply(rd"1d", Date(2019,9,27), cal_mgr)
2019-09-28
julia> apply(rd"1d@WEEKEND[NBD]", Date(2019,9,27), cal_mgr)
2019-09-30
julia> apply(rd"2m - 1d", Date(2019,7,23), cal_mgr)
2019-09-22
julia> apply(rd"(2m - 1d)@WEEKEND[PBD]", Date(2019,7,23), cal_mgr)
2019-09-20
```
"""
@auto_hash_equals struct CalendarAdj{R <: RDate, S <: HolidayRoundingConvention} <: RDate
    calendar_names::Vector{String}
    part::R
    rounding::S

    CalendarAdj(calendar_names, part::R, rounding::S) where {R <: RDate, S <: HolidayRoundingConvention} = new{R, S}(calendar_names, part, rounding)
    CalendarAdj(calendar_names::String, part::R, rounding::S) where {R<:RDate, S <: HolidayRoundingConvention} = new{R,S}(split(calendar_names, "|"), part, rounding)
end

function apply(rdate::CalendarAdj, date::Dates.Date, cal_mgr::CalendarManager)
    base_date = apply(rdate.part, date, cal_mgr)
    cal = calendar(cal_mgr, rdate.calendar_names)
    apply(rdate.rounding, base_date, cal)
end
multiply(rdate::CalendarAdj, count::Integer) = CalendarAdj(rdate.calendar_names, multiply(rdate.part, count), rdate.rounding)

Base.:-(x::CalendarAdj) = CalendarAdj(x.calendar_names, -x.part, x.rounding)
Base.show(io::IO, rdate::CalendarAdj) = print(io, "($(rdate.part))@$(join(rdate.calendar_names, "|"))[$(rdate.rounding)]")

"""
    Next(parts, inclusive::Bool = false)

Next is a mechanism through which we can find the next closest date in the future, given a list of
rdates to apply. We can choose whether today is also deemed a valid date.

This is commonly used in conjunction with rdates which don't necessarily always give a date in the
future, such as asking for the next Easter from today.

### Examples
```julia-repl
julia> RDates.Next([RDates.Easter(0), RDates.Easter(1)]) + Date(2019,1,1)
2019-04-21
julia> rd"Next(0E,1E)" + Date(2019,1,1)
2019-04-21
julia> RDates.Next([RDates.Easter(0), RDates.Easter(1)]) + Date(2019,4,21)
2020-04-12
julia> RDates.Next([RDates.Easter(0), RDates.Easter(1)], true) + Date(2019,4,21)
2019-04-21
julia> rd"Next!(0E,1E)" + Date(2019,4,21)
2019-04-21
```

!!! note
    The negation of `Next` will actually produce a `Previous`.

    ```julia-repl
    julia> -rd"Next(1d,2d)"
    Previous(-1d, -2d)
    julia> -3 * rd"Next(1d,2d)"
    Previous(-3d, -6d)
    ```

!!! warning
    While `Next` is a powerful operator, it does require application of every rdate
    every time, so can be expensive.

    When combining with ranges, it can often be useful to use an appropriate pivot
    point to start from instead.
"""
@auto_hash_equals struct Next <: RDate
    parts::Vector{RDate}
    inclusive::Bool

    Next(parts, inclusive::Bool = false) = new(parts, inclusive)
end

"""
    Previous(parts, inclusive::Bool = false)

Previous is a mechanism through which we can find the next closest date in the past, given a list of
rdates to apply. We can choose whether today is also deemed a valid date.

This is commonly used in conjunction with rdates which don't necessarily always give a date in the
future, such as asking for the previous Easter from today.

### Examples
```julia-repl
julia> RDates.Previous([RDates.Easter(0), RDates.Easter(-1)]) + Date(2019,12,31)
2019-04-21
julia> rd"Previous(0E,-1E)" + Date(2019,12,31)
2019-04-21
julia> RDates.Previous([RDates.Easter(0), RDates.Easter(-1)]) + Date(2019,4,21)
2018-04-01
julia> RDates.Previous([RDates.Easter(0), RDates.Easter(-1)], true) + Date(2019,4,21)
2019-04-21
julia> rd"Previous!(0E,-1E)" + Date(2019,4,21)
2019-04-21
```

!!! note
    The negation of `Previous` will actually produce a `Next`.

    ```julia-repl
    julia> -rd"Previous(-1d,-2d)"
    Next(1d, 2d)
    julia> -3 * rd"Previous(-1d,-2d)"
    Next(3d, 6d)
    ```

!!! warning
    While `Previous` is a powerful operator, it does require application of every rdate
    every time, so can be expensive.

    When combining with ranges, it can often be useful to use an appropriate pivot
    point to start from instead.
"""
@auto_hash_equals struct Previous <: RDate
    parts::Vector{RDate}
    inclusive::Bool

    Previous(parts, inclusive::Bool = false) = new(parts, inclusive)
end

function apply(rdate::Next, date::Dates.Date, cal_mgr::CalendarManager)
    dates = map(part -> apply(part, date, cal_mgr), rdate.parts)
    op = rdate.inclusive ? Base.:>= : Base.:>
    dates = [d for d in dates if op(d, date)]
    length(dates) > 0 || error("$rdate failed to apply to $date")
    min(dates...)
end

function apply(rdate::Previous, date::Dates.Date, cal_mgr::CalendarManager)
    dates = map(part -> apply(part, date, cal_mgr), rdate.parts)
    op = rdate.inclusive ? Base.:<= : Base.:<
    dates = [d for d in dates if op(d, date)]
    length(dates) > 0 || error("$rdate failed to apply to $date")
    max(dates...)
end

function multiply(rdate::Next, count::Integer)
    method = count >= 0 ? Next : Previous
    method(map(x -> multiply(x, count), rdate.parts), rdate.inclusive)
end

function multiply(rdate::Previous, count::Integer)
    method = count >= 0 ? Previous : Next
    method(map(x -> multiply(x, count), rdate.parts), rdate.inclusive)
end

Base.:-(rdate::Next) = Previous(map(-, rdate.parts), rdate.inclusive)
Base.:-(rdate::Previous) = Next(map(-, rdate.parts), rdate.inclusive)

function Base.show(io::IO, rdate::Next)
    incsign = rdate.inclusive ? "!" : ""
    print(io, "Next$incsign(")
    for (i,part) in enumerate(rdate.parts)
        show(io, part)
        if i < length(rdate.parts)
            print(io, ", ")
        end
    end
    print(io, ")")
end

function Base.show(io::IO, rdate::Previous)
    incsign = rdate.inclusive ? "!" : ""
    print(io, "Previous$incsign(")
    for (i,part) in enumerate(rdate.parts)
        show(io, part)
        if i < length(rdate.parts)
            print(io, ", ")
        end
    end
    print(io, ")")
end


"""
 Tst
"""
function tosiunits(s)
    if !ismissing(s)
        a = split(s, " ")
        r = sum(Quantity.(eval.(Meta.parse.(a)), u"inch")) |> u"cm"
        round(typeof(1u"cm"), r)
    else
        missing
    end
end


# TODO
#	- Implment broadcasting
#
#	- Consider generalizing this to static bit arrays of any dimension.
#	- Also, consider incorporating this into the BitArray framework. This would
#		require a reworking BitArray; possibly parameterize by backing type?
#		e.g.	BitArray{N} = AbstractBitArray{N, Array{N,UInt64}}
#				SBitArray{S,N} = AbstractBitArray{N, SArray{S,N,UInt64}}
#				MBitArray{S,N} = AbstractBitArray{N, MArray{S,N,UInt64}}
#		then SBitVector{L} = SBitArray{Tuple{L}, 1}, etc.
"""
	BitStrings (module)

Dense-packed, stack-allocated bit strings.
(Basically a merging of SVector and BitVector.)
"""
module BitStrings

#using BaseExtensions: allequal
using StaticArrays
import Base: convert, promote_type, promote_rule
import Base: string, print, show, display
import Base: length, size, checkbounds, ndims, axes, getindex, setindex, setindex!, iterate
import Base: vcat
import Base: map, map!		#, bit_map!
import Base.Broadcast: broadcastable, BroadcastStyle, Broadcasted, broadcasted
import Base: count, sum
import Base: +, -, *, /
import LinearAlgebra: dot

export AbstractBitString, SBitString, MBitString, parity, hammdist, setindex

# TODO:	Support for other binary element types
# TODO:	Support multidimensional arrays?

## From BaseExtensions

const Iterable = Union{Tuple, AbstractArray, UnitRange, Base.Generator}

function allequal(it::Iterable)
	if length(it) <= 1
		return true
	end
	v = first(it)
	for v_ in it
		v_ === v || return false
	end
	return true
end

## OK, Now the actual stuff

abstract type AbstractBitString{L,N} <: AbstractVector{Bool} end

struct SBitString{L,N} <: AbstractBitString{L,N}
	chunks::SVector{N,UInt64}
end

struct MBitString{L,N} <: AbstractBitString{L,N}
	chunks::MVector{N,UInt64}
	MBitString{L,N}(ch) where {L,N} = new{L,N}(ch)
	MBitString{L,N}(::UndefInitializer) where {L,N} = new{L,N}(MVector{N,UInt64}(undef))
end

# These determine the appropriate constructor type for operations
promote_type(::Type{<:SBitString}) = SBitString
promote_type(::Type{<:MBitString}) = MBitString
# These are needed to strip the {L,N} parameters
promote_type(::Type{T}, ::Type{T}) where {T<:SBitString{L,N}} where {L,N} = SBitString
promote_type(::Type{T}, ::Type{T}) where {T<:MBitString{L,N}} where {L,N} = MBitString
promote_rule(::Type{<:SBitString}, ::Type{<:MBitString}) = MBitString

SBitString(b::Bool) = SBitString{1,1}((UInt64(b),))
MBitString(b::Bool) = MBitString{1,1}((UInt64(b),))

convert(::Type{SBitString}, b::Bool) = SBitString(b)
convert(::Type{MBitString}, b::Bool) = MBitString(b)

# AbstractBitString(bs::AbstractBitString{L,N}) where {L,N} = AbstractBitString{L,N}(bs.chunks)
#
#
# # Construction from integer
# AbstractBitString(i::Integer) = AbstractBitString{8*sizeof(typeof(i)),1}(SVector{1}(UInt64(i)))
#

# Construction from BitVector (mre efficient than for general iterables)
function (::Type{T})(b::BitVector) where T<:AbstractBitString
	T{length(b), length(b.chunks)}(b.chunks)
end

# Construction from any iterable
function (::Type{T})(A) where T<:AbstractBitString
	chunks = compute_chunks(A)
	T{length(A), length(chunks)}(chunks)
end

convert(::Type{SBitString{L,N}}, v::SVector{N,UInt64}) where {L,N} = SBitString{L,N}(v)
convert(::Type{MBitString{L,N}}, v::MVector{N,UInt64}) where {L,N} = MBitString{L,N}(v)

convert(::Type{T}, bs::T) where {T<:AbstractBitString} = bs
convert(::Type{SBitString}, bs::AbstractBitString{L,N}) where {L,N} = SBitString{L,N}(bs.chunks)
convert(::Type{SBitString{L,N}}, bs::AbstractBitString{L,N}) where {L,N} = SBitString{L,N}(bs.chunks)
convert(::Type{MBitString}, bs::AbstractBitString{L,N}) where {L,N} = MBitString{L,N}(bs.chunks)
convert(::Type{MBitString{L,N}}, bs::AbstractBitString{L,N}) where {L,N} = MBitString{L,N}(bs.chunks)

nchunks(len::Int) = Base._div64(len + 63)
nchunks(b::AbstractBitString{L,N}) where {L,N} = N

# Compute chunks from an iterable
function compute_chunks(A)
	len = length(A)
	nch = nchunks(len)

	# empty vector
	nch == 0 && return MVector{0,UInt64}()

	chunks = MVector{nch,UInt64}(undef)
	itr = iterate(A)
	@inbounds begin
		for ich = 1:nch-1
			ch = UInt64(0)
			for j = 0:63
				ch |= (UInt64(convert(Bool, itr[1])) << j)
				itr = iterate(A, itr[2])
			end
			chunks[ich] = ch
		end
		ch = UInt64(0)
		for j = 0:Base._mod64(len-1)
			ch |= (UInt64(convert(Bool, itr[1])) << j)
			itr = iterate(A, itr[2])
		end
		chunks[nch] = ch
	end
	return chunks
end

length(b::AbstractBitString{L}) where {L} = L
size(b::AbstractBitString{L}) where {L} = (L,)
ndims(b::AbstractBitString)  = 1
ndims(::Type{T}) where {T<:AbstractBitString} = 1
axes(b::AbstractBitString{L}) where {L} = (Base.OneTo(L),)

# # Conversion to BitVector
# convert(::Type{BitVector}, bs::AbstractBitString) = BitVector(bs)
# convert(::Type{AbstractBitString}, bv::BitVector) = AbstractBitString(bv)
#
# function BitVector(bs::AbstractBitString)
# 	bv = BitVector(undef, length(bs))
# 	bv.chunks = Vector(bs.chunks)
# 	bv
# end

"""
	string(bs::AbstractBitString)

Create a string representation of a `AbstractBitString`.
Bits are shown as 0,1 in order from left to right, with an underscore every 8 bits.
"""
function string(bs::AbstractBitString)
	mv = MVector{length(bs) + max(0,(length(bs)-1))>>3, Char}(undef)
	j = 1
	for i in 1:length(bs)
		mv[j] = bs[i] + '0'
		j += 1
		if mod(i, 8) == 0 && i < length(bs)
			mv[j] = '_'
			j += 1
		end
	end
	string(mv...)
end

display(bs::AbstractBitString) = show(bs)
show(io::IO, bs::SBitString) = print(io, string(bs))
show(io::IO, bs::MBitString) = print(io, "(", string(bs), ")")
#printstyled(io, string((bs.bvec .+ '0')...); bold=true)

##  Indexing

checkbounds(b::AbstractBitString{L}, I...) where {L} = Base.checkbounds_indices(Bool, (Base.OneTo(L),), I) || Base.throw_boundserror(b, I)

@inline function get_chunk_index(i::Integer)
	i1, i2 = Base.get_chunks_id(i)
	msk = UInt64(1) << i2
	return (i1, msk)
end

# getindex - entry point
@inline function getindex(b::AbstractBitString, i)
   @boundscheck checkbounds(b, i)
	_getindex(b, i)
end

# implementations (no bounds checking)
@inline _getindex(b::AbstractBitString, i::CartesianIndex{1}) = _getindex(b, i[1])
@inline _getindex(b::AbstractBitString, i::Integer) = _getindex(b, get_chunk_index(i))

# unsafe getindex using precomputed chunk index and mask
@inline function _getindex(b::AbstractBitString, (ich,msk)::Tuple{Int64, UInt64})
	# @info "_getindex (chunk): ich = $ich, msk = $msk"
	@inbounds r = (b.chunks[ich] & msk) != 0
	return r
end

# fallback / index by an iterable
function _getindex(b::T, itr) where T<:AbstractBitString
	 v = [_getindex(b, get_chunk_index(i)) for i in itr]
	 promote_type(T)(v)
end

function setindex!(bs::MBitString, val, i)
	@boundscheck checkbounds(bs, i)
	_setindex!(bs, val, i)
end

@inline function _setindex!(bs::MBitString, val::Bool, i::Integer)
	i1, i2 = Base.get_chunks_id(i)
	msk = ~(UInt64(1) << i2)
	@inbounds bs.chunks[i1] = (bs.chunks[i1] & msk) | (val << i2)
end

function _setindex!(bs::MBitString, val, itr)
	for (iv,ib) in enumerate(itr)
		_setindex!(bs, val[iv], ib)
	end
end

# function setindex(bs::AbstractBitString, val, i)
# 	@boundscheck checkbounds(bs, i)
# 	_setindex(bs, val, i)
# end
#
#
# @inline function _setindex(bs::AbstractBitString{L,N}, val::Bool, i::Integer) where {L,N}
# 	temp = MVector{N,UInt64}(bs.chunks)
# 	i1, i2 = Base.get_chunks_id(i)
# 	msk = ~(UInt64(1) << i2)
# 	@inbounds temp[i1] = (temp[i1] & msk) | (val << i2)
# 	AbstractBitString{L,N}(SVector{N,UInt64}(temp))
# end
#
#
#
# @inline function _setindex(bs::AbstractBitString{L,N}, val::AbstractVector{Bool}, itr) where {L,N}
# 	temp = MVector{N,UInt64}(bs.chunks)
# 	for (iv,ib) in enumerate(itr)
# 		i1, i2 = Base.get_chunks_id(ib)
# 		msk = ~(UInt64(1) << i2)
# 		@inbounds temp[i1] = (temp[i1] & msk) | (val[iv] << i2)
# 	end
# 	AbstractBitString{L,N}(SVector{N,UInt64}(temp))
# end
#
#
#

##  Iteration

function iterate(bs::AbstractBitString, i::Int=0)
    i >= length(bs) && return nothing
    (bs.chunks[Base._div64(i)+1] & (UInt64(1)<<Base._mod64(i)) != 0, i+1)
end

vcat(a::AbstractBitString) = a

@inline function vcat(a::AbstractBitString, b::AbstractBitString)
	lena = length(a)
	lenb = length(b)
	nch = nchunks(lena + lenb)
	na = length(a.chunks)
	nb = length(b.chunks)
	chunks = MVector{nch, UInt64}(undef)
	@inbounds for i = 1:na
		chunks[i] = a.chunks[i]
	end
	off = Base._mod64(length(a))
	@inbounds for j = 1:nb
		chunks[na + j-1]  |= b.chunks[j] << off
		if na + j <= nch
			chunks[na + j] = b.chunks[j] >> (64-off)
		end
	end
	return promote_type(typeof(a), typeof(b)){lena+lenb, nch}(chunks)
end
#
# totallength() = 0
# totallength(a::AbstractBitString) = length(a)
# totallength(a::AbstractBitString, others::AbstractBitString...) = length(a) + totallength(others...)
#
# # Without inlining it is SLOW
# @inline function vcat(args::AbstractBitString...)
# 	totlen = totallength(args...)
# 	nch = nchunks(totlen)
# 	chunks = MVector{nch, UInt64}(undef)
# 	a = args[1]
# 	for i = 1:length(a.chunks)
# 		chunks[i] = a.chunks[i]
# 	end
# 	len = length(a)
#
# 	for a in Base.tail(args)
# 		ich = nchunks(len)
# 		off = Base._mod64(len)
# 		for j = 1:length(a.chunks)
# 			chunks[ich - 1 + j]  |= a.chunks[j] << off
# 			if ich + j <= nch
# 				chunks[ich + j] = a.chunks[j] >> (64-off)
# 			end
# 		end
# 		len += length(a)
# 	end
# 	return AbstractBitString{totlen, nch}(chunks)
# end

# Basic arithmetic operations -- result in a non-dense array
-(b::AbstractBitString) = (-).(b)
*(b::AbstractBitString, x::Number) = b .* x
*(x::Number, b::AbstractBitString) = x .* b
/(b::AbstractBitString, x::Number) = b ./ x

# Efficient versions of map and map! for boolean functions.
map(::Union{typeof(~), typeof(!)}, a::AbstractBitString) = bit_map(~, a)
map(::Union{typeof(&), typeof(*), typeof(min)}, a::AbstractBitString, b::AbstractBitString) = bit_map(&, a, b)
map(::Union{typeof(|), typeof(max)}, a::AbstractBitString, b::AbstractBitString) = bit_map(|, a, b)
map(::Union{typeof(⊻), typeof(!=)}, a::AbstractBitString, b::AbstractBitString) = bit_map(⊻, a, b)
map(::typeof(*), a::AbstractBitString, b::AbstractBitString) = bit_map(*, a, b)
map(::typeof(==), a::AbstractBitString, b::AbstractBitString) = bit_map((x,y) -> ~xor(x,y), a, b)
map(::typeof(^), a::AbstractBitString, b::AbstractBitString) = bit_map((x,y) -> x | ~y, a, b)
map(::typeof(>), a::AbstractBitString, b::AbstractBitString) = bit_map((x,y) -> x & ~y, a, b)
map(::typeof(>=), a::AbstractBitString, b::AbstractBitString) = bit_map((x,y) -> x | ~y, a, b)
map(::typeof(<), a::AbstractBitString, b::AbstractBitString) = bit_map((x,y) -> y & ~x, a, b)
map(::typeof(<=), a::AbstractBitString, b::AbstractBitString) = bit_map((x,y) -> y | ~x, a, b)
map(::typeof(min), args...) where T = bit_map(&, args...)
map(::typeof(max), args...) = bit_map(|, args...)

# 1-ary functions
@inline function bit_map(f::F, a::AbstractBitString{L,N}) where {L,N,F}
	temp = MVector{N,UInt64}(undef)
#	isempty(A) && return AbstractBitString{0,1}()
	for i = 1:N
   	temp[i] = f(a.chunks[i])
   end
   temp[N] &= Base._msk_end(L)
   promote_type(typeof(a))(temp)
end

# 2-ary functions
function bit_map(f::F, a::AbstractBitString{L,N},  b::AbstractBitString{L,N}) where {F,L,N}
	#	isempty(A) && return AbstractBitString{0,1}()
	temp = MVector{N,UInt64}(undef)
	for i = 1:N
   	temp[i] = f(a.chunks[i], b.chunks[i])
   end
   temp[N] &= Base._msk_end(L)
   promote_type(typeof(a), typeof(b)){L,N}(temp)
end

# n-ary functions
@inline function bit_map(f::F, args::AbstractBitString{L,N}...) where {F,L,N}
	allequal(length(arg) for arg in args) || throw(DimensionMismatch("sizes of all arguments must match"))
	temp = MVector{N,UInt64}(undef)
	for i = 1:N
   	temp[i] = f((arg.chunks[i] for arg in args)...)
   end
   temp[N] &= Base._msk_end(L)
   AbstractBitString{L,N}(SVector{N}(temp))
end
#

map!(::Union{typeof(~), typeof(!)}, dest::MBitString, a::AbstractBitString) = bit_map!(~, dest, a)
map!(::Union{typeof(&), typeof(*), typeof(min)}, a::AbstractBitString, b::AbstractBitString) = bit_map!(&, dest, a, b)
map!(::Union{typeof(|), typeof(max)}, a::AbstractBitString, b::AbstractBitString) = bit_map!(|, dest, a, b)
map!(::typeof(⊻), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!(⊻, dest, a, b)
map!(::typeof(*), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!(*, dest,a, b)
map!(::typeof(!=), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!(xor, dest, a, b)
map!(::typeof(==), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!((x,y) -> ~xor(x,y), dest, a, b)
map!(::typeof(^), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!((x,y) -> x | ~y, dest, a, b)
map!(::typeof(>), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!((x,y) -> x & ~y, dest, a, b)
map!(::typeof(>=), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!((x,y) -> x | ~y, dest, a, b)
map!(::typeof(<), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!((x,y) -> y & ~x, dest, a, b)
map!(::typeof(<=), dest::MBitString, a::AbstractBitString, b::AbstractBitString) = bit_map!((x,y) -> y | ~x, dest, a, b)
map!(::typeof(min), dest::MBitString, args::AbstractBitString...) where T = bit_map!(&, dest, args...)
map!(::typeof(max), dest::MBitString, args::AbstractBitString...) = bit_map!(|, dest, args...)

@inline function bit_map!(f, dest, args...)
	checklengths(args...)
	unsafe_bit_map!(f, dest, args...)
end

checklengths(a) = nothing

@inline function checklengths(a, b)
	length(a) == length(b) || throw(DimensionMismatch("sizes of A and B must match"))
	nothing
end

@inline function checklengths(args...)
	allequal(length(arg) for arg in args) || throw(DimensionMismatch("all arguments must have the same length"))
	nothing
end

# 1-ary functions
function unsafe_bit_map!(f::F, dest::MBitString{L,N}, A::AbstractBitString{L,N}) where {F,L,N}
	destc = dest.chunks
	for i = 1:N
   	destc[i] = f(A.chunks[i])
   end
	destc[N] &= Base._msk_end(L)
	dest
end

# 2-ary functions
@inline function unsafe_bit_map!(f::F, dest::MBitString{L,N}, A::AbstractBitString{L,N}, B::AbstractBitString{L,N}) where {L,N,F}
	destc = dest.chunks
	@inbounds for i = 1:N
   	destc[i] = f(A.chunks[i], B.chunks[i])
   end
   @inbounds destc[N] &= Base._msk_end(L)
	dest
end

# n-ary functions
function unsafe_bit_map!(f::F, dest::MBitString{L,N}, args::AbstractBitString{L,N}...) where {L,N,F}
	destc = dest.chunks
	@inbounds for i = 1:N
   	destc[i] = f((arg.chunks[i] for arg in args)...)
   end
	@inbounds destc[N] &= Base._msk_end(L)
	dest
end

# ## Broadcasting
#
# # broadcastable(b::AbstractBitString) = b
# #
# # # associative bitwise functions
# # const BitwiseFun = Union{typeof(&), typeof(|), typeof(xor), typeof(~)}
# #
# #
# # # Create a custom style so that we can dispatch to our custom implementation
# # struct BitStringStyle{L,N} <: Broadcast.AbstractArrayStyle{1} end
# # BroadcastStyle(::Type{AbstractBitString{L,N}}) where {L,N} = BitStringStyle{L,N}()
# # BroadcastStyle(::BitStringStyle, t::Broadcast.DefaultArrayStyle{N}) where {N} = Broadcast.DefaultArrayStyle{max(1,N)}()
# #
# # # fallback -- default to ArrayStyle
# # broadcasted(bc::BitStringStyle, f::F, args...) where {F} = broadcasted(Broadcast.DefaultArrayStyle{1}(), f, args...)
# #
# # broadcasted(bc::BitStringStyle, f::BitwiseFun, args...) = bs_broadcast(f, args...)
# # broadcasted(bc::BitStringStyle, ::typeof(min), args...) = bs_broadcast(&, args...)
# # broadcasted(bc::BitStringStyle, ::typeof(max), args...) = bs_broadcast(|, args...)
# # broadcasted(bc::BitStringStyle, ::typeof(!=), a, b) = bs_broadcast(xor, a, b)
# # broadcasted(bc::BitStringStyle, ::typeof(==), a, b) = bs_broadcast((x,y) -> ~xor(x,y), a, b)
# # broadcasted(bc::BitStringStyle, ::typeof(^),  a, b) = bs_broadcast((x,y) -> x | ~y, a, b)
# # broadcasted(bc::BitStringStyle, ::typeof(>), a, b) = bs_broadcast((x,y) -> x & ~y, a, b)
# # broadcasted(bc::BitStringStyle, ::typeof(>=), a, b) = bs_broadcast((x,y) -> x | ~y, a, b)
# # broadcasted(bc::BitStringStyle, ::typeof(<), a, b) = bs_broadcast((x,y) -> y & ~x, a, b)
# # broadcasted(bc::BitStringStyle, ::typeof(<=), a, b) = bs_broadcast((x,y) -> y | ~x, a, b)
# #
# # # Custom implementation.  For N=1, should take about 2ns (2μs per 1000)
# # function bs_broadcast(f::F, a::AbstractBitString{L,N}, b::AbstractBitString{L,N}) where {F,L,N}
# # 	mv = MVector{N,UInt64}(undef)
# # 	for i in 1:N
# # 		mv[i] = f(a.chunks[i], b.chunks[i])
# # 	end
# # 	AbstractBitString{L,N}(Tuple(mv))
# # end

## Linear algebra

@inline function dot(x::AbstractBitString, y::AbstractBitString)
    # simplest way to mimic Array dot behavior
    length(x) == length(y) || throw(DimensionMismatch())
    s = 0
    xc = x.chunks
    yc = y.chunks
    @inbounds for i = 1:length(xc)
        s += count_ones(xc[i] & yc[i])
    end
    s
end

# Just like dot, bit with xor instead of &
function hammdist(a::AbstractBitString, b::AbstractBitString)
	# simplest way to mimic Array dot behavior
	length(x) == length(y) || throw(DimensionMismatch())
	s = 0
	xc = x.chunks
	yc = y.chunks
	@inbounds for i = 1:length(xc)
		 s += count_ones(xc[i] ⊻ yc[i])
	end
	s
end

sum(b::AbstractBitString) = count(b)

function count(b::AbstractBitString)
	s = 0
	chk = b.chunks
	@inbounds for i = 1:length(chk)
		 s += count_ones(chk[i])
	end
	s
end

function parity(b::AbstractBitString)
	s = false
	chk = b.chunks
	@inbounds for i = 1:length(chk)
		 s ⊻= isodd(count_ones(chk[i]))
	end
	s
end

end	# module


# offset.jl
# https://github.com/JuliaPlots/Plots.jl/issues/2410

using MIRTjim: jim
using OffsetArrays
using Test: @testset

include("isplot.jl")

@testset "unit" begin
    z = OffsetArrays.centered(ones(7) * (1:4)')

    p = jim(z, "offset")
    @isplot p
end


"""
    MapFun <: AugmentorBase.Operation

Description
--------------

Maps the given function over all individual array elements.

This means that the given function is called with an individual
elements and is expected to return a transformed element that
should take the original's place. This further implies that the
function is expected to be unary. It is encouraged that the
function should be consistent with its return type and
type-stable.

Usage
--------------

    MapFun(fun)

Arguments
--------------

- **`fun`** : The unary function that should be mapped over all
    individual array elements.

See also
--------------

[`AggregateThenMapFun`](@ref), [`ConvertEltype`](@ref), [`augment`](@ref)

Examples
--------------

```julia
using Augmentor, ColorTypes
input = testpattern()

# subtract the constant RGBA value from each pixel
augment(input, MapFun(px -> px - RGBA(0.5, 0.3, 0.7, 0.0)))

# separate channels to scale each numeric element by a constant value
pl = SplitChannels() |> MapFun(el -> el * 0.5) |> CombineChannels(RGBA)
augment(input, pl)
```
"""
struct MapFun{T} <: Operation
    fun::T
end

@inline supports_lazy(::Type{<:MapFun}) = true

function applyeager(op::MapFun, input::AbstractArray, param)
    maybe_copy(map(op.fun, input))
end

function applylazy(op::MapFun, input::AbstractArray, param)
    mappedarray(op.fun, input)
end

function showconstruction(io::IO, op::MapFun)
    print(io, nameof(typeof(op)), '(', op.fun, ')')
end

function Base.show(io::IO, op::MapFun)
    if get(io, :compact, false)
        print(io, "Map function \"", op.fun, "\" over image")
    else
        print(io, "AugmentorBase.")
        showconstruction(io, op)
    end
end

# --------------------------------------------------------------------

"""
    AggregateThenMapFun <: AugmentorBase.Operation

Description
--------------

Compute some aggregated value of the current image using the
given function `aggfun`, and map that value over the current
image using the given function `mapfun`.

This is particularly useful for achieving effects such as
per-image normalization.

Usage
--------------

    AggregateThenMapFun(aggfun, mapfun)

Arguments
--------------

- **`aggfun`** : A function that takes the whole current image as
    input and which result will also be passed to `mapfun`. It
    should have a signature of `input -> agg`, where `input` will the
    the current image. What type and value `agg` should be is up
    to the user.

- **`mapfun`** : The binary function that should be mapped over
    all individual array elements. It should have a signature of
    `(px, agg) -> new_px` where `px` is a single element of the
    current image, and `agg` is the output of `aggfun`.

See also
--------------

[`MapFun`](@ref), [`ConvertEltype`](@ref), [`augment`](@ref)

Examples
--------------

```julia
using Augmentor
input = testpattern()

# subtract the average RGB value of the current image
augment(input, AggregateThenMapFun(input -> mean(input), (px, agg) -> px - agg))
```
"""
struct AggregateThenMapFun{A,M} <: Operation
    aggfun::A
    mapfun::M
end

@inline supports_lazy(::Type{<:AggregateThenMapFun}) = true

function applyeager(op::AggregateThenMapFun, input::AbstractArray, param)
    agg = op.aggfun(input)
    maybe_copy(map(x -> op.mapfun(x, agg), input))
end

function applylazy(op::AggregateThenMapFun, input::AbstractArray, param)
    agg = op.aggfun(input)
    mappedarray(x -> op.mapfun(x, agg), input)
end

function showconstruction(io::IO, op::AggregateThenMapFun)
    print(io, nameof(typeof(op)), '(', op.aggfun, ", ", op.mapfun, ')')
end

function Base.show(io::IO, op::AggregateThenMapFun)
    if get(io, :compact, false)
        print(io, "Map result of \"", op.aggfun, "\" using \"", op.mapfun, "\" over image")
    else
        print(io, "AugmentorBase.")
        showconstruction(io, op)
    end
end


@testset "Time grid generation" begin
    @test isempty( MP.generateTimeGrid( mpSim, -12 ) )
    @test MP.generateTimeGrid( mpSim, 12 ) == [0.0]
    run( mpSim.sim )
    @test MP.generateTimeGrid( mpSim, 12 ) == collect( 0.0:12.0:300.0 )
end  # @testset "Time grid generation"


###############################################################################
#
#   fmpq_mat.jl : Flint matrices over the rationals
#
###############################################################################

export fmpq_mat, FmpqMatSpace, gso, hilbert

###############################################################################
#
#   Data type and parent object methods
#
###############################################################################

elem_type(::Type{FmpqMatSpace}) = fmpq_mat

parent_type(::Type{fmpq_mat}) = FmpqMatSpace

base_ring(a::FmpqMatSpace) = a.base_ring

base_ring(a::fmpq_mat) = a.base_ring

parent(a::fmpq_mat, cached::Bool = true) =
      FmpqMatSpace(rows(a), cols(a), cached)

function check_parent(a::fmpq_mat, b::fmpq_mat)
   (rows(a) != rows(b) || cols(a) != cols(b) || base_ring(a) != base_ring(b)) &&
      error("Incompatible matrices")
end

###############################################################################
#
#   Similar
#
###############################################################################

function similar(x::fmpq_mat)
   z = fmpq_mat(rows(x), cols(x))
   z.base_ring = x.base_ring
   return z
end

function similar(x::fmpq_mat, r::Int, c::Int)
   z = fmpq_mat(r, c)
   z.base_ring = x.base_ring
   return z
end

###############################################################################
#
#   Windows - handle with care!!!
#
###############################################################################

function Base.view(x::fmpq_mat, r1::Int, c1::Int, r2::Int, c2::Int)
  Generic._checkbounds(x, r1, c1)
  Generic._checkbounds(x, r2, c2)
  (r1 > r2 || c1 > c2) && error("Invalid parameters")
  b = fmpq_mat()
  b.base_ring = x.base_ring
  ccall((:fmpq_mat_window_init, :libflint), Void,
        (Ref{fmpq_mat}, Ref{fmpq_mat}, Int, Int, Int, Int),
            b, x, r1 - 1, c1 - 1, r2, c2)
  finalizer(b, _fmpq_mat_window_clear_fn)
  return b
end

function Base.view(x::fmpq_mat, r::UnitRange{Int}, c::UnitRange{Int})
  return Base.view(x, r.start, c.start, r.stop, c.stop)
end

function _fmpq_mat_window_clear_fn(a::fmpq_mat)
   ccall((:fmpq_mat_window_clear, :libflint), Void, (Ref{fmpq_mat},), a)
end

size(x::fmpq_mat) = tuple(rows(x), cols(x))

size(t::fmpq_mat, d) = d <= 2 ? size(t)[d] : 1

###############################################################################
#
#   Basic manipulation
#
###############################################################################

function getindex!(v::fmpq, a::fmpq_mat, r::Int, c::Int)
   z = ccall((:fmpq_mat_entry, :libflint), Ref{fmpq},
             (Ref{fmpq_mat}, Int, Int), a, r - 1, c - 1)
   ccall((:fmpq_set, :libflint), Void, (Ref{fmpq}, Ref{fmpq}), v, z)
end

@inline function getindex(a::fmpq_mat, r::Int, c::Int)
   @boundscheck Generic._checkbounds(a, r, c)
   v = fmpq()
   z = ccall((:fmpq_mat_entry, :libflint), Ref{fmpq},
             (Ref{fmpq_mat}, Int, Int), a, r - 1, c - 1)
   ccall((:fmpq_set, :libflint), Void, (Ref{fmpq}, Ref{fmpq}), v, z)
   return v
end

@inline function setindex!(a::fmpq_mat, d::fmpz, r::Int, c::Int)
   @boundscheck Generic._checkbounds(a, r, c)
   z = ccall((:fmpq_mat_entry_num, :libflint), Ref{fmpz},
             (Ref{fmpq_mat}, Int, Int), a, r - 1, c - 1)
   ccall((:fmpz_set, :libflint), Void, (Ref{fmpz}, Ref{fmpz}), z, d)
   z = ccall((:fmpq_mat_entry_den, :libflint), Ref{fmpz},
             (Ref{fmpq_mat}, Int, Int), a, r - 1, c - 1)
   ccall((:fmpz_set_si, :libflint), Void, (Ref{fmpz}, Int), z, 1)
end

@inline function setindex!(a::fmpq_mat, d::fmpq, r::Int, c::Int)
   @boundscheck Generic._checkbounds(a, r, c)
   z = ccall((:fmpq_mat_entry, :libflint), Ref{fmpq},
             (Ref{fmpq_mat}, Int, Int), a, r - 1, c - 1)
   ccall((:fmpq_set, :libflint), Void, (Ref{fmpq}, Ref{fmpq}), z, d)
end

Base.@propagate_inbounds setindex!(a::fmpq_mat, d::Integer,
                                 r::Int, c::Int) =
         setindex!(a, fmpq(d), r, c)

@inline function setindex!(a::fmpq_mat, d::Int, r::Int, c::Int)
   @boundscheck Generic._checkbounds(a, r, c)
   z = ccall((:fmpq_mat_entry, :libflint), Ref{fmpq},
             (Ref{fmpq_mat}, Int, Int), a, r - 1, c - 1)
   ccall((:fmpq_set_si, :libflint), Void, (Ref{fmpq}, Int, Int), z, d, 1)
end

Base.@propagate_inbounds setindex!(a::fmpq_mat, d::Rational,
                                 r::Int, c::Int) =
         setindex!(a, fmpq(d), r, c)

rows(a::fmpq_mat) = a.r

cols(a::fmpq_mat) = a.c

zero(a::FmpqMatSpace) = a()

one(a::FmpqMatSpace) = a(1)

iszero(a::fmpq_mat) = ccall((:fmpq_mat_is_zero, :libflint), Bool,
                            (Ref{fmpq_mat},), a)

isone(a::fmpq_mat) = ccall((:fmpq_mat_is_one, :libflint), Bool,
                           (Ref{fmpq_mat},), a)

function deepcopy_internal(d::fmpq_mat, dict::ObjectIdDict)
   z = fmpq_mat(d)
   z.base_ring = d.base_ring
   return z
end

###############################################################################
#
#   Canonicalisation
#
###############################################################################

canonical_unit(a::fmpq_mat) = canonical_unit(a[1, 1])

###############################################################################
#
#   AbstractString I/O
#
###############################################################################

function show(io::IO, a::FmpqMatSpace)
   print(io, "Matrix Space of ")
   print(io, a.rows, " rows and ", a.cols, " columns over ")
   print(io, "Rational Field")
end

function show(io::IO, a::fmpq_mat)
   r = rows(a)
   c = cols(a)
   for i = 1:r
      print(io, "[")
      for j = 1:c
         print(io, a[i, j])
         if j != c
            print(io, " ")
         end
      end
      print(io, "]")
      if i != r
         println(io, "")
      end
   end
end

show_minus_one(::Type{fmpq_mat}) = show_minus_one(fmpq)

###############################################################################
#
#   Unary operations
#
###############################################################################

function -(x::fmpq_mat)
   z = similar(x)
   ccall((:fmpq_mat_neg, :libflint), Void,
         (Ref{fmpq_mat}, Ref{fmpq_mat}), z, x)
   return z
end

###############################################################################
#
#   transpose
#
###############################################################################

function transpose(x::fmpq_mat)
   z = similar(x, cols(x), rows(x))
   ccall((:fmpq_mat_transpose, :libflint), Void,
         (Ref{fmpq_mat}, Ref{fmpq_mat}), z, x)
   return z
end

###############################################################################
#
#   Binary operations
#
###############################################################################

function +(x::fmpq_mat, y::fmpq_mat)
   check_parent(x, y)
   z = similar(x)
   ccall((:fmpq_mat_add, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat},  Ref{fmpq_mat}),
               z, x, y)
   return z
end

function -(x::fmpq_mat, y::fmpq_mat)
   check_parent(x, y)
   z = similar(x)
   ccall((:fmpq_mat_sub, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat},  Ref{fmpq_mat}),
               z, x, y)
   return z
end

function *(x::fmpq_mat, y::fmpq_mat)
   cols(x) != rows(y) && error("Incompatible matrix dimensions")
   z = similar(x, rows(x), cols(y))
   ccall((:fmpq_mat_mul, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat},  Ref{fmpq_mat}),
               z, x, y)
   return z
end

###############################################################################
#
#   Ad hoc binary operators
#
###############################################################################

function *(x::fmpz, y::fmpq_mat)
   z = similar(y)
   ccall((:fmpq_mat_scalar_mul_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpz}), z, y, x)
   return z
end

function *(x::fmpq, y::fmpq_mat)
   z = similar(y)
   ccall((:fmpq_mat_scalar_mul_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq}), z, y, numerator(x))
   ccall((:fmpq_mat_scalar_div_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq}), z, z, denominator(x))
   return z
end

*(x::fmpq_mat, y::fmpq) = y*x

*(x::fmpq_mat, y::fmpz) = y*x

*(x::Integer, y::fmpq_mat) = fmpz(x)*y

*(x::fmpq_mat, y::Integer) = fmpz(y)*x

*(x::Rational, y::fmpq_mat) = fmpq(x)*y

*(x::fmpq_mat, y::Rational) = fmpq(y)*x

for T in [Integer, fmpz, fmpq]
   @eval begin
      function +(x::fmpq_mat, y::$T)
         z = deepcopy(x)
         for i = 1:min(rows(x), cols(x))
            z[i, i] += y
         end
         return z
      end

      +(x::$T, y::fmpq_mat) = y + x

      function -(x::fmpq_mat, y::$T)
         z = deepcopy(x)
         for i = 1:min(rows(x), cols(x))
            z[i, i] -= y
         end
         return z
      end

      function -(x::$T, y::fmpq_mat)
         z = -y
         for i = 1:min(rows(y), cols(y))
            z[i, i] += x
         end
         return z
      end
   end
end

function +(x::fmpq_mat, y::Rational)
   z = deepcopy(x)
   for i = 1:min(rows(x), cols(x))
      z[i, i] += y
   end
   return z
end

+(x::Rational, y::fmpq_mat) = y + x

function -(x::fmpq_mat, y::Rational)
   z = deepcopy(x)
   for i = 1:min(rows(x), cols(x))
      z[i, i] -= y
   end
   return z
end

function -(x::Rational, y::fmpq_mat)
   z = -y
   for i = 1:min(rows(y), cols(y))
      z[i, i] += x
   end
   return z
end

###############################################################################
#
#   Comparisons
#
###############################################################################

function ==(x::fmpq_mat, y::fmpq_mat)
   check_parent(x, y)
   ccall((:fmpq_mat_equal, :libflint), Bool,
                                       (Ref{fmpq_mat}, Ref{fmpq_mat}), x, y)
end

###############################################################################
#
#   Ad hoc comparisons
#
###############################################################################

function ==(x::fmpq_mat, y::Integer)
   for i = 1:min(rows(x), cols(x))
      if x[i, i] != y
         return false
      end
   end
   for i = 1:rows(x)
      for j = 1:cols(x)
         if i != j && x[i, j] != 0
            return false
         end
      end
   end
   return true
end

==(x::Integer, y::fmpq_mat) = y == x

==(x::fmpq_mat, y::Rational{T}) where T <: Union{Int, BigInt} = x == fmpq(y)

==(x::Rational{T}, y::fmpq_mat) where T <: Union{Int, BigInt} = y == x

###############################################################################
#
#   Inversion
#
###############################################################################

function inv(x::fmpq_mat)
   z = similar(x)
   success = ccall((:fmpq_mat_inv, :libflint), Cint,
         (Ref{fmpq_mat}, Ref{fmpq_mat}), z, x)
   success == 0 && error("Matrix not invertible")
   return z
end

###############################################################################
#
#   Exact division
#
###############################################################################

function divexact(x::fmpq_mat, y::fmpq_mat)
   cols(x) != cols(y) && error("Incompatible matrix dimensions")
   x*inv(y)
end

###############################################################################
#
#   Ad hoc exact division
#
###############################################################################

function divexact(x::fmpq_mat, y::fmpq)
   z = similar(x)
   ccall((:fmpq_mat_scalar_div_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpz}), z, x, numerator(y))
   ccall((:fmpq_mat_scalar_mul_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpz}), z, z, denominator(y))
   return z
end

function divexact(x::fmpq_mat, y::fmpz)
   z = similar(x)
   ccall((:fmpq_mat_scalar_div_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpz}), z, x, y)
   return z
end

divexact(x::fmpq_mat, y::Integer) = divexact(x, fmpz(y))

divexact(x::fmpq_mat, y::Rational{T}) where T <: Union{Int, BigInt} = divexact(x, fmpq(y))

###############################################################################
#
#   Characteristic polynomial
#
###############################################################################

function charpoly(R::FmpqPolyRing, x::fmpq_mat)
   rows(x) != cols(x) && error("Non-square")
   z = R()
   ccall((:fmpq_mat_charpoly, :libflint), Void,
                (Ref{fmpq_poly}, Ref{fmpq_mat}), z, x)
   return z
end

###############################################################################
#
#   Minimal polynomial
#
###############################################################################

function minpoly(R::FmpqPolyRing, x::fmpq_mat)
   rows(x) != cols(x) && error("Non-square")
   z = R()
   ccall((:fmpq_mat_minpoly, :libflint), Void,
                (Ref{fmpq_poly}, Ref{fmpq_mat}), z, x)
   return z
end

###############################################################################
#
#   Determinant
#
###############################################################################

function det(x::fmpq_mat)
   rows(x) != cols(x) && error("Non-square matrix")
   z = fmpq()
   ccall((:fmpq_mat_det, :libflint), Void,
                (Ref{fmpq}, Ref{fmpq_mat}), z, x)
   return z
end

###############################################################################
#
#   Gram-Schmidt orthogonalisation
#
###############################################################################

doc"""
    gso(x::fmpq_mat)
> Return the Gram-Schmidt Orthogonalisation of the matrix $x$.
"""
function gso(x::fmpq_mat)
   z = similar(x)
   ccall((:fmpq_mat_gso, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}), z, x)
   return z
end

###############################################################################
#
#   Hilbert matrix
#
###############################################################################

doc"""
    hilbert(R::FmpqMatSpace)
> Return the Hilbert matrix in the given matrix space. This is the matrix with
> entries $H_{i,j} = 1/(i + j - 1)$.
"""
function hilbert(R::FmpqMatSpace)
   z = R()
   ccall((:fmpq_mat_hilbert_matrix, :libflint), Bool,
                   (Ref{fmpq_mat},), z)
   return z
end

###############################################################################
#
#   Rank
#
###############################################################################

function rank(x::fmpq_mat)
   z = similar(x)
   r = ccall((:fmpq_mat_rref, :libflint), Int,
         (Ref{fmpq_mat}, Ref{fmpq_mat}), z, x)
   return r
end

###############################################################################
#
#   Reduced row echelon form
#
###############################################################################

function rref(x::fmpq_mat)
   z = similar(x)
   r = ccall((:fmpq_mat_rref, :libflint), Int,
         (Ref{fmpq_mat}, Ref{fmpq_mat}), z, x)
   return r, z
end

###############################################################################
#
#   Linear solving
#
###############################################################################

function solve(a::fmpq_mat, b::fmpq_mat)
   rows(a) != cols(a) && error("Not a square matrix in solve")
   rows(b) != rows(a) && error("Incompatible dimensions in solve")
   z = similar(b)
   nonsing = ccall((:fmpq_mat_solve_fraction_free, :libflint), Bool,
      (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq_mat}), z, a, b)
   !nonsing && error("Singular matrix in solve")
   return z
end

doc"""
    solve_dixon(a::fmpq_mat, b::fmpq_mat)
> Solve $ax = b$ by clearing denominators and using Dixon's algorithm. This is
> usually faster for large systems.
"""
function solve_dixon(a::fmpq_mat, b::fmpq_mat)
   rows(a) != cols(a) && error("Not a square matrix in solve")
   rows(b) != rows(a) && error("Incompatible dimensions in solve")
   z = similar(b)
   nonsing = ccall((:fmpq_mat_solve_dixon, :libflint), Bool,
      (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq_mat}), z, a, b)
   !nonsing && error("Singular matrix in solve")
   return z
end

###############################################################################
#
#   Trace
#
###############################################################################

function trace(x::fmpq_mat)
   rows(x) != cols(x) && error("Not a square matrix in trace")
   d = fmpq()
   ccall((:fmpq_mat_trace, :libflint), Void,
                (Ref{fmpq}, Ref{fmpq_mat}), d, x)
   return d
end

###############################################################################
#
#   Concatenation
#
###############################################################################

function hcat(a::fmpq_mat, b::fmpq_mat)
  rows(a) != rows(b) && error("Incompatible number of rows in hcat")
  c = similar(a, rows(a), cols(a) + cols(b))
  ccall((:fmpq_mat_concat_horizontal, :libflint), Void,
        (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq_mat}), c, a, b)
  return c
end

function vcat(a::fmpq_mat, b::fmpq_mat)
  cols(a) != cols(b) && error("Incompatible number of columns in vcat")
  c = similar(a, rows(a) + rows(b), cols(a))
  ccall((:fmpq_mat_concat_vertical, :libflint), Void,
        (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq_mat}), c, a, b)
  return c
end

###############################################################################
#
#   Similarity
#
###############################################################################

function similarity!(z::fmpq_mat, r::Int, d::fmpq)
   ccall((:fmpq_mat_similarity, :libflint), Void,
         (Ref{fmpq_mat}, Int, Ref{fmpq}), z, r - 1, d)
end

###############################################################################
#
#   Unsafe functions
#
###############################################################################

function mul!(z::fmpq_mat, x::fmpq_mat, y::fmpq_mat)
   ccall((:fmpq_mat_mul, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq_mat}), z, x, y)
   return z
end

function mul!(y::fmpq_mat, x::Int)
   ccall((:fmpq_mat_scalar_mul_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq}), y, y, fmpz(x))
   return y
end

function mul!(y::fmpq_mat, x::fmpz)
   ccall((:fmpq_mat_scalar_mul_fmpz, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpz}), y, y, x)
   return y
end

function addeq!(z::fmpq_mat, x::fmpq_mat)
   ccall((:fmpq_mat_add, :libflint), Void,
                (Ref{fmpq_mat}, Ref{fmpq_mat}, Ref{fmpq_mat}), z, z, x)
   return z
end

function zero!(z::fmpq_mat)
   ccall((:fmpq_mat_zero, :libflint), Void,
                (Ref{fmpq_mat},), z)
   return z
end

###############################################################################
#
#   Parent object call overloads
#
###############################################################################

function (a::FmpqMatSpace)()
   z = fmpq_mat(a.rows, a.cols)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{fmpq, 2})
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, arr)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{fmpz, 2})
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, arr)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{T, 2}) where {T <: Integer}
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, arr)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{Rational{T}, 2}) where {T <: Integer}
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, map(fmpq, arr))
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{fmpq, 1})
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, arr)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{fmpz, 1})
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, arr)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{T, 1}) where {T <: Integer}
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, arr)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(arr::Array{Rational{T}, 1}) where {T <: Integer}
   _check_dim(a.rows, a.cols, arr)
   z = fmpq_mat(a.rows, a.cols, map(fmpq, arr))
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(d::fmpq)
   z = fmpq_mat(a.rows, a.cols, d)
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(d::fmpz)
   z = fmpq_mat(a.rows, a.cols, fmpq(d))
   z.base_ring = a.base_ring
   return z
end

function (a::FmpqMatSpace)(d::Integer)
   z = fmpq_mat(a.rows, a.cols, fmpq(d))
   z.base_ring = a.base_ring
   return z
end

(a::FmpqMatSpace)(d::Rational) = a(fmpq(d))

function (a::FmpqMatSpace)(M::fmpz_mat)
   (a.cols == cols(M) && a.rows == rows(M)) || error("wrong matrix dimension")
   z = a()
   ccall((:fmpq_mat_set_fmpz_mat, :libflint), Void, (Ref{fmpq_mat}, Ref{fmpz_mat}), z, M)
   return z
end

(a::FmpqMatSpace)(d::fmpq_mat) = d

###############################################################################
#
#   Promotions
#
###############################################################################

promote_rule(::Type{fmpq_mat}, ::Type{T}) where {T <: Integer} = fmpq_mat

promote_rule(::Type{fmpq_mat}, ::Type{fmpq}) = fmpq_mat

promote_rule(::Type{fmpq_mat}, ::Type{fmpz}) = fmpq_mat

promote_rule(::Type{fmpq_mat}, ::Type{Rational{T}}) where T <: Union{Int, BigInt} = fmpq_mat

###############################################################################
#
#   Matrix constructor
#
###############################################################################

function matrix(R::FlintRationalField, arr::Array{fmpq, 2})
   z = fmpq_mat(size(arr, 1), size(arr, 2), arr)
   z.base_ring = FlintQQ
   return z
end

function matrix(R::FlintRationalField, arr::Array{<: Union{fmpz, Int, BigInt}, 2})
   z = fmpq_mat(size(arr, 1), size(arr, 2), arr)
   z.base_ring = FlintQQ
   return z
end

function matrix(R::FlintRationalField, arr::Array{Rational{T}, 2}) where {T <: Integer}
   z = fmpq_mat(size(arr, 1), size(arr, 2), map(fmpq, arr))
   z.base_ring = FlintQQ
   return z
end

function matrix(R::FlintRationalField, r::Int, c::Int, arr::Array{fmpq, 1})
   _check_dim(r, c, arr)
   z = fmpq_mat(r, c, arr)
   z.base_ring = FlintQQ
   return z
end

function matrix(R::FlintRationalField, r::Int, c::Int, arr::Array{<: Union{fmpz, Int, BigInt}, 1})
   _check_dim(r, c, arr)
   z = fmpq_mat(r, c, arr)
   z.base_ring = FlintQQ
   return z
end

function matrix(R::FlintRationalField, r::Int, c::Int, arr::Array{Rational{T}, 1}) where {T <: Union{fmpz, Int, BigInt}}
   _check_dim(r, c, arr)
   z = fmpq_mat(r, c, map(fmpq, arr))
   z.base_ring = FlintQQ
   return z
end

###############################################################################
#
#  Zero matrix
#
###############################################################################

function zero_matrix(R::FlintRationalField, r::Int, c::Int)
   z = fmpq_mat(r, c)
   z.base_ring = FlintQQ
   return z
end

###############################################################################
#
#  Identity matrix
#
###############################################################################

function identity_matrix(R::FlintRationalField, n::Int)
   z = fmpq_mat(n, n)
   ccall((:fmpq_mat_one, :libflint), Void, (Ref{fmpq_mat}, ), z)
   z.base_ring = FlintQQ
   return z
end

###############################################################################
#
#   MatrixSpace constructor
#
###############################################################################

function MatrixSpace(R::FlintRationalField, r::Int, c::Int; cached = true)
   return FmpqMatSpace(r, c, cached)
end


using CSV
using DataFrames

function read_congress(fname)
    party_map = Dict{Int64,Int64}()
    name_map = Dict{Int64,String}()
    house = CSV.File(fname) |> DataFrame
    for grp in groupby(house, :id)
        party = unique(grp[!, :party])
        if length(party) != 1
            println("more than one party affiliation")
            @show grp
            error(0)
        end
        party = party[1]
        id = grp[1, :id]
        if party != 100 && party != 200
            if id == 13100
                # James L. Buckley (conservative, map to republican)
                party = 200
            elseif id == 29147
                # Bernie Sanders (independent, map to democratic)
                party = 100
            elseif id == 10802
                # Harry F. Byrd, Jr. (independent democrat, map to democrat)
                party = 100
            elseif id == 94240
                # Jim Jeffords after he left Republicans. This is Probably the
                # most complicated, but he mostly voted with the democrats.
                party = 100
            else
                println("nontraditional party affiliation")
                @show grp
                error(0)
            end
        else
            party_map[id] = party
            name_map[id] = grp[1, :name]
        end
    end
    return party_map, name_map
end

function get_members()
    spm, snm = read_congress("senate.csv")
    hpm, hnm = read_congress("house.csv")
    for key in keys(spm) ∩ keys(hpm)
        if spm[key] != hpm[key]
            println("different parties...")
            @show key
            error(0)
        end
    end

    # merge party maps
    pm = Dict{Int64,Int64}()
    for (key, val) in spm
        pm[key] = val ÷ 100
    end
    for (key, val) in hpm
        pm[key] = val ÷ 100
    end

    # merge name maps
    nm = Dict{Int64,String}()
    for (key, val) in snm
        nm[key] = val
    end
    for (key, val) in hnm
        nm[key] = val
    end

    return pm, nm
end

# Get out hyperedges of sponsor + cosponsors, partitioned by bill type
function bill_hedges(pm)
    sponsors = readlines("sponsors.txt")
    all_cosponsors = readlines("cosponsors.txt")
    bill_types = [String(bill[1:2]) for bill in readlines("bills.txt")]
    hedges = Dict{String, Vector{Vector{Int64}}}()
    for bill_type in unique(bill_types)
        hedges[bill_type] = Vector{Vector{Int64}}()
    end

    for (sponsor, cosponsors, bill_type) in zip(sponsors, all_cosponsors, bill_types)
        bill_sponsors = [sponsor]
        for cosponsor in split(cosponsors)
            push!(bill_sponsors, cosponsor)
        end
        bill_sponsors = filter(sp -> sp != "NA", bill_sponsors)
        bill_sponsors = map(sp -> parse(Int64, sp), bill_sponsors)
        bill_sponsors = filter(sp -> haskey(pm, sp), bill_sponsors)
        if length(bill_sponsors) > 1
            push!(hedges[bill_type], bill_sponsors)
        end
    end

    return hedges
end

# Map hypergraph node IDs to 1...N
function mapped_hedges(hedges)
    id_map = Dict{Int64,Int64}()
    inv_id_map = Dict{Int64,Int64}()
    function get_key(v)
        if !haskey(id_map, v)
            val = length(id_map) + 1
            id_map[v] = val
            inv_id_map[val] = v
        end
        return id_map[v]
    end

    new_hedges = []
    for hedge in hedges
        new_hedge = [get_key(v) for v in hedge]
        push!(new_hedges, new_hedge)
    end

    return new_hedges, inv_id_map
end

function write_dataset(hedges, pm, nm, basename)
    new_hedges, inv_id_map = mapped_hedges(hedges)

    dir = string("$(basename)-congress-bills")
    if !isdir(dir)
        mkdir(dir)
    end

    # hyperedge data
    open("$dir/hyperedges-$dir.txt", "w") do f
        for hedge in new_hedges
            write(f, join(hedge, ','))
            write(f, "\n")
        end
    end

    # names of the integer labels
    open("$dir/label-names-$dir.txt", "w") do f
        write(f, "Democrat\n")
        write(f, "Republican\n")
    end

    # labels for each node
    open("$dir/node-labels-$dir.txt", "w") do f
        for i = 1:length(inv_id_map)
            party = pm[inv_id_map[i]]
            write(f, "$(party)\n")
        end
    end

    # names of the nodes
    open("$dir/node-names-$dir.txt", "w") do f
        for i = 1:length(inv_id_map)
            name = nm[inv_id_map[i]]
            write(f, "$(name)\n")
        end
    end
end

function main()
    pm, nm = get_members()
    all_hedges = bill_hedges(pm)
    for (bt, hedges) in all_hedges
        # Ignore house ammendments (small)
        if bt != "HZ"
            write_dataset(hedges, pm, nm, bt)
        end
    end
end


# This file was generated, do not modify it. # hide
Booster = @load EvoTreeClassifier pkg=EvoTrees


"""
     One()
The Differential which is the multiplicative identity.
Basically, this represents `1`.
"""
struct One <: AbstractDifferential end

extern(x::One) = true  # true is a strong 1.

Base.Broadcast.broadcastable(::One) = Ref(One())
Base.Broadcast.broadcasted(::Type{One}) = One()

Base.iterate(x::One) = (x, nothing)
Base.iterate(::One, ::Any) = nothing


"""
	FunctionCell{T,S,U,V,W}(x::T, y::S, f::U, fargs::V<:Tuple, fkwargs::W<:Tuple, layer::Int, tinfo::String, oinfo::String)

Constructs the basic object for processing data, the `FunctionCell`. The constructor is parametric so new varieties of
`FunctionCells` can be aliased to cover new functionality.

# Main `FunctionCell` constructors
  * `FunctionCell(f::Function, fargs::Tuple, tinfo=string(f); fkwargs...)` creates a `fixed` FunctionCell or transform.
  * `FunctionCell(f::Function, fargs::Tuple, mprops::ModelProperties, tinfo=string(f); fkwargs...)` creates an `untrained` FunctionCell.
  * `FunctionCell(f::Function, m::Model, tinfo=string(f); fkwargs...)` creates an `trained` FunctionCell

where, `m` is the model, an object of type Model{T} (which is in itself a simple wrapper).

  In the above constructors, `f` is an overloaded function that, depending on circumstances, is applied on the input data piped
or called by the cell, `fargs` are `f`'s arguments, `tinfo` is the title and `fkwargs` are the keyword arguments of `f`. For
fixed function cells, `f` is the the transform applied on the input; for untrained function cells, `f` is the training algorithm which
generates a model and some arbitrary properties of the model, such as priors, expected input/output dimensions etc.
generated in training. The body of `f` for untrained cells should return a trained cell where `f` is the model execution function,
`m` is the model and `mprops` the model properties.
"""
#################
# FunctionCells #
#################
struct FunctionCell{T,S,U, V<:Tuple, W<:Tuple} <: AbstractCell{T,S,U}
	x::T                                                                                # Model
    	y::S                                                                                # Model properties
    	f::U                                                                                # Function for transforming/training/predicting
	fargs::V										# Function arguments
	fkwargs::W										# Function keyword arguments
	layer::Int										# Layer
    	tinfo::String										# Title or name
    	oinfo::String                                                                        	# Other information
end

"""
	ModelProperties([idim ,odim, labels, other])

Basic type used to describe the properties of a model.

# Arguments
  * `idim::Int` is the required input data dimension (0 stands for any dimension)
  * `odim::Int` is the expected dimension of the output of the model (0 stands for any dimension)
  * `labels::LabelEncoding` the encoding of the labels
  * `other::Dict` are other properties of the data i.e. priors, custom execution arguments
"""
struct ModelProperties{T<:LabelEncoding, S<:Dict}
	idim::Int
	odim::Int
	labels::T
	other::S
end

ModelProperties(idim::Int=0, odim::Int=0, enclabels::LabelEncoding=LabelEnc.NativeLabels([],Val{0})) =
	ModelProperties(idim, odim, enclabels, Dict())

labelencn(labels::T where T<:AbstractVector{S}) where S = begin
	ulabels::Vector{S} = sort(unique(labels))
 	return LabelEnc.NativeLabels(ulabels)
end

ModelProperties(idim::Int, odim::Int, labels::Void, other::Dict=Dict()) =
	ModelProperties(idim, odim, LabelEnc.NativeLabels([],Val{0}), other)

ModelProperties(idim::Int, odim::Int, labels::T where T<:AbstractVector, other::Dict=Dict()) =	# Known dimensions and labels
	ModelProperties(idim, odim, labelencn(labels), other)

ModelProperties(idim::Int, odim::Int, labels::T where T<:AbstractMatrix, other::Dict=Dict()) =	# Known dimensions and labels
	ModelProperties(idim, odim, labelenc(labels), other)

Base.show(io::IO, mprop::ModelProperties{T,S}) where {T,S} = begin
	println("Model properties")
	println("`- I/O dimensions: $(mprop.idim)/$(mprop.odim)")
	println("`- label encoding: $(typeof(mprop.labels))")
	print("`- other: $(mprop.other)")
end

"""
	Model(data [,properties])

Basic wrapper around model data and propertiesm, containing all the data needed for the execution of a model,
given an execution function of the form:
	`f(x, m)`
where `x` is the input data and `m::Model` the model.

# Arguments
  * `data` is the input data on which the model is applied
  * `properties::ModelProperties` are the properties of the model, containing parameters necessary for its correct operation
"""
struct Model{T, S<:ModelProperties}
	data::T
	properties::S
end

Model() = Model(nothing, ModelProperties()) 							# Empty model

Model(data) = Model(data, ModelProperties())							# Model without properties

Model(properties::S where S<:ModelProperties) = Model(nothing, properties)			# Model without data

Model(model::T where T<:Model, properties::S where S<:ModelProperties) =			# Grabdata from a model and replace properties
	Model(model.data, properties)

Base.show(io::IO, m::Model) = begin
	println("Model")
	println("`- data: $(typeof(m.data))")
	print("`- properties: Int/Int/$(typeof(m.properties.labels))/$(typeof(m.properties.other))")
end

# Constructor for fixed FunctionCells e.g. simple transforms
FunctionCell(f::U where U<:Function, fargs::V where V<: Tuple, tinfo::String = string(f); fkwargs...) =
	FunctionCell(nothing, nothing, f, fargs, ntuple(i->fkwargs[i], Val{length(fkwargs)}), 0, tinfo, oinfoglobal)

# Constructor for untrained FunctionCells e.g. untrained classifiers or trainable transforms
FunctionCell(f::U where U<:Function, fargs::V where V<:Tuple, mprops::ModelProperties, tinfo::String = string(f); fkwargs...) =
	FunctionCell(nothing, mprops, f, fargs, ntuple(i->fkwargs[i], Val{length(fkwargs)}), 0, tinfo, oinfoglobal)

# Constructor for trained FunctionCells (no arguments, keyword arguments, the model should fully describe the operation)
FunctionCell(f::U where U<:Function, m::T where T<:Model, tinfo::String = string(f)) =
	FunctionCell(m, nothing, f, (), (),  0, tinfo, oinfoglobal)

########################
# FunctionCell aliases #
########################
const CellFunF{U<:Function} = 			 FunctionCell{Void, Void, U}			# Fixed function cell: simple transforms
const CellFunU{S<:ModelProperties, U<:Function}= FunctionCell{Void, S, U}			# Untrained function cell: untrained classifier or transform
const CellFunT{T<:Model, S<:Void, U<:Function} = FunctionCell{T, S, U}				# Trained function cell: trained classifier, transform
const CellFun{T,S, U<:Function} = 		 FunctionCell{T, S, U}

##############################################################################################################################
# Operators [Function cells]																								 #
##############################################################################################################################

# Pipe operators for function cells (a different execution method for each FunctionCell variety: fixed, untrained and trained)
|>(x::T where T, c::S where S<:CellFunF) = getf!(c)(x, c.fargs...; c.fkwargs...)
|>(x::T where T, c::S where S<:CellFunU) = getf!(c)(x, c.fargs...; c.fkwargs...)
|>(x::T where T, c::S where S<:CellFunT) = begin
	# Check input dimension (if input dimension is 0, skip the check, input dimension can vary)
	getx!(c).properties.idim != 0 && getx!(c).properties.idim !=nvars(x) &&
		error("Expected $(getx!(c).properties.idim) input variables(s), got $(nvars(x)).")
	return getf!(c)(x, getx!(c))
end

"""
	functioncell([x,y,z])

Calls the `FunctionCell` constructor for one, two or three arguments. If no arguments are provided,
a `FunctionCell` that expects as input a collection with contents compatible with the arguments of
the `FunctionCell` constructor is returned.
"""
functioncell() = FunctionCell((args)->FunctionCell(args...), (), "FunctionCell creator")
functioncell(x::T) where T = FunctionCell(x)					# calls to convert methods
functioncell(x::T, y::S) where {T,S}= FunctionCell(x, y)			# Calls any 2 argument parametric constructor
functioncell(x::T, y::S, z::U) where {T,S,U}= FunctionCell(x, y, z)		# Calls any 3 argument parametric constructor


function clear(obj::StorelessUnivariateStatistic)
    return jcall(obj, "clear", void, ())
end

function copy(obj::StorelessUnivariateStatistic)
    return jcall(obj, "copy", StorelessUnivariateStatistic, ())
end

function get_result(obj::StorelessUnivariateStatistic)
    return jcall(obj, "getResult", jdouble, ())
end

function increment(obj::StorelessUnivariateStatistic, arg0::jdouble)
    return jcall(obj, "increment", void, (jdouble,), arg0)
end


using StateMachineIterators
using Base.Test
import Iterators: chain
import IJulia

# Test non-iterator interface:

let
    machine = StateMachine(:going_up, :final,
       [(:going_up, x -> x >= 10, :going_down),
        (:going_down, x -> x <= 1, :final)])

    behaviors = Dict(:going_up => x -> x + 1,
                     :going_down => x -> x - 1,
                     :final => x -> x)

    values = []
    x = 1
    state = initial(machine)
    while state != final(machine)
        push!(values, x)
        state = nextstate(machine, state, x)
        x = behaviors[state](x)
    end

    @test all(values .== [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1])

end

# Test iterator interface:

let
    machine = StateMachine(:going_up, :final,
       [(:going_up, x -> x >= 5, :going_down),
        (:going_down, x -> x <= 1, :final)])

    behaviors = Dict(:going_up => x -> x + 1,
                     :going_down => x -> x - 1,
                     :final => x -> x)

    values = []
    x = 1
    for state in StateMachineIterator(machine, () -> x)
        push!(values, x)
        x = behaviors[state](x)
    end

    @test all(values .== [1, 2, 3, 4, 5, 4, 3, 2, 1])
end

let
    x_machine = StateMachine(:going_up, :final,
       [(:going_up, x -> x >= 10, :going_down),
        (:going_down, x -> x <= 1, :final)])

    y_machine = StateMachine(:going_up, :final,
       [(:going_up, y -> y >= 3, :going_down),
        (:going_down, y -> y <= 1, :going_up)])

    behaviors = Dict(:going_up => a -> a + 1,
                     :going_down => a -> a - 1,
                     :final => a -> a)

    values = []
    x = 1
    y = 1
    for (x_state, y_state) in zip(StateMachineIterator(x_machine, () -> x),
                                  StateMachineIterator(y_machine, () -> y))
        push!(values, (x, y))
        x = behaviors[x_state](x)
        y = behaviors[y_state](y)
    end

    @test all(values .== [
        (1, 1),
        (2, 2),
        (3, 3),
        (4, 2),
        (5, 1),
        (6, 2),
        (7, 3),
        (8, 2),
        (9, 1),
        (10, 2),
        (9, 3),
        (8, 2),
        (7, 1),
        (6, 2),
        (5, 3),
        (4, 2),
        (3, 1),
        (2, 2),
        (1, 3)])
end

let
    machine = StateMachine(:going_up, :final,
       [(:going_up, x -> x >= 5, :going_down),
        (:going_down, x -> x <= 1, :final)])

    behaviors = Dict(:going_up => x -> x + 1,
                     :going_down => x -> x - 1,
                     :final => x -> x)

    values = []

    x = 1
    for state in chain(repeated(StateMachineIterator(machine, () -> x), 2)...)
        push!(values, x)
        x = behaviors[state](x)
    end
    @test all(values .== [1, 2, 3, 4, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 4, 3, 2, 1])
end

jupyter = IJulia.jupyter
notebook = "../examples/state_machines.ipynb"
kernel = "julia-$(VERSION.major).$(VERSION.minor)"
run(`$jupyter nbconvert --to notebook --execute $notebook --output $notebook --ExecutePreprocessor.kernel_name="$(kernel)"`)


using OrdinaryDiffEq
using Trixi

###############################################################################
# semidiscretization of the linear advection equation

advection_velocity = (0.2, -0.7)
equations = LinearScalarAdvectionEquation2D(advection_velocity)

# Create DG solver with polynomial degree = 3 and (local) Lax-Friedrichs/Rusanov flux as surface flux
solver = DGSEM(polydeg=3, surface_flux=flux_lax_friedrichs)

# Deformed rectangle that looks like a waving flag,
# lower and upper faces are sinus curves, left and right are vertical lines.
f1(s) = SVector(-1.0, s - 1.0)
f2(s) = SVector( 1.0, s + 1.0)
f3(s) = SVector(s, -1.0 + sin(0.5 * pi * s))
f4(s) = SVector(s,  1.0 + sin(0.5 * pi * s))

# Create P4estMesh with 3 x 2 trees,
# approximate the geometry with a smaller polydeg for testing.
cells_per_dimension = (3, 2)
mesh = P4estMesh(cells_per_dimension, polydeg=3,
                 faces=(f1, f2, f3, f4))

# Refine bottom left quadrant of each tree to level 4
function refine_fn(p4est, which_tree, quadrant)
  if quadrant.x == 0 && quadrant.y == 0 && quadrant.level < 4
    # return true (refine)
    return Cint(1)
  else
    # return false (don't refine)
    return Cint(0)
  end
end

# Refine recursively until each bottom left quadrant of a tree has level 4
# The mesh will be rebalanced before the simulation starts
refine_fn_c = @cfunction(refine_fn, Cint, (Ptr{Trixi.p4est_t}, Ptr{Trixi.p4est_topidx_t}, Ptr{Trixi.p4est_quadrant_t}))
Trixi.refine_p4est!(mesh.p4est, true, refine_fn_c, C_NULL)
Trixi.balance!(mesh)

# Refine everything again for convergence tests
refine_fn2(p4est, which_tree, quadrant) = Cint(1)
refine_fn_c2 = @cfunction(refine_fn2, Cint, (Ptr{Trixi.p4est_t}, Ptr{Trixi.p4est_topidx_t}, Ptr{Trixi.p4est_quadrant_t}))

# Abuse the variable `initial_refinement_level` for this because this will be increased
# during convergence tests
initial_refinement_level = 0
for i in 1:initial_refinement_level
  Trixi.refine_p4est!(mesh.p4est, false, refine_fn_c2, C_NULL)
end

# A semidiscretization collects data structures and functions for the spatial discretization
semi = SemidiscretizationHyperbolic(mesh, equations, initial_condition_convergence_test, solver)

###############################################################################
# ODE solvers, callbacks etc.

# Create ODE problem with time span from 0.0 to 1.0
ode = semidiscretize(semi, (0.0, 1.0));

# At the beginning of the main loop, the SummaryCallback prints a summary of the simulation setup
# and resets the timers
summary_callback = SummaryCallback()

# The AnalysisCallback allows to analyse the solution in regular intervals and prints the results
analysis_callback = AnalysisCallback(semi, interval=100)

# The SaveSolutionCallback allows to save the solution to a file in regular intervals
save_solution = SaveSolutionCallback(interval=100,
                                     solution_variables=cons2prim)

# The StepsizeCallback handles the re-calculcation of the maximum Δt after each time step
stepsize_callback = StepsizeCallback(cfl=1.2)

# Create a CallbackSet to collect all callbacks such that they can be passed to the ODE solver
callbacks = CallbackSet(summary_callback, analysis_callback, save_solution, stepsize_callback)

###############################################################################
# run the simulation

# OrdinaryDiffEq's `solve` method evolves the solution in time and executes the passed callbacks
sol = solve(ode, CarpenterKennedy2N54(williamson_condition=false),
            dt=1.0, # solve needs some value here but it will be overwritten by the stepsize_callback
            save_everystep=false, callback=callbacks);

# Print the timer summary
summary_callback()