using Test

@info "CANalyze.Signals tests..."
@testset "bit_signal" begin
    using CANalyze.Signals

    @testset "bit_signal_1" begin
        signal = Bit(20)
        @test true
    end

    @testset "bit_signal_2" begin
        signal = Bit(start=20)
        @test true
    end

    @testset "start_1" begin
        signal = Bit(start=20)
        @test start(signal) == 20
    end

    @testset "byte_order_1" begin
        signal = Bit(start=20)
        @test byte_order(signal) == :little_endian
    end

    @testset "length_1" begin
        signal = Bit(start=20)
        @test length(signal) == 1
    end
end

@testset "unsigned_signal" begin
    using CANalyze.Signals

    @testset "unsigned_signal_1" begin
        signal = Signals.Unsigned(0, 8, 1.0, 0.0, :little_endian)
        @test true
    end

    @testset "unsigned_signal_2" begin
        signal = Signals.Unsigned(start=0, length=8, factor=1.0, offset=0.0,
                                  byte_order=:little_endian)
        @test true
    end

    @testset "unsigned_signal_3" begin
        signal = Signals.Unsigned{Float16}(0, 8)
        @test true
    end

    @testset "unsigned_signal_4" begin
        signal = Signals.Unsigned{Float16}(start=0, length=8)
        @test true
    end

    @testset "unsigned_signal_5" begin
        @test_throws DomainError Signals.Unsigned{Float16}(start=0, length=0)
    end

    @testset "unsigned_signal_6" begin
        @test_throws DomainError Signals.Unsigned{Float16}(start=0, length=0,
                                                           byte_order=:mixed_endian)
    end

    @testset "start_1" begin
        signal = Signals.Unsigned(23, 8, 1.0, 0.0, :little_endian)
        @test start(signal) == 23
    end

    @testset "length_1" begin
        signal = Signals.Unsigned(0, 8, 1.0, 0.0, :little_endian)
        @test length(signal) == 8
    end

    @testset "factor_1" begin
        signal = Signals.Unsigned(0, 8, 1.0, 0.0, :little_endian)
        @test factor(signal) == 1
    end

    @testset "offset_1" begin
        signal = Signals.Unsigned(0, 8, 1.0, 1337, :little_endian)
        @test offset(signal) == 1337
    end

    @testset "byte_order_1" begin
        signal = Signals.Unsigned(0, 8, 1.0, 0.0, :little_endian)
        @test byte_order(signal) == :little_endian
    end

    @testset "byte_order_2" begin
        signal = Signals.Unsigned(0, 8, 1.0, 0.0, :big_endian)
        @test byte_order(signal) == :big_endian
    end
end

@testset "signed_signal" begin
    using CANalyze.Signals

    @testset "signed_signal_1" begin
        signal = Signals.Signed(0, 8, 1.0, 0.0, :little_endian)
        @test true
    end

    @testset "signed_signal_2" begin
        signal = Signals.Signed(start=0, length=8, factor=1.0, offset=0.0,
                                  byte_order=:little_endian)
        @test true
    end

    @testset "signed_signal_3" begin
        signal = Signals.Signed{Float16}(0, 8)
        @test true
    end

    @testset "signed_signal_4" begin
        signal = Signals.Signed{Float16}(start=0, length=8)
        @test true
    end

    @testset "signed_signal_5" begin
        @test_throws DomainError Signals.Signed{Float16}(start=0, length=0)
    end

    @testset "signed_signal_6" begin
        @test_throws DomainError Signals.Signed{Float16}(start=0, length=0,
                                                         byte_order=:mixed_endian)
    end

    @testset "start_1" begin
        signal = Signals.Signed(23, 8, 1.0, 0.0, :little_endian)
        @test start(signal) == 23
    end

    @testset "length_1" begin
        signal = Signals.Signed(0, 8, 1.0, 0.0, :little_endian)
        @test length(signal) == 8
    end

    @testset "factor_1" begin
        signal = Signals.Signed(0, 8, 1.0, 0.0, :little_endian)
        @test factor(signal) == 1
    end

    @testset "offset_1" begin
        signal = Signals.Signed(0, 8, 1.0, 1337, :little_endian)
        @test offset(signal) == 1337
    end

    @testset "byte_order_1" begin
        signal = Signals.Signed(0, 8, 1.0, 0.0, :little_endian)
        @test byte_order(signal) == :little_endian
    end

    @testset "byte_order_2" begin
        signal = Signals.Signed(0, 8, 1.0, 0.0, :big_endian)
        @test byte_order(signal) == :big_endian
    end
end

@testset "float16_signal" begin
    using CANalyze.Signals

    @testset "float16_signal_1" begin
        signal = Signals.Float16Signal(0)
        @test true
    end

    @testset "float16_signal_2" begin
        signal = Signals.Float16Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test true
    end

    @testset "float16_signal_3" begin
        signal = Signals.Float16Signal(start=0, factor=1, offset=0,
                                       byte_order=:little_endian)
        @test true
    end

    @testset "start_1" begin
        signal = Signals.Float16Signal(start=42, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test start(signal) == 42
    end

    @testset "length_1" begin
        signal = Signals.Float16Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test length(signal) == 16
    end

    @testset "factor_1" begin
        signal = Signals.Float16Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test factor(signal) == 1
    end

    @testset "offset_1" begin
        signal = Signals.Float16Signal(start=0, factor=1.0, offset=1337,
                                       byte_order=:little_endian)
        @test offset(signal) == 1337
    end

    @testset "byte_order_1" begin
        signal = Signals.Float16Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test byte_order(signal) == :little_endian
    end

    @testset "byte_order_2" begin
        signal = Signals.Float16Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:big_endian)
        @test byte_order(signal) == :big_endian
    end
end

@testset "float32_signal" begin
    using CANalyze.Signals

    @testset "float32_signal_1" begin
        signal = Signals.Float32Signal(0)
        @test true
    end

    @testset "float32_signal_2" begin
        signal = Signals.Float32Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test true
    end

    @testset "float32_signal_3" begin
        signal = Signals.Float32Signal(start=0, factor=1, offset=0,
                                       byte_order=:little_endian)
        @test true
    end

    @testset "start_1" begin
        signal = Signals.Float32Signal(start=42, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test start(signal) == 42
    end

    @testset "length_1" begin
        signal = Signals.Float32Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test length(signal) == 32
    end

    @testset "factor_1" begin
        signal = Signals.Float32Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test factor(signal) == 1
    end

    @testset "offset_1" begin
        signal = Signals.Float32Signal(start=0, factor=1.0, offset=1337,
                                       byte_order=:little_endian)
        @test offset(signal) == 1337
    end

    @testset "byte_order_1" begin
        signal = Signals.Float32Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test byte_order(signal) == :little_endian
    end

    @testset "byte_order_2" begin
        signal = Signals.Float32Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:big_endian)
        @test byte_order(signal) == :big_endian
    end
end

@testset "float64_signal" begin
    using CANalyze.Signals

    @testset "float64_signal_1" begin
        signal = Signals.Float64Signal(0)
        @test true
    end

    @testset "float64_signal_2" begin
        signal = Signals.Float64Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test true
    end

    @testset "float64_signal_3" begin
        signal = Signals.Float64Signal(start=0, factor=1, offset=0,
                                       byte_order=:little_endian)
        @test true
    end

    @testset "start_1" begin
        signal = Signals.Float64Signal(start=42, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test start(signal) == 42
    end

    @testset "length_1" begin
        signal = Signals.Float64Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test length(signal) == 64
    end

    @testset "factor_1" begin
        signal = Signals.Float64Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test factor(signal) == 1
    end

    @testset "offset_1" begin
        signal = Signals.Float64Signal(start=0, factor=1.0, offset=1337,
                                       byte_order=:little_endian)
        @test offset(signal) == 1337
    end

    @testset "byte_order_1" begin
        signal = Signals.Float64Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:little_endian)
        @test byte_order(signal) == :little_endian
    end

    @testset "byte_order_2" begin
        signal = Signals.Float64Signal(start=0, factor=1.0, offset=0.0,
                                       byte_order=:big_endian)
        @test byte_order(signal) == :big_endian
    end
end

@testset "float_signal" begin
    using CANalyze.Signals

    @testset "float_signal_1" begin
        signal = Signals.FloatSignal(start=0, factor=2, offset=-1337,
                                     byte_order=:big_endian)
        @test true
    end

end

@testset "raw_signal" begin
    using CANalyze.Signals

    @testset "raw_signal_1" begin
        signal = Signals.Raw(0, 8, :little_endian)
        @test true
    end

    @testset "raw_signal_2" begin
        signal = Signals.Raw(start=0, length=8, byte_order=:little_endian)
        @test true
    end

    @testset "raw_signal_3" begin
        @test_throws DomainError Signals.Raw(start=0, length=0, byte_order=:little_endian)
    end

    @testset "raw_signal_4" begin
        @test_throws DomainError Signals.Raw(start=0, length=1, byte_order=:mixed_endian)
    end

    @testset "start_1" begin
        signal = Signals.Raw(start=42, length=8, byte_order=:little_endian)
        @test start(signal) == 42
    end

    @testset "length_1" begin
        signal = Signals.Raw(start=0, length=23, byte_order=:little_endian)
        @test length(signal) == 23
    end

    @testset "byte_order_1" begin
        signal = Signals.Raw(start=0, length=8, byte_order=:little_endian)
        @test byte_order(signal) == :little_endian
    end

    @testset "byte_order_2" begin
        signal = Signals.Raw(start=0, length=8, byte_order=:big_endian)
        @test byte_order(signal) == :big_endian
    end
end

@testset "named_signal" begin
    using CANalyze.Signals

    @testset "named_signal_1" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal("ABC", nothing, nothing, s)
        @test true
    end

    @testset "named_signal_2" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal(name="ABC", unit=nothing, default=nothing,
                                     signal=s)
        @test true
    end

    @testset "named_signal_3" begin
        s = Signals.Raw(0, 8, :little_endian)
        @test_throws DomainError Signals.NamedSignal(name="",
                                                     unit=nothing,
                                                     default=nothing,
                                                     signal=s)
    end

    @testset "name_1" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal(name="ABC", unit=nothing, default=nothing,
                                     signal=s)
        @test name(signal) == "ABC"
    end

    @testset "unit_1" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal(name="ABC", unit=nothing, default=nothing,
                                     signal=s)
        @test unit(signal) == nothing
    end

    @testset "unit_2" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal(name="ABC", unit="Ah", default=nothing,
                                     signal=s)
        @test unit(signal) == "Ah"
    end

    @testset "default_1" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal(name="ABC", unit="Ah", default=nothing,
                                     signal=s)
        @test default(signal) == nothing
    end

    @testset "default_2" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal(name="ABC", unit="Ah", default=UInt(1337),
                                     signal=s)
        @test default(signal) == 1337
    end

    @testset "signal_1" begin
        s = Signals.Raw(0, 8, :little_endian)
        signal = Signals.NamedSignal(name="ABC", unit="Ah", default=nothing,
                                     signal=s)
        @test Signals.signal(signal) == s
    end
end

@testset "bits" begin
    using CANalyze.Signals

    @testset "bit_1" begin
        signal = Bit(42)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(42)
    end

    @testset "unsigned_1" begin
        signal = Signals.Unsigned(7, 5, 1.0, 0.0, :little_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(7, 8, 9, 10, 11)
    end

    @testset "unsigned_2" begin
        signal = Signals.Unsigned(7, 5, 1.0, 0.0, :big_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(7, 6, 5, 4, 3)
    end

    @testset "signed_1" begin
        signal = Signals.Signed(7, 5, 1.0, 0.0, :little_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(7, 8, 9, 10, 11)
    end

    @testset "signed_2" begin
        signal = Signals.Signed(3, 5, 1.0, 0.0, :big_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(3, 2, 1, 0, 15)
    end

    @testset "float16_1" begin
        signal = Signals.Float16Signal(0; byte_order=:little_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(Set{UInt16}([i for i=0:15]))
    end

    @testset "float16_2" begin
        signal = Signals.Float16Signal(0; byte_order=:big_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(0, 15, 14, 13, 12, 11, 10, 9, 8, 23, 22, 21, 20, 19, 18, 17)
    end

    @testset "float32_1" begin
        signal = Signals.Float32Signal(0; byte_order=:little_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(Set{UInt16}([i for i=0:31]))
    end

    @testset "float32_2" begin
        signal = Signals.Float32Signal(39; byte_order=:big_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(Set{UInt16}([i for i=32:63]))
    end

    @testset "float64_1" begin
        signal = Signals.Float64Signal(0; byte_order=:little_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(Set{UInt16}([i for i=0:63]))
    end

    @testset "float64_2" begin
        signal = Signals.Float64Signal(7; byte_order=:big_endian)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(Set{UInt16}([i for i=0:63]))
    end

    @testset "named_signal_1" begin
        s = Signals.Float64Signal(7; byte_order=:big_endian)
        signal = Signals.NamedSignal("ABC", nothing, nothing, s)
        bits = Signals.Bits(signal)
        @test bits == Signals.Bits(Set{UInt16}([i for i=0:63]))
    end

end

@testset "share_bits" begin
    using CANalyze.Signals

    @testset "share_bits_1" begin
        sig1 = Signals.Float16Signal(0; byte_order=:little_endian)
        sig2 = Signals.Float16Signal(0; byte_order=:little_endian)

        bits1 = Signals.Bits(sig1)
        bits2 = Signals.Bits(sig2)
        @test Signals.share_bits(bits1, bits2)
    end

    @testset "share_bits_2" begin
        sig1 = Signals.Float16Signal(0; byte_order=:little_endian)
        sig2 = Signals.Float16Signal(16; byte_order=:little_endian)

        bits1 = Signals.Bits(sig1)
        bits2 = Signals.Bits(sig2)
        @test !Signals.share_bits(bits1, bits2)
    end
end

@testset "overlap" begin
    using CANalyze.Signals

    @testset "overlap_1" begin
        sig1 = Signals.Float16Signal(0; byte_order=:little_endian)
        sig2 = Signals.Float16Signal(0; byte_order=:little_endian)

        @test Signals.overlap(sig1, sig2)
    end

    @testset "overlap_2" begin
        sig1 = Signals.Float16Signal(0; byte_order=:little_endian)
        sig2 = Signals.Float16Signal(16; byte_order=:little_endian)

        @test !Signals.overlap(sig1, sig2)
    end
end

@testset "check" begin
    using CANalyze.Signals

    @testset "bit_1" begin
        signal = Signals.Bit(0)
        @test Signals.check(signal, UInt8(1))
    end

    @testset "bit_2" begin
        signal = Signals.Bit(8)
        @test !Signals.check(signal, UInt8(1))
    end

    @testset "unsigned_1" begin
        signal = Signals.Unsigned(7, 5, 1.0, 0.0, :little_endian)
        @test Signals.check(signal, UInt8(2))
    end

    @testset "unsigned_2" begin
        signal = Signals.Unsigned(7, 9, 1.0, 0.0, :big_endian)
        @test Signals.check(signal, UInt8(2))
    end

    @testset "signed_1" begin
        signal = Signals.Signed(7, 5, 1.0, 0.0, :little_endian)
        @test Signals.check(signal, UInt8(2))
    end

    @testset "signed_2" begin
        signal = Signals.Signed(7, 9, 1.0, 0.0, :big_endian)
        @test Signals.check(signal, UInt8(2))
    end

    @testset "float16_1" begin
        signal = Signals.Float16Signal(0; byte_order=:little_endian)
        @test Signals.check(signal, UInt8(2))
    end

    @testset "float16_2" begin
        signal = Signals.Float16Signal(0; byte_order=:big_endian)
        @test !Signals.check(signal, UInt8(2))
    end

    @testset "float32_1" begin
        signal = Signals.Float32Signal(0; byte_order=:little_endian)
        @test Signals.check(signal, UInt8(4))
    end

    @testset "float32_2" begin
        signal = Signals.Float32Signal(0; byte_order=:big_endian)
        @test !Signals.check(signal, UInt8(4))
    end

    @testset "float64_1" begin
        signal = Signals.Float64Signal(0; byte_order=:little_endian)
        @test Signals.check(signal, UInt8(8))
    end

    @testset "float64_2" begin
        signal = Signals.Float64Signal(0; byte_order=:big_endian)
        @test !Signals.check(signal, UInt8(8))
    end

    @testset "raw_1" begin
        signal = Signals.Raw(0, 64, :little_endian)
        @test Signals.check(signal, UInt8(8))
    end

    @testset "raw_2" begin
        signal = Signals.Raw(0, 64, :big_endian)
        @test Signals.check(signal, UInt8(9))
    end

    @testset "named_signal_1" begin
        s = Signals.Raw(0, 64, :big_endian)
        signal = Signals.NamedSignal("ABC", nothing, nothing, s)
        @test Signals.check(signal, UInt8(9))
    end
end

@testset "equal" begin
    using CANalyze.Signals

    @testset "bit_1" begin
        bit1 = Signals.Bit(20)
        bit2 = Signals.Bit(20)
        @test bit1 == bit2
    end

    @testset "bit_2" begin
        bit1 = Signals.Bit(20)
        bit2 = Signals.Bit(21)
        @test !(bit1 == bit2)
    end

    @testset "unsigned_1" begin
        sig1 = Signals.Unsigned(0, 8, 1, 0, :little_endian)
        sig2 = Signals.Unsigned(0, 8, 1, 0, :little_endian)
        @test sig1 == sig2
    end

    @testset "unsigned_2" begin
        sig1 = Signals.Unsigned(0, 8, 1, 0, :little_endian)
        sig2 = Signals.Unsigned(1, 8, 1, 0, :little_endian)
        @test !(sig1 == sig2)
    end

    @testset "unsigned_3" begin
        sig1 = Signals.Unsigned(0, 8, 1, 0, :little_endian)
        sig2 = Signals.Unsigned(0, 9, 1, 0, :little_endian)
        @test !(sig1 == sig2)
    end

    @testset "unsigned_4" begin
        sig1 = Signals.Unsigned(0, 8, 1, 0, :little_endian)
        sig2 = Signals.Unsigned(0, 8, 2, 0, :little_endian)
        @test !(sig1 == sig2)
    end

    @testset "unsigned_5" begin
        sig1 = Signals.Unsigned(0, 8, 1, 0, :little_endian)
        sig2 = Signals.Unsigned(0, 8, 1, -1, :little_endian)
        @test !(sig1 == sig2)
    end

    @testset "unsigned_6" begin
        sig1 = Signals.Unsigned(0, 8, 1, 0, :little_endian)
        sig2 = Signals.Unsigned(0, 8, 1, 0, :big_endian)
        @test !(sig1 == sig2)
    end
end


import HoTRG_lemon.LatticeModule:
	getHausdorffDim,
	getFractalDim,
	getLegextension,
	getTensorP,
	getTensorQ,
	setTensorP!,
	setTensorQ!

FractalSimulators = [
					 :Classical2dFractalSimulator,
					 :Quantum2dFractalSimulator,
					 :Quantum2dFractalInititerSimulator
					]

#
##### update the SimulatorModule

for FractalSimulator in FractalSimulators

	eval(quote

	  function setNorm!{T}(simulator::$(FractalSimulator){T}, norm::T, which::Int, iteration::Int)
			simulator.normalizationFactor[iteration, which] = norm
	  end

		function getNorm{T}(simulator::$(FractalSimulator){T}, which::Int, iteration::Int)
			return simulator.normalizationFactor[iteration, which]
		end

		function setCoefficient!{T}(simulator::$(FractalSimulator){T}, coefficient::T, which::Int, iteration::Int)
			simulator.coefficients[iteration, which] = coefficient
		end

		function getCoefficient{T}(simulator::$(FractalSimulator){T}, which::Int, iteration::Int)
			if which == 0
				coef = simulator.coefficients[iteration,:]
			else
				coef = simulator.coefficients[iteration, which]
			end
			return coef
		end

	end)

		# inherit from FractalLattice
		single_args = [
					   :getHausdorffDim,
					   :getFractalDim,
					   :getLegextension,
					   :getTensorQ
					   ]

		for f in single_args
			eval(quote

			  function $(f)(simulator::$(FractalSimulator))
			  return $(f)(simulator.lattice)
				end

			end)
		end

		getTensors = [
					 :getTensorP
					 ]

		for f in getTensors
			eval(quote

			  function $(f)(simulator::$(FractalSimulator), which::Int = 0)
				  return $(f)(simulator.lattice, which)
			  end

			 end)
		end

		setTensors = [
					  :setTensorP!
					  ]
		for f in setTensors
			eval(quote

			  function $(f)(simulator::$(FractalSimulator), which::Int, tensor::Array)
			  return $(f)(simulator.lattice, which, tensor)
			  end

			 end)
		end

		setTensors2 = [
					  :setTensorQ!
					 ]
		for f in setTensors2
			eval(quote

			  function $(f)(simulator::$(FractalSimulator), tensor::Array)
			  return $(f)(simulator.lattice, tensor)
			  end

			 end)
		end
end


const PyFraction_Type__ref = Ref(PyNULL)
PyFraction_Type() = begin
    ptr = PyFraction_Type__ref[]
    if isnull(ptr)
        m = PyImport_ImportModule("fractions")
        isnull(m) && return ptr
        ptr = PyObject_GetAttrString(m, "Fraction")
        Py_DecRef(m)
        isnull(m) && return ptr
        PyFraction_Type__ref[] = ptr
    end
    ptr
end

PyFraction_From(x::Union{Rational,Integer}) = begin
    t = PyFraction_Type()
    isnull(t) && return PyNULL
    a = PyTuple_New(2)
    isnull(a) && return PyNULL
    b = PyLong_From(numerator(x))
    isnull(b) && (Py_DecRef(a); return PyNULL)
    err = PyTuple_SetItem(a, 0, b)
    ism1(err) && (Py_DecRef(a); return PyNULL)
    b = PyLong_From(denominator(x))
    isnull(b) && (Py_DecRef(a); return PyNULL)
    err = PyTuple_SetItem(a, 1, b)
    ism1(err) && (Py_DecRef(a); return PyNULL)
    r = PyObject_CallObject(t, a)
    Py_DecRef(a)
    return r
end


# Make sure filtration building is also benchmarked.
function _ripserer(data, complex, complex_kwargs, ripserer_kwargs)
    return ripserer(complex(data; complex_kwargs...); ripserer_kwargs...)
end

function _ripserer(filename::String, complex, complex_kwargs, ripserer_kwargs)
    data = load_data(filename)
    return ripserer(complex(data; complex_kwargs...); ripserer_kwargs...)
end

function ripserer_benchmark(
    b::Benchmark; extras=false, from_file=false, sparse=nothing, kwargs...
)
    if !isnothing(b.threshold)
        complex_kwargs = (;threshold=b.threshold)
    else
        complex_kwargs = (;)
    end
    if extras
        complex_kwargs = (;complex_kwargs..., b.extra_kwargs...)
    end
    if !isnothing(sparse)
        complex_kwargs = (;complex_kwargs..., sparse=sparse)
    end
    if from_file
        input = b.filename
    else
        input = b.data
    end
    kwargs = (;kwargs..., b.kwargs...)
    return @benchmarkable _ripserer(
        $(input), $(b.complex), $(complex_kwargs), $(kwargs)
    ) samples=5 seconds=1000 gcsample=true
end


Ns() = [1_000,]
# Ns() = [10,]
Ds() = [1,]

generate_x̄s(x, x̄s::Nothing) = [fill(x, N) for N in Ns()]
generate_x̄s(x, x̄s) = x̄s

function create_benchmarks(name, grads::Bool, f, x...)
    if grads
        @benchset name begin
            @bench "eval" $f($x...)
            __forward() = Zygote.pullback(f, x...)
            out, back = Zygote.pullback(f, x...)
            @bench "forward" $__forward()
            @bench "back" $back($out)
            # @bench "gradient" $__gradient()
        end
    else
        @bench name $f($x...)
    end
    return nothing
end

# Common benchmarks to run for mean functions.
function create_benchmarks(μ::MeanFunction; grads=true, x=5.0, x̄s=nothing)
    create_benchmarks("μ(x)", grads, μ, x)

    for x̄ in generate_x̄s(x, x̄s)
        N = length(x̄)
        create_benchmarks("map ($(length(x̄)))", grads, x->map(μ, x), x̄)
    end
end

# Common benchmarks for CrossKernel.
function create_benchmarks(k::CrossKernel; x=5.0, x′=4.0, x̄s=nothing, x̄′s=nothing, grads=true)
    create_benchmarks("k(x, x′)", grads, k, x, x′)

    for (x̄, x̄′) in zip(generate_x̄s(x, x̄s), generate_x̄s(x′, x̄′s))
        N = length(x̄)
        @benchset "$N" begin
            create_benchmarks("map k(x̄, x̄′) $N", grads, (x̄, x̄′)->map(k, x̄, x̄′), x̄, x̄′)
            create_benchmarks("pw k(x̄, x̄′) $N", grads, (x̄, x̄′)->pairwise(k, x̄, x̄′), x̄, x̄′)
        end
    end
end

# Common benchmarks to run for Kernels.
function create_benchmarks(k::Kernel; x=5.0, x′=4.0, x̄s=nothing, x̄′s=nothing, grads=true)
    create_benchmarks("k(x)", grads, k, x)
    create_benchmarks("k(x, x′)", grads, k, x, x′)

    for (x̄, x̄′) in zip(generate_x̄s(x, x̄s), generate_x̄s(x′, x̄′s))
        N = length(x̄)
        @benchset "$N" begin
            create_benchmarks("map k(x̄) $N", grads, x̄->map(k, x̄), x̄)
            create_benchmarks("map k(x̄, x̄′) $N", grads, (x̄, x̄′)->map(k, x̄, x̄′), x̄, x̄′)
            create_benchmarks("pw k(x̄) $N", grads, x̄->pairwise(k, x̄), x̄)
            create_benchmarks("pw k(x̄, x̄′) $N", grads, (x̄, x̄′)->pairwise(k, x̄, x̄′), x̄, x̄′)
        end
    end
end

function pretty_print(d::BenchmarkGroup, pre=1)
    pretty_print(d.data, pre)
end

function pretty_print(d::Dict, pre=1)
    for (k, v) in d
        if v isa BenchmarkGroup
            s = "$(repr(k))"
            println(join(fill(" ", pre)) * s)
            pretty_print(v, pre+1+4)
        else
            println(join(fill(" ", pre)) * "$(repr(k)) => $(repr(v))")
        end
    end
    nothing
end


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

function get_mnist_providers(batch_size::Int; data_name=:data, label_name=:softmax_label, flat=true)
  # download MNIST into Pkg.dir("MXNet")/data/mnist if not exist
  filenames = mx.get_mnist_ubyte()

  # data provider
  train_provider = mx.MNISTProvider(image=filenames[:train_data],
                                    label=filenames[:train_label],
                                    data_name=data_name, label_name=label_name,
                                    batch_size=batch_size, shuffle=true, flat=flat, silent=true)
  eval_provider = mx.MNISTProvider(image=filenames[:test_data],
                                   label=filenames[:test_label],
                                   data_name=data_name, label_name=label_name,
                                   batch_size=batch_size, shuffle=false, flat=flat, silent=true)

  return (train_provider, eval_provider)
end


# Copyright 2021 The FirstOrderLp Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

@testset "termination" begin
  # Different infeasibility information scenarios.
  infeas_info_no_infeasibility1 = FirstOrderLp.InfeasibilityInformation()

  infeas_info_no_infeasibility2 = FirstOrderLp.InfeasibilityInformation()
  infeas_info_no_infeasibility2.primal_ray_linear_objective = -1.0
  infeas_info_no_infeasibility2.primal_ray_quadratic_norm = 1.0
  infeas_info_no_infeasibility2.max_dual_ray_infeasibility = 1.0

  infeas_info_dual_infeasible = deepcopy(infeas_info_no_infeasibility1)
  infeas_info_dual_infeasible.primal_ray_linear_objective = -1.0
  infeas_info_primal_infeasible = deepcopy(infeas_info_no_infeasibility1)
  infeas_info_primal_infeasible.dual_ray_objective = 1.0

  # Test infeasibility criteria.
  eps_primal_infeasible = 1e-6
  eps_dual_infeasible = 1e-6

  @testset "primal infeasibility" begin
    @test !FirstOrderLp.primal_infeasibility_criteria_met(
      eps_primal_infeasible,
      infeas_info_no_infeasibility1,
    )
    @test !FirstOrderLp.primal_infeasibility_criteria_met(
      eps_primal_infeasible,
      infeas_info_no_infeasibility2,
    )
    @test !FirstOrderLp.primal_infeasibility_criteria_met(
      eps_primal_infeasible,
      infeas_info_dual_infeasible,
    )
    @test FirstOrderLp.primal_infeasibility_criteria_met(
      eps_primal_infeasible,
      infeas_info_primal_infeasible,
    )
  end

  @testset "dual infeasibility" begin
    @test !FirstOrderLp.dual_infeasibility_criteria_met(
      eps_dual_infeasible,
      infeas_info_no_infeasibility1,
    )
    @test !FirstOrderLp.dual_infeasibility_criteria_met(
      eps_dual_infeasible,
      infeas_info_no_infeasibility2,
    )
    @test FirstOrderLp.dual_infeasibility_criteria_met(
      eps_dual_infeasible,
      infeas_info_dual_infeasible,
    )
    @test !FirstOrderLp.dual_infeasibility_criteria_met(
      eps_dual_infeasible,
      infeas_info_primal_infeasible,
    )
  end

  # Different convergence information  stats scenarios.
  convergence_information_optimal = FirstOrderLp.ConvergenceInformation()
  convergence_information_optimal.primal_objective = 1.0
  convergence_information_optimal.dual_objective = 1.0
  convergence_information_optimal.l_inf_primal_variable = 1.0
  convergence_information_optimal.l2_primal_variable = 1.0
  convergence_information_optimal.l_inf_dual_variable = 2.0
  convergence_information_optimal.l2_dual_variable = 2.0

  convergence_info_dont_terminate1 = deepcopy(convergence_information_optimal)
  convergence_info_dont_terminate1.primal_objective = 10.0
  convergence_info_dont_terminate2 = deepcopy(convergence_information_optimal)
  convergence_info_dont_terminate2.l_inf_primal_residual = 1.0
  convergence_info_dont_terminate2.l2_primal_residual = 1.0
  convergence_info_dont_terminate3 = deepcopy(convergence_information_optimal)
  convergence_info_dont_terminate3.l_inf_dual_residual = 1.0
  convergence_info_dont_terminate3.l2_dual_residual = 1.0

  iteration_stats_optimal = FirstOrderLp.IterationStats()
  iteration_stats_optimal.iteration_number = 5
  iteration_stats_optimal.cumulative_kkt_matrix_passes = 100.5
  iteration_stats_optimal.cumulative_time_sec = 5.0
  iteration_stats_optimal.convergence_information =
    [convergence_information_optimal]
  iteration_stats_optimal.infeasibility_information =
    [infeas_info_no_infeasibility1]

  iteration_stats_dont_terminate1 = FirstOrderLp.IterationStats()
  iteration_stats_dont_terminate1.iteration_number = 5
  iteration_stats_dont_terminate1.cumulative_kkt_matrix_passes = 100.5
  iteration_stats_dont_terminate1.cumulative_time_sec = 5.0
  iteration_stats_dont_terminate1.convergence_information =
    [convergence_info_dont_terminate1]
  iteration_stats_dont_terminate1.infeasibility_information =
    [infeas_info_no_infeasibility1]

  qp_cache = FirstOrderLp.cached_quadratic_program_info(example_qp())

  for optimality_norm in [FirstOrderLp.L_INF, FirstOrderLp.L2]
    eps_optimal_relative = 1e-4
    eps_optimal_absolute = 1e-4

    @testset "optimality" begin
      # Test optimality criteria.
      @test !FirstOrderLp.optimality_criteria_met(
        optimality_norm,
        eps_optimal_relative,
        eps_optimal_absolute,
        convergence_info_dont_terminate1,
        qp_cache,
      )
      @test !FirstOrderLp.optimality_criteria_met(
        optimality_norm,
        eps_optimal_relative,
        eps_optimal_absolute,
        convergence_info_dont_terminate2,
        qp_cache,
      )
      @test !FirstOrderLp.optimality_criteria_met(
        optimality_norm,
        eps_optimal_relative,
        eps_optimal_absolute,
        convergence_info_dont_terminate3,
        qp_cache,
      )
      @test FirstOrderLp.optimality_criteria_met(
        optimality_norm,
        eps_optimal_relative,
        eps_optimal_absolute,
        convergence_information_optimal,
        qp_cache,
      )
    end

    full_termination_criteria = FirstOrderLp.construct_termination_criteria(
      optimality_norm = optimality_norm,
      eps_optimal_absolute = eps_optimal_absolute,
      eps_optimal_relative = eps_optimal_relative,
      eps_primal_infeasible = eps_primal_infeasible,
      eps_dual_infeasible = eps_dual_infeasible,
      time_sec_limit = 100.0,
      iteration_limit = 10,
      kkt_matrix_pass_limit = 10000.0,
    )

    @testset "check_termination_criteria" begin
      @test FirstOrderLp.check_termination_criteria(
        full_termination_criteria,
        qp_cache,
        iteration_stats_optimal,
      ) == FirstOrderLp.TERMINATION_REASON_OPTIMAL

      @test !FirstOrderLp.check_termination_criteria(
        full_termination_criteria,
        qp_cache,
        iteration_stats_dont_terminate1,
      )

      full_termination_criteria.time_sec_limit = 1.0
      @test FirstOrderLp.check_termination_criteria(
        full_termination_criteria,
        qp_cache,
        iteration_stats_dont_terminate1,
      ) == FirstOrderLp.TERMINATION_REASON_TIME_LIMIT

      full_termination_criteria.time_sec_limit = 10.0
      full_termination_criteria.iteration_limit = 1

      @test FirstOrderLp.check_termination_criteria(
        full_termination_criteria,
        qp_cache,
        iteration_stats_dont_terminate1,
      ) == FirstOrderLp.TERMINATION_REASON_ITERATION_LIMIT

      full_termination_criteria.iteration_limit = 10
      full_termination_criteria.kkt_matrix_pass_limit = 40.0
      @test FirstOrderLp.check_termination_criteria(
        full_termination_criteria,
        qp_cache,
        iteration_stats_dont_terminate1,
      ) == FirstOrderLp.TERMINATION_REASON_KKT_MATRIX_PASS_LIMIT
    end
  end
end


struct ResizableViewVector{T,Data<:AbstractVector{T}} <: AbstractVector{T}
    data::Data
    offset::Int
end

function resizableview(data::AbstractVector, offset::Integer)
    @argcheck firstindex(data) - 1 <= offset <= lastindex(data)
    return ResizableViewVector(data, offset)
end

resizabletail(data::AbstractVector) = resizableview(data, lastindex(data))

Base.size(xs::ResizableViewVector) = (length(xs.data) - xs.offset,)

Base.@propagate_inbounds Base.getindex(xs::ResizableViewVector, i::Int) =
    xs.data[xs.offset+i]

Base.@propagate_inbounds Base.setindex!(xs::ResizableViewVector, x, i::Int) =
    xs.data[xs.offset+i] = x

function Base.resize!(xs::ResizableViewVector, n::Integer)
    resize!(xs.data, xs.offset + n)
    return xs
end

function Base.push!(xs::ResizableViewVector, x)
    push!(xs.data, x)
    return xs
end


#include("time_series_opf.jl")

# function to plot the given data files with given plot choice parameters
#
# function parameters
#################################################
# month_st, day_st, period_st, month_en, day_en, period_en: the start and end time for the period of data to plot
# br_arr_raw: the array containg the indexex of subnets to plot (in format of Int64). If using the single for plot_choice, the function only reads in the first index for default.
# folder: the target folder containg the data files for plot
# output_folder: the folder specified to save the output files
# plot_choice::String = "single": default single,
#       "single": optimal result & frequency of a network in a range
#       "compare": compare average value of multiple networks

# all network indexes:
# [1 2 3 4 5 6 8 9 10 11 12 13 14 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
# 43 44 45 46 47 49 50 51 52 53 54 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
# 83 84 85 87 88 89 90 91 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
# 118 119]

# the function of plotting results of multifrequency-opf on different netwoks and in different periods
function multi_plot_new(month_st, day_st, period_st, month_en, day_en, period_en, br_arr_raw, folder, output_folder,
      plot_choice::String = "single")

      # time range
      start_t = DateTime(2020,month_st,day_st,period_st)
      end_t = DateTime(2020,month_en,day_en,period_en)
      range_t = start_t:Hour(1):end_t

      all_range = DateTime(2020,1,1,1):Hour(1):DateTime(2020,12,31,24)
      part_range = DateTime(2020, month_st, day_st, period_st):Hour(1):DateTime(2020,month_en, day_en, period_en)
      # num_br: the number of branches we want to plot
      num_br = max(size(br_arr_raw, 1), size(br_arr_raw, 2))

      # br_result: the optimization result of the subnets (temporary storage)
      # res_summary_dict: the final data structure of optimization results of the subnets
      # br_arr: array of the indexes of all the subnets into string format (for further indexing format)
      br_result = Dict()
      res_summary_dict = Dict()
      br_arr = Array{String, 1}()
      for i = 1:num_br
            push!(br_arr, string(br_arr_raw[i]))
      end

      base_info = JSON.parsefile("D:/github/VariableFrequencyOPF/time_series_out_modified/res_summary_base_(1.1.1__12.31.24).json")
      # base_info_part = Dict()
      # for i = 1:size(part_range, 1)
      #       base_info_part[i] = deepcopy(base_info)
      # end
      for br_ind = 1:num_br
            # input result summary from specified input folder (output folder of the optimization function)
            #dict_info = JSON.parsefile("$(folder)/res_summary_br$(br_arr[br_ind])_(1.1.1__12.31.24).json")
            #dict_info = JSON.parsefile("$(folder)/res_summary_br$(br_arr[br_ind])_(7.15.1_7.15.24).json")
            dict_info = JSON.parsefile("$(folder)/res_summary_br$(br_arr[br_ind])_(1.1.1__12.31.24).json")
            br_result[br_ind] = dict_info
            # the result dictionary can be of different data structure
            res_summary_dict[br_arr[br_ind]] = deepcopy(br_result[br_ind])

            # if (plot_choice == "comp")
            #       # modify all the types of the fields to be Float64
            #       for i in 1:size(all_range, 1)
            #             # convert the optimal value to Float64
            #             if typeof(res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][20][1]) == String
            #                   res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][20][1] =
            #                        parse(Float64, res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][20][1])
            #             end
            #
            #             # # convert the optimal frequency to Float64
            #             # if typeof(res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][2][1]) == String
            #             #       res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][2][1] =
            #             #            parse(Float64, res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][2][1])
            #             # end
            #       end

            # normalize the data formats
            # loop through the locations of results from the output file of optimization function
            # ( the data format need to be changed for different )
            for i in 1:size(part_range, 1)
            #elseif (plot_choice == "single")

                  # convert the optimal value to Float64
                  # if typeof(res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1]) == String
                  #       res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1] =
                  #            parse(Float64, res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1])
                  # end

                  # convert the optimal frequency to Float64
                  if typeof(res_summary_dict[br_arr[br_ind]]["$(i)"]["frequency (Hz)"][1]) == String
                        res_summary_dict[br_arr[br_ind]]["$(i)"]["frequency (Hz)"][1] =
                             parse(Float64, res_summary_dict[br_arr[br_ind]]["$(i)"]["frequency (Hz)"][1])
                            #FIXME parse(Float64, res_summary_dict[br_arr[br_ind]]["$(i)"]["f"][1])
                  end

                  # if typeof(res_summary_dict[br_arr[br_ind]]["$(i)"]["f"][1]) == String
                  #       res_summary_dict[br_arr[br_ind]]["$(i)"]["f"][1] =
                  #            parse(Float64, res_summary_dict[br_arr[br_ind]]["$(i)"]["f"][1])
                  #           #FIXME parse(Float64, res_summary_dict[br_arr[br_ind]]["$(i)"]["f"][1])
                  # end
            end

      end

      # change the type of the object for base case

      for i in 1:size(all_range, 1)
            if typeof(base_info["$(i)"]["objective"][1]) == String
                  base_info["$(i)"]["objective"][1] =
                       parse(Float64, base_info["$(i)"]["objective"][1])
            end
      end

      # the dictionaries to store output values (objective value/frequency/size), "_n" represents the array of points
      # where the solver did not find objective values
      objval = Dict()
      objval_n = Dict()
      obj_freq = Dict()
      obj_freq_n = Dict()
      size_obj = Array{DateTime, 1}()
      size_obj_n = Array{DateTime, 1}()
      objval_base = Array{Float64, 1}()

      # this is the single array that store the data for single branch for comparison to base case
      objval_all = Dict()
      size_obj_all = Array{DateTime, 1}()
      #objfreq_base = Array{Float64, 1}()

      #if (plot_choice == "comp")
            for br_ind = 1:num_br
                  objval[br_ind] = Array{Float64, 1}()
                  objval_n[br_ind] = Array{Float64, 1}()
                  obj_freq[br_ind] = Array{Float64, 1}()
                  obj_freq_n[br_ind] = Array{Float64, 1}()
                  objval_all[br_ind] = Array{Float64, 1}()
            end
      #end
      #
      # for i in 1:size(range_t, 1)
      #       if ((res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][22][1] in ["LOCALLY_SOLVED", "ALMOST_LOCALLY_SOLVED"])
      #          && (res_summary_dict[br_arr[br_ind]]["$(i)"]["columns"][22][2] in ["LOCALLY_SOLVED", "ALMOST_LOCALLY_SOLVED"]))
      #          push!(size_obj[br_ind], range_t[i])
      #       else
      #          push!(size_obj_n[br_ind], range_t[i])
      #       end
      # end

      # find the starting time for the indexing below based on input time range and the whole year time range
      start_t_ind = findfirst(isequal(start_t), part_range)
      start_all_ind = findfirst(isequal(start_t), all_range)
      println("$(start_t_ind)")
      println("$(start_all_ind)")

      # for plot average value comparison:

      # here divide each time index into feasible array and infeasible array
      # index start from the start time above and loop for the size of the range we want to plot for
      for i in start_t_ind:(start_t_ind + size(range_t, 1) - 1)
            push!(size_obj_all, range_t[i-start_t_ind + 1])
            if ((res_summary_dict[br_arr[1]]["$(i)"]["status"] in ["LOCALLY_SOLVED", "ALMOST_LOCALLY_SOLVED"]))
               push!(size_obj, range_t[i-start_t_ind + 1])
            else
               push!(size_obj_n, range_t[i-start_t_ind + 1])
               println("infeasible$(i), $(res_summary_dict[br_arr[1]]["$(i)"]["status"])")
            end
      end

      # store data for base case

      for i in start_t_ind:(start_t_ind + size(range_t, 1) - 1)
            all_ind = start_all_ind + i - 1
            println(all_ind)
            push!(objval_base, base_info["$(all_ind)"]["objective"][1])
      end

      # store the values of objective values
      for br_ind = 1:num_br
            for i in start_t_ind:(start_t_ind + size(range_t, 1) - 1)
                  #println("absd$(i)")
                  push!(objval_all[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1])

                  # if ((res_summary_dict[br_arr[br_ind]]["$(i)"]["status"] in ["LOCALLY_SOLVED", "ALMOST_LOCALLY_SOLVED"]))
                  #    push!(objval[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1])
                  #    push!(obj_freq[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["frequency (Hz)"][1])
                  #    #push!(size_obj[br_ind], range_t[i])
                  # else
                  #    push!(objval_n[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1])
                  #    push!(obj_freq_n[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["frequency (Hz)"][1])
                  #    #push!(size_obj_n[br_ind], range_t[i])
                  # end

                  if ((res_summary_dict[br_arr[br_ind]]["$(i)"]["status"] in ["LOCALLY_SOLVED", "ALMOST_LOCALLY_SOLVED"]))
                     push!(objval[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1])
                     push!(obj_freq[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["frequency (Hz)"][1])
                     #push!(size_obj[br_ind], range_t[i])
                  else
                     push!(objval_n[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["objective"][1])
                     push!(obj_freq_n[br_ind], res_summary_dict[br_arr[br_ind]]["$(i)"]["frequency (Hz)"][1])
                     #push!(size_obj_n[br_ind], range_t[i])
                  end
            end
      end

      # string = JSON.json(res_summary_dict)
      # open("test.json", "w") do f
      #     write(f, string)
      # end

      #size_obj_n = size(range_t, 1) - size_obj
      # below are the parts of plotting
      #println(range_t)
      #println(objval)

      # plot if the plot choice is "single", which means we plot the average optimal values and frequency
      # versus the branch numbers.
      if (plot_choice == "single")
            gr()

            # plot for the optimal cost

            # p = bar(size_obj, objval[1],  color = :blue,  linecolor = :transparent, title = "optimal result: br$(br_arr[1])",
            #           xlabel = "time/h", ylabel = "optimal cost", titlefontsize=20,
            #           right_margin = 10px)
            #          #xtickfontsize=18,ytickfontsize=18,
            # plot!(size_obj_n, objval_n[1], color = :red, seriestype = :scatter)
            # if ((num_br - 1) > 0)
            #       for br_ind = 2:num_br
            #             plot!(size_obj, objval[br_ind], color = :green, seriestype = :line)
            #             plot!(size_obj_n, objval_n[br_ind], color = :yellow, seriestype = :line)
            #       end
            # end

            # plot for the optimal frequency
            p2 = plot(size_obj, obj_freq[1], color = :blue, seriestype = :line, title = "optimal frequency: br$(br_arr[1])",
                     xlabel = "time/h", ylabel = "optimal frequency", titlefontsize=20,
                     right_margin = 10px, labels = ["feasible" "infeasible"])
            plot!(size_obj_n, obj_freq_n[1], color = :red, seriestype = :scatter,
                  markersize = 2.5)
            if ((num_br - 1) > 0)
                  for br_ind = 2:num_br
                        plot!(size_obj, obj_freq[br_ind], color = :green, seriestype = :line)
                        plot!(size_obj_n, obj_freq_n[br_ind], color = :blue, seriestype = :line)
                  end
            end

            # get the data difference compared to base case and store to array
            base_diff = Array{Float64, 1}()
            base_diff_percent = Array{Float64, 1}()
            objval_all_mod = Array{Float64, 1}()

            number_bins = 30
            total_cost_improve = 0
            total_cost_base = 0

            # FIXME get the max improve rate
            max_improve = 0;

            for i = 1:size(range_t, 1)
                  push!(base_diff, objval_base[i] - objval_all[1][i])
                  push!(objval_all_mod, objval_all[1][i]-25000)
                  push!(base_diff_percent, (objval_base[i] - objval_all[1][i])/objval_base[i])
                  total_cost_improve += (objval_base[i] - objval_all[1][i]);
                  if ((objval_base[i] - objval_all[1][i]) < 0)
                        println("time point $(i) is not right.")
                  end

                  # calculate the total cost of the base case
                  total_cost_base += objval_base[i];
            end

            max_base_diff_percent = maximum(base_diff_percent)
            min_base_diff_percent = minimum(base_diff_percent)
            println("The maximum percent of difference is $(max_base_diff_percent)")

            total_improve_rate = total_cost_improve/total_cost_base
            #bins_use=range(min_base_diff_percent,max_base_diff_percent,length=number_bins)
            bins=range(min_base_diff_percent,max_base_diff_percent,length=number_bins)

            base_comp_data = [base_diff objval_all[1]]
            base1 = JSON.json(base_comp_data)
            open("time_series_out/opt_res_plot_single_10_(1.1.1__1.1.3).json", "w") do f
              write(f, base1)
            end
            # zeros(Float64, size(range_t, 1), 2)
            # for i = 1:size(range_t, 1)
            #       push!(base_diff, objval_base[i] - objval_all[1][i])
            #       base_comp_data[i][1] = objval_all[1][i]
            #       base_comp_data[i][2] = base_diff[i]
            # end

            p1 = groupedbar(base_comp_data, bar_position = :stack, bar_width=0.7, linecolor = :transparent,
                        title = "optimal value VS base: br$(br_arr[1])",
                      xlabel = "time/h", ylabel = "opt val diff", titlefontsize=20,
                      right_margin = 10px)
            # p3 = plot(size_obj_all, base_diff, color = :yellow, seriestype = :bar, title = "optimal value VS base",
            #          xlabel = "time/h", ylabel = "optimal value difference", right_margin = 10px, labels = ["feasible" "infeasible"])

            # combine two plots together on a single figure

            p4 = bar(size_obj_all, base_diff_percent, line_color = :transparent, title = "Rate of improvement: br$(br_arr[1])",
                        linecolor = :transparent, xlabel = "time/h", ylabel = "rate improved", titlefontsize=20,
                        right_margin = 10px)

            p5 = histogram(base_diff_percent, bins = bins, linecolor = :transparent, yaxis=(:log10), title = "Num of hours: br$(br_arr[1])",
                        xlabel = "rate of improvements (cost reduceed/base cost)", ylabel = "amount of hours", titlefontsize=20,
                        right_margin = 10px)

            p_result = plot(p1, p2, layout = (2,1), legend = false)

            savefig(p_result, "$(output_folder)/opt_res_plot_single_$(br_arr[1])_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).pdf")

            p_improve = plot(p4, p5, layout = (2,1), legend = false)

            savefig(p_improve, "$(output_folder)/opt_improve_single_$(br_arr[1])_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).pdf")

            println("the current total improvement rate is $(total_improve_rate)")
      end

      # plot if the plot choice is "compare", which means we plot the average optimal values and frequency
      # versus the time steps (the comparison between different ).
      if (plot_choice == "compare")
            #objval_ave_dic = Dict()
            objval_ave = Array{Float64, 1}()
            opt_f_ave = Array{Float64, 1}()
            # objval_ave_dic[1] = Array{String, 1}()
            # objval_ave_dic[2] = Array{Float64, 1}()

            # the base value to help make the plot easy to analyze ( show the differences between the different cases)
            base_val_plot = 25000
            for br_ind = 1:num_br
                  # for i in 1:size()
                  # push!(objval_ave_dic[1], br_arr[br_ind])
                  # push!(objval_ave_dic[2], sum(objval[br_ind])/size(objval[br_ind], 1))
                  push!(objval_ave, (sum(objval[br_ind])/size(objval[br_ind], 1)) - base_val_plot)
                  push!(opt_f_ave, (sum(obj_freq[br_ind])/size(obj_freq[br_ind], 1)))
                  #end
            end
            #sort(collect(objval_ave_dic), by=x->x[2])
            ind = sortperm(objval_ave)
            println(ind)

            objval_order = Array{Float64, 1}()
            subnet_order = Array{Int64, 1}()
            for i = 1:num_br
                  push!(subnet_order, br_arr_raw[ind[i]])
                  push!(objval_order, objval_ave[ind[i]])
            end
            println(subnet_order)
            println(objval_order)
            println("result is above")

            net_order = JSON.json(subnet_order)
            val_order = JSON.json(objval_order)
            open("$(output_folder)/opt_data_comp_$(num_br)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
                  write(f, net_order)
                  write(f, "\n")
                  write(f, val_order)
            end
            # string = JSON.json(res_summary_dict)
            # open("test.json", "w") do f
            #     write(f, string)
            # end

            gr()
            # p1 = plot(br_arr[ind], objval_ave[ind],  color = :blue, seriestype = :bar, title = "partial optimal result plot ($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en))",
            #          xlabel = "subnet number", ylabel = "opt_val - $(base_val_plot)", xtickfontsize=18,ytickfontsize=18,titlefontsize=20,right_margin = 10px, legend = false)
            # p2 = plot(br_arr[ind], opt_f_ave[ind], color = :blue, seriestype = :bar, title = "partial optimal frequency plot ($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en))",
            #          xlabel = "subnet number", ylabel = "optimal f", xtickfontsize=18,ytickfontsize=18,titlefontsize=20, right_margin = 10px, legend = false)
            #
            # p_all = plot(p1, p2, layout = (2,1), legend = false)
            # savefig(p_all, "$(output_folder)/opt_res_plot_multi_comp_$(num_br)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).pdf")

            # get the data difference compared to base case and store to array
            base_diff = Dict()
            base_diff_percent = Dict()
            base_diff_percent_posi = Dict()
            base_diff_percent_all = Dict()
            objval_all_mod = Dict()

            for br_ind = 1:num_br
                  base_diff[br_ind] = Array{Float64, 1}()
                  base_diff_percent[br_ind] = Array{Float64, 1}()
                  base_diff_percent_posi[br_ind] = Array{Float64, 1}()
                  base_diff_percent_all[br_ind] = Array{Float64, 1}()
                  objval_all_mod[br_ind] = Array{Float64, 1}()
            end

            number_bins = 30
            total_cost_improve = 0
            total_cost_base = 0

            # FIXME get the max improve rate
            max_improve = 0;
            for br_ind = 1:num_br
                  for i = 1:size(range_t, 1)
                        push!(base_diff[br_ind], objval_base[i] - objval_all[br_ind ][i])
                        push!(objval_all_mod[br_ind], objval_all[br_ind ][i]-25000)
                        push!(base_diff_percent[br_ind], (objval_base[i] - objval_all[br_ind ][i])/objval_base[i])
                        if ((objval_base[i] - objval_all[br_ind ][i])/objval_base[i])>0
                              push!(base_diff_percent_posi[br_ind], (objval_base[i] - objval_all[br_ind ][i])/objval_base[i])
                              push!(base_diff_percent_all[br_ind], (objval_base[i] - objval_all[br_ind ][i])/objval_base[i])
                        else
                              push!(base_diff_percent_all[br_ind], 0.0)
                        end
                        #total_cost_improve += (objval_base[i] - objval_all[br_ind ][i]);
                        if ((objval_base[i] - objval_all[br_ind][i]) < 0)
                              println("br$(br_ind), time point $(i) is not right.")
                        end

                        # calculate the total cost of the base case
                        #total_cost_base += objval_base[i];
                  end
            end

            max_base_diff_percent = 0;
            for br_ind = 1:num_br
                  for i = 1:size(base_diff_percent[br_ind], 1)
                        if max_base_diff_percent < base_diff_percent[br_ind][i]
                              max_base_diff_percent = base_diff_percent[br_ind][i]
                        else
                              #
                        end
                  end
            end
            # min_base_diff_percent = minimum(base_diff_percent)
            println("The maximum percent of difference is $(max_base_diff_percent)")

            #total_improve_rate = total_cost_improve/total_cost_base
            #bins_use=range(min_base_diff_percent,max_base_diff_percent,length=number_bins)
            bins=range(0,max_base_diff_percent,length=number_bins)

            # base_comp_data = Dict()
            # for br_ind = 1:num_br
            #       push!(base_comp_data, [base_diff[br_ind] objval_all[1][br_ind]])
            # end
            # base1 = JSON.json(base_diff_percent)
            # open("opt_res_plot_single_10_(1.1.1__12.31.24).json", "w") do f
            #   write(f, base1)
            # end
            # zeros(Float64, size(range_t, 1), 2)
            # for i = 1:size(range_t, 1)
            #       push!(base_diff, objval_base[i] - objval_all[1][i])
            #       base_comp_data[i][1] = objval_all[1][i]
            #       base_comp_data[i][2] = base_diff[i]
            # end

            p3 = bar(size_obj_all, base_diff_percent_all[1], line_color = :transparent, title = "rate of improvement: br$(br_arr[1])",
                       linecolor = :transparent, xlabel = "time/h", ylabel = "rate improved", titlefontsize=20,
                       right_margin = 10px, )
            p4 = bar(size_obj_all, base_diff_percent_all[2], line_color = :transparent, title = "rate of improvement: br$(br_arr[2])",
                       linecolor = :transparent, xlabel = "time/h", ylabel = "rate improved", titlefontsize=20,
                       right_margin = 10px)
            p5 = bar(size_obj_all, base_diff_percent_all[3], line_color = :transparent, title = "rate of improvement: br$(br_arr[3])",
                       linecolor = :transparent, xlabel = "time/h", ylabel = "rate improved", titlefontsize=20,
                       right_margin = 10px)
            p6 = bar(size_obj_all, base_diff_percent_all[4], line_color = :transparent, title = "rate of improvement: br$(br_arr[4])",
                        linecolor = :transparent, xlabel = "time/h", ylabel = "rate improved", titlefontsize=20,
                        right_margin = 10px)

            p7 = histogram(base_diff_percent_posi[1], bins = bins, linecolor = :transparent, yaxis=(:log10), title = "br$(br_arr[1])",
                       xlabel = "rate of improvements (cost reduceed/base cost)", ylabel = "amount of hours", titlefontsize=20,
                       right_margin = 10px, xlim = max_base_diff_percent)
            p8 = histogram(base_diff_percent_posi[2], bins = bins, linecolor = :transparent, yaxis=(:log10), title = "br$(br_arr[2])",
                       xlabel = "rate of improvements (cost reduceed/base cost)", ylabel = "amount of hours", titlefontsize=20,
                       right_margin = 10px, xlim = max_base_diff_percent)
            p9 = histogram(base_diff_percent_posi[3], bins = bins, linecolor = :transparent, yaxis=(:log10), title = "br$(br_arr[3])",
                       xlabel = "rate of improvements (cost reduceed/base cost)", ylabel = "amount of hours", titlefontsize=20,
                       right_margin = 10px, xlim = max_base_diff_percent)
            p10 = histogram(base_diff_percent_posi[4], bins = bins, linecolor = :transparent, yaxis=(:log10), title = "br$(br_arr[4])",
                       xlabel = "rate of improvements (cost reduceed/base cost)", ylabel = "amount of hours", titlefontsize=20,
                       right_margin = 10px, xlim = max_base_diff_percent)
            #Plots.link_axes!()
            p_first = plot(p3, p4, p5, p6, layout = (4,1), link = :all,legend = false, size = (500, 800))
            savefig(p_first, "$(output_folder)/opt_improve_plot_comp_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).pdf")

            # title = "amount of hours with each rate of improvements"
            p_second = plot(p7, p8, p9, p10, layout = (4,1), link = :x,legend = false, size = (500, 800) )
            savefig(p_second, "$(output_folder)/opt_rate_plot_comp_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).pdf")
            #for br_ind = 1:num_br
            #end
      end

      println("The objective value of the current period")
end


"""
	grdblend(cmd0::String="", arg1=nothing, arg2=nothing, kwargs...)

Reads a listing of grid files and blend parameters, or up to 2 GTMgrid types, and creates
a grid by blending the other grids using cosine-taper weights.

Full option list at [`grdblend`]($(GMTdoc)grdblend.html)

Parameters
----------

- $(GMT.opt_I)
    ($(GMTdoc)grdblend.html#i)
- $(GMT.opt_R)
- **G** | **save** | **outgrid** | **outfile** :: [Type => Str]

    Output grid file name. Note that this is optional and to be used only when saving
    the result directly on disk. Otherwise, just use the G = grdblend(....) form.
    ($(GMTdoc)grdblend.html#g)

- **C** | **clobber** :: [Type => Str | []]      ``Arg = f|l|o|u[±]``

    Clobber mode: Instead of blending, simply pick the value of one of the grids that covers a node.
    ($(GMTdoc)grdproject.html#c)
- **N** | **nodata** :: [Type => Str | Number]

    No data. Set nodes with no input grid to this value [Default is NaN].
    ($(GMTdoc)grdblend.html#n)
- **Q** | **headless** :: [Type => Bool]

    Create plain header-less grid file (for use with external tools). Requires that the output
    grid file is a native format (i.e., not netCDF). DO NOT USE WITH **G**.
    ($(GMTdoc)grdblend.html#q)
- **W** | **no_blend** :: [Type => Str | []]

    Do not blend, just output the weights used for each node [Default makes the blend].
    Append ``z`` to write the weight*z sum instead.
    ($(GMTdoc)grdblend.html#w)
- **Z** | **scale** :: [Type => Number]

    Scale output values by scale before writing to file.
    ($(GMTdoc)grdblend.html#z)
- $(GMT.opt_V)
- $(GMT.opt_f)
- $(GMT.opt_n)
- $(GMT.opt_r)
"""
function grdblend(cmd0::String="", arg1=nothing, arg2=nothing; kwargs...)

	d = init_module(false, kwargs...)[1]		# Also checks if the user wants ONLY the HELP mode
	cmd, = parse_common_opts(d, "", [:G :I :R :V_params :f :n :r])
	cmd  = parse_these_opts(cmd, d, [[:C :clobber], [:N :nodata], [:Q :headless], [:W :no_blend], [:Z :scale]])

	cmd, got_fname, arg1, arg2 = find_data(d, cmd0, cmd, arg1, arg2)
	return common_grd(d, "grdblend " * cmd, arg1, arg2)		# Finish build cmd and run it
end

# ---------------------------------------------------------------------------------------------------
grdblend(arg1, arg2=nothing, cmd0::String=""; kw...) = grdblend(cmd0, arg1, arg2; kw...)


module CellMaps

using Test
using CellwiseValues
using CellwiseValues.CellValues: _test_iter_cell_value

export CellMap
export IterCellMap
export IndexCellMap
import CellwiseValues: evaluate
export test_iter_cell_map
export test_index_cell_map
export test_index_cell_map_with_index_arg

import Base: iterate
import Base: length
import Base: size
import Base: getindex

"""
Abstract object that traverses a set of cells and at every cell returns a
`Map{S,M,T,N}`
"""
const IterCellMap{S,M,T,N,R<:Map{S,M,T,N}} = IterCellValue{R}

"""
Abstract array indexed by cells that returns a `Map{S,M,T,N}`
"""
const IndexCellMap{S,M,T,N,C,R<:Map{S,M,T,N}} = IndexCellValue{R,C}

"""
Abstract object that for a given cell index returns a `Map{S,M,T,N}`
"""
const CellMap{S,M,T,N} = Union{IterCellMap{S,M,T,N},IndexCellMap{S,M,T,N}}

# evaluation

"""
Return the cellwise maps of a `CellMap` on a cellwise set of points
"""
function evaluate(cm::CellMap{S,M,T,N},ca::CellArray{S,M}) where {S,M,T,N}
  IterCellMapValue(cm,ca)
end

function evaluate(cm::IndexCellMap{S,M,T,N},ca::IndexCellArray{S,M}) where {S,M,T,N}
  IndexCellMapValue(cm,ca)
end

struct IterCellMapValue{T,N,V,A} <: IterCellArray{T,N,CachedArray{T,N,Array{T,N}}}
  cm::V
  ca::A
end

function IterCellMapValue(cm::CellMap{S,M,T,N},ca::CellArray{S,M}) where {S,M,T,N}
  V = typeof(cm)
  A = typeof(ca)
  IterCellMapValue{T,N,V,A}(cm,ca)
end

length(v::IterCellMapValue) = length(v.ca)

@inline function iterate(v::IterCellMapValue{T,N}) where {T,N}
  cmnext = iterate(v.cm)
  canext = iterate(v.ca)
  r = CachedArray(T,N)
  _iterate(cmnext,canext,r)
end

@inline function iterate(v::IterCellMapValue,state)
  cmstate, castate, r = state
  cmnext = iterate(v.cm,cmstate)
  canext = iterate(v.ca,castate)
  _iterate(cmnext,canext,r)
end

@inline function _iterate(cmnext,canext,r)
  if cmnext === nothing; return nothing end
  if canext === nothing; return nothing end
  m, cmstate = cmnext
  a, castate = canext
  s = return_size(m,size(a))
  setsize!(r,s)
  evaluate!(m,a,r)
  state = (cmstate,castate,r)
  (r, state)
end

struct IndexCellMapValue{T,N,V,A} <: IndexCellArray{T,N,CachedArray{T,N,Array{T,N}},1}
  cm::V
  ca::A
  r::CachedArray{T,N,Array{T,N}}
end

function IndexCellMapValue(cm::IndexCellMap{S,M,T,N},ca::IndexCellArray{S,M}) where {S,M,T,N}
  V = typeof(cm)
  A = typeof(ca)
  r = CachedArray(T,N)
  IndexCellMapValue{T,N,V,A}(cm,ca,r)
end

length(v::IndexCellMapValue) = length(v.ca)

size(v::IndexCellMapValue) = (length(v),)

function getindex(v::IndexCellMapValue,i::Integer)
  m = v.cm[i]
  a = v.ca[i]
  s = return_size(m,size(a))
  setsize!(v.r,s)
  evaluate!(m,a,v.r)
  v.r
end

# Testers

function test_iter_cell_map(
  m::CellMap{S,M,T,N},
  a::CellArray{S,M},
  b::AbstractArray{<:AbstractArray{T,N}}) where {S,M,T,N}

  @test length(m) == length(b)
  @test length(m) == length(a)

  c = evaluate(m,a)
  _test_iter_cell_value(c,b)

  for (mi,ai,bi) in zip(m,a,b)
    @assert isa(mi,Map{S,M,T,N})
    @assert evaluate(mi,ai) ≈ bi
    @assert typeof(mi) == eltype(m)
  end

end

function test_index_cell_map(
  m::IndexCellMap{S,M,T,N},
  a::CellArray{S,M},
  b::AbstractArray{<:AbstractArray{T,N}}) where {S,M,T,N}

  @test size(m) == size(b)

  @test IndexStyle(m) == IndexStyle(b)

  test_iter_cell_map(m,a,b)

  for (i,ai) in enumerate(a)
    mi = m[i]
    bi = b[i]
    @assert isa(mi,Map{S,M,T,N})
    @assert evaluate(mi,ai) ≈ bi
    @assert typeof(mi) == eltype(m)
  end

end

function test_index_cell_map_with_index_arg(
  m::IndexCellMap{S,M,T,N},
  a::IndexCellArray{S,M},
  b::AbstractArray{<:AbstractArray{T,N}}) where {S,M,T,N}

  test_index_cell_map(m,a,b)

  c = evaluate(m,a)
  @test isa(c,IndexCellArray)

  @test length(c) == length(m)
  @test length(c) == length(a)

  for i in 1:length(c)
    mi = m[i]
    bi = b[i]
    ci = c[i]
    @assert bi ≈ ci
  end

  for i in 1:length(m)
    mi = m[i]
    ai = a[i]
    bi = b[i]
    @assert isa(mi,Map{S,M,T,N})
    @assert evaluate(mi,ai) ≈ bi
    @assert typeof(mi) == eltype(m)
  end

end

end # module CellMaps


```
Snapshot information for a currency.
```
function ticker(id::String)
  res = HTTP.get(ENDPOINT * "products/$id/ticker")
  rawData = JSON.parse(String(res.body))
  df = DataFrame(rawData)
  df.ask = parse.(Float64, df.ask)
  df.bid = parse.(Float64, df.bid)
  df.size = parse.(Float64, df.size)
  df.volume = parse.(Float64, df.volume)
  df.price = parse.(Float64, df.price)
  #df.time = DateTime.(chop.(df.time), dateformat"yyyy-mm-ddTHH:MM:SS.ssssss")
  df
end


"""
Modia module for elaboration of models.

* Developer: Hilding Elmqvist, Mogram AB
* First version: July-August 2016
* Copyright (c) 2016-2018: Hilding Elmqvist, Toivo Henningsson, Martin Otter
* License: MIT (expat)
"""
module ModelElaboration

const PrintOriginalModel = false # Needed for the moment to set partial attribute.
const PrintInstantiated = false
const PrintFlattened = true
const PrintElaborated = true
const PrintJSON = false
const fileStdOut = false

# ----------------------------------------------------------------------------

#using Debug
using Zygote
using ..Instantiation
using ..Execution
using Base.Meta: quot, isexpr
using Base: isapprox
using DataStructures
import ..Instantiation: GetField, This, Der, Symbolic, vars_of, Instance
import ..Execution: logTiming
using ..Synchronous
using ..BLTandPantelides
using ..BLTandPantelidesUtilities
using ..BasicStructuralTransform
using ..StructuralTransform
using ..SymbolicTransform
using ..Utilities
using ..ModiaLogging
#= Temporarily removed due to problem with PyPlot
using PyPlot
=#
import ModiaMath

using JSON

@static if VERSION < v"0.7.0-DEV.2005"
else
    using Printf
end

@static if VERSION < v"0.7.0-DEV.2005"
    using Base.Test
else
    using Test
end

export elaborate, prettyPrint, simulateModel, simulate, simulate_der, skewCoords, skew, residue, residue_der, modiaCross, showExpr, transformModel, simulateMultiModeModel, allInstances
# export modiaSwitches, defineSwitch, setSwitch, showSwitches, getSwitch,
export checkSimulation

const useIncidenceMatrix = BasicStructuralTransform.useIncidenceMatrix
const elaborate = true
#@show elaborate

const noResult = Dict{Symbol,AbstractArray{T,1} where T}()

allInstances(s) = return s

function dummyDer(e)
    f = e
    if typeof(f) == Instantiation.GetField
        f = f.name
    end

    if typeof(f) == Instantiation.Der
        f = dummyDer(f.base)
    end
    s = string("der_", f)
    return Symbol(s)
end

function dummyState(e)
    f = e
    if typeof(f) == Instantiation.GetField
        f = f.name
    end

    if typeof(f) == Instantiation.Der
        f = dummyState(f.base)
    end
    string(f)
end

function substituteDer(ex, dummyDerivatives, unassigned)
    if typeof(ex) == Instantiation.Der
        newname = dummyDer(ex.base)
        dumSt = dummyState(ex)
        unass = map(string, unassigned)

        if dumSt in unass
            # println("substituteDer for unassigned: ", newname)
            ex = GetField(This(), newname)
        elseif newname in dummyDerivatives
            #  println("substituteDer: ", newname)
            ex = GetField(This(), newname)
        else
            ex
        end

    elseif typeof(ex) == Expr
        Expr(ex.head, [substituteDer(arg, dummyDerivatives, unassigned) for arg in ex.args]...)
    else
        ex
    end
end

function elaborateModel(flat_model)
    loglnModia("elaborate:")
    solved_model = flat_model

    #  print("Elaborate:          ")
    #  @time
    (equations, variables, assign, components, Avar, Bequ, states, deriv, unassignedNames, incidenceMatrix, varSizes, varTypes, equSizes, equTypes) = StructuralTransform.transformStructurally(flat_model)

    if equations == nothing # Connector
        return
    end

    flat_model.equations = equations

    loglnModia("\nSYMBOLIC PROCESSING")

    if logTiming
        print("Symbolic processing:   ")
        @time solvedComponents, algebraic = differentiateSortedEquations(equations, variables, components, assign, Avar, Bequ)
    else
        solvedComponents, algebraic = differentiateSortedEquations(equations, variables, components, assign, Avar, Bequ)
    end

    if length(SymbolicTransform.dummyDerivatives.keys) > 0
        printSymbolList("\nDummy derivatives", SymbolicTransform.dummyDerivatives.keys)
    end

    unassignedVar = []
    solved_model.equations = []
    initSynchronousCounters()
    previousVariables = []

    for c in solvedComponents
        for (e, solved) in c
            e = substituteClocked(previousVariables, e)
            e = substituteDer(e, SymbolicTransform.dummyDerivatives.keys, unassignedVar)

            if !solved # typeof(e.args[1]) == Instantiation.Der || length(c) > 1   #
                # Use equation
                push!(solved_model.equations, Expr(:(=), e.args[1], e.args[2]))
            else
                # Use assignment
                push!(solved_model.equations, Expr(:(:=), e.args[1], e.args[2]))
            end
        end
    end

    # solved_model.variables = [solved_model.variables; SymbolicTransform.dummyDerivatives]

    for d in SymbolicTransform.dummyDerivatives.keys
    # Remove der_
        v = string(d)
        if length(v) > 4 && v[1:4] == "der_"
            v = v[5:end]
        end

        if length(v) > 4 && v[1:4] == "der_"
            v = v[5:end]
        end

        vsize = varSizes[Symbol(v)]
        st = if vsize == (); 0.0 else zeros(vsize) end
        solved_model.variables[d] = Instantiation.Variable(start=st)
    end

    #= For test:
    for d in map(string, unassignedVar)
        @show Symbol(d)
        solved_model.variables[Symbol("der_"*d)] = Instantiation.Variable()  # add typ
        solved_model.variables[Symbol("der_der_"*d)] = Instantiation.Variable()
    end
    =#

    solved_model.initial_pre = []
    solved_model.F_pre = []
    solved_model.F_post = []

    for p in previousVariables
        push!(solved_model.initial_pre, :($(p.name) = 0))
        push!(solved_model.F_pre, :($(p.name) = 0))
        push!(solved_model.F_post, :(dummy = $(Synchronous.updatePrevious)($(p.name), $(SymbolicTransform.simulationModel_symbol), 1)))
    end

    if PrintElaborated
        loglnModia("\nSTRUCTURALLY AND SYMBOLICALLY PROCESSED MODEL")
        loglnModia("  Sorted equations, := used for solved algebraic variables")
        showInstance(solved_model)
        loglnModia()
    end

    return solved_model
end

# Handle allInstances operator
function allInstances!(all, instName, inst, var, class)
    for key in keys(inst.variables)
        if isa(inst.variables[key], Instance)
            allInstances!(all, key, inst.variables[key], var, class)
        else
            if inst.model_name == class && key == var
                push!(all, instName)
            end
        end
    end
end

function substituteAllInstances(ex, modified_model, class, flat)
    if typeof(ex) == Expr && ex.head == :call && ex.args[1] == allInstances
        all = []
        name = string(ex.args[2].name)

@static if VERSION < v"0.7.0-DEV.2005"
        name = name[rsearchindex(name, ".") + 1:end]
else
        name = name[first(something(findlast(".", name), 0:-1)) + 1:end]
end

        name = Symbol(name)

        if !flat
            loglnModia("Found name: $name in $class")
            Expr(ex.head, ex.args..., class)
        else
            allInstances!(all, :(), modified_model, name, ex.args[3])
            loglnModia("all = $all")
            ref = [GetField(This(), Symbol(string(b) * "." * string(name))) for b in all]
            loglnModia("ref = $ref")
            Expr(:vect, ref...)
        end

    elseif typeof(ex) == Expr
        Expr(ex.head, [substituteAllInstances(arg, modified_model, class, flat) for arg in ex.args]...)
    else
        ex
    end
end

function traverseAndSubstituteAllInstances(instName, modified_model, inst, flat)
    for key in keys(inst.variables)
        if isa(inst.variables[key], Instance)
            traverseAndSubstituteAllInstances(key, modified_model, inst.variables[key], flat)
        end
    end

    newEqu = []
    for e in inst.equations
        newE = substituteAllInstances(e, modified_model, inst.model_name, flat)
        if newE != e
            # @show e
            # @show newE
    end
        push!(newEqu, newE)
    end
    inst.equations = newEqu
end

function simulateModelWithOptions(model, t; options=Dict())
    opt = Dict(options)
    ModiaLogging.setDefaultLogName(string(model.name))
    ModiaLogging.setOptions(opt)
    StructuralTransform.setOptions(opt)
    BasicStructuralTransform.setOptions(opt)
    Instantiation.setOptions(opt)
    Execution.setOptions(opt)

    logSimulation = false
    if haskey(opt, :logSimulation)
        logSimulation = opt[:logSimulation]
        @show logSimulation
        delete!(opt, :logSimulation)
    end

    relTol = 1e-4
    if haskey(opt, :relTol)
        relTol = opt[:relTol]
        @show relTol
        delete!(opt, :relTol)
    end

    hev = 1e-8
    if haskey(opt, :hev)
        hev = opt[:hev]
        @show hev
        delete!(opt, :hev)
    end

    if length(keys(opt)) > 0
        println("Option(s) not found: ", keys(opt))
    end

    if fileStdOut
        @static if VERSION < v"0.7.0-DEV.2005"
            originalSTDOUT = STDOUT
        else
            originalSTDOUT = stdout
        end
        (outRead, outWrite) = redirect_stdout()
    end

    openLogModia()

    start = time_ns()
    println("check")
    setDerAsFunction(false)
    if BasicStructuralTransform.logStatistics
        println("\nSimulating model: ", model.name)
    end
    loglnModia("\nSimulating model: ", model.name)

    if PrintOriginalModel
        loglnModia("ORIGINAL MODEL:")
        showModel(model)
    end

    if logTiming
        print("Instantiate:           ")
        @time modified_model = instantiate(model, first(t))
    else
        modified_model = instantiate(model, first(t))
    end
    traverseAndSubstituteAllInstances(:(), modified_model, modified_model, false)

    if PrintInstantiated
        loglnModia("INSTANTIATED MODEL:")
        showInstance(modified_model)
        loglnModia()
    end

    if PrintJSON
        name = string(modified_model.model_name)
        file = open(name * ".JSON", "w")
        printJSON(file, modified_model, name, name)
        close(file)
    end

    if logTiming
        print("Flatten:               ")
        @time flat_model = flatten(modified_model)
    else
        flat_model = flatten(modified_model)
    end
    traverseAndSubstituteAllInstances(:(), modified_model, flat_model, true)

    if PrintFlattened
        loglnModia("\nFLATTENED MODEL")
        showInstance(flat_model)
        loglnModia()
    end

    if elaborate
        solved_model = elaborateModel(flat_model)
        if solved_model == nothing
            return nothing
        end
    else
        solved_model = flat_model
    end

    #=
    #  JSON.json(t::This) = "this."

    if PrintJSONsolved
       jsonSolved = JSON.json(solved_model.equations[1])
        @show jsonSolved
    end
    =#
    @static if VERSION < v"0.7.0-DEV.2005"
        disableSimulation = false
    else
        disableSimulation = false
    end

    if !BasicStructuralTransform.newStateSelection && !disableSimulation ## Disable simulation for the moment
        loglnModia("\nSIMULATION")
        # @show incidenceMatrix
        t = vec(collect(Float64, t))
        if logTiming
            print("Code generation and simulation:         ")
            # @show solved_model t useIncidenceMatrix logSimulation
            @time res = simulate_ida(solved_model, t, if useIncidenceMatrix; incidenceMatrix else nothing end, log=logSimulation, relTol=relTol, hev=hev)
        else
            res = simulate_ida(solved_model, t, if useIncidenceMatrix; incidenceMatrix else nothing end, log=logSimulation, relTol=relTol, hev=hev)
        end
    else
        res = Dict{Symbol,AbstractArray{T,1} where T}()
    end

    setDerAsFunction(true)

    if fileStdOut
        #  close(outWrite)
        output = readavailable(outRead)
        close(outRead)
        redirect_stdout(originalSTDOUT)
        print(String(output))
    end

    if logTiming
        println(@sprintf("Total time: %0.3f", (time_ns() - start) * 1E-9), " seconds")
    end
    closeLogModia()
    println("something")
    println("noRes = $noResult")
    println("res = $res")
    #test_expr!("@test", ex, kws...)
    #orig_ex = Expr(:inert, ex)
    #result = get_test_result(ex, __source__)
    #:(do_test($result, $orig_ex))
    if res != noResult
        return res
    else
        throw(DomainError("res = $res"))
    end
    return res
end

function simulateModelWithOptionsDer(model, t, var, var_val, key; options=Dict())
    opt = Dict(options)
    ModiaLogging.setDefaultLogName(string(model.name))
    ModiaLogging.setOptions(opt)
    StructuralTransform.setOptions(opt)
    BasicStructuralTransform.setOptions(opt)
    Instantiation.setOptions(opt)
    Execution.setOptions(opt)

    logSimulation = false
    if haskey(opt, :logSimulation)
        logSimulation = opt[:logSimulation]
        @show logSimulation
        delete!(opt, :logSimulation)
    end

    relTol = 1e-4
    if haskey(opt, :relTol)
        relTol = opt[:relTol]
        @show relTol
        delete!(opt, :relTol)
    end

    hev = 1e-8
    if haskey(opt, :hev)
        hev = opt[:hev]
        @show hev
        delete!(opt, :hev)
    end

    if length(keys(opt)) > 0
        println("Option(s) not found: ", keys(opt))
    end

    if fileStdOut
        @static if VERSION < v"0.7.0-DEV.2005"
            originalSTDOUT = STDOUT
        else
            originalSTDOUT = stdout
        end
        (outRead, outWrite) = redirect_stdout()
    end

    openLogModia()

    start = time_ns()
    println("check")
    setDerAsFunction(false)
    if BasicStructuralTransform.logStatistics
        println("\nSimulating model: ", model.name)
    end
    loglnModia("\nSimulating model: ", model.name)

    if PrintOriginalModel
        loglnModia("ORIGINAL MODEL:")
        showModel(model)
    end

    if logTiming
        print("Instantiate:           ")
        @time modified_model = instantiate(model, first(t))
    else
        modified_model = instantiate(model, first(t))
    end
    traverseAndSubstituteAllInstances(:(), modified_model, modified_model, false)

    if PrintInstantiated
        loglnModia("INSTANTIATED MODEL:")
        showInstance(modified_model)
        loglnModia()
    end

    if PrintJSON
        name = string(modified_model.model_name)
        file = open(name * ".JSON", "w")
        printJSON(file, modified_model, name, name)
        close(file)
    end

    if logTiming
        print("Flatten:               ")
        @time flat_model = flatten(modified_model)
    else
        flat_model = flatten(modified_model)
    end
    traverseAndSubstituteAllInstances(:(), modified_model, flat_model, true)

    if PrintFlattened
        loglnModia("\nFLATTENED MODEL")
        showInstance(flat_model)
        loglnModia()
    end

    if elaborate
        solved_model = elaborateModel(flat_model)
        if solved_model == nothing
            return nothing
        end
    else
        solved_model = flat_model
    end

    #=
    #  JSON.json(t::This) = "this."

    if PrintJSONsolved
       jsonSolved = JSON.json(solved_model.equations[1])
        @show jsonSolved
    end
    =#
    @static if VERSION < v"0.7.0-DEV.2005"
        disableSimulation = false
    else
        disableSimulation = false
    end

    if !BasicStructuralTransform.newStateSelection && !disableSimulation ## Disable simulation for the moment
        loglnModia("\nSIMULATION")
        # @show incidenceMatrix
        println("model = ", typeof(solved_model.variables))
        res = Dict{AbstractString,Any}
        println("key = $key")
        k = String(key)
        t = vec(collect(Float64, t))

        sim_der(a) = simulate_ida_der(solved_model, var, a, k, t, if useIncidenceMatrix; incidenceMatrix else nothing end, log=logSimulation, relTol=relTol, hev=hev)

        g = gradient(Params(t)) do
         sim_der(var_val)
       end

          if logTiming
                print("Code generation and simulation:         ")
                # @show solved_model t useIncidenceMatrix logSimulation
                @time res = g(t)#simulate_ida(solved_model, t, if useIncidenceMatrix; incidenceMatrix else nothing end, log=logSimulation, relTol=relTol, hev=hev)
            else
                s = g(t)#sim_der'(var_val)
                println(s)
                res = s#simulate_ida(solved_model, t, if useIncidenceMatrix; incidenceMatrix else nothing end, log=logSimulation, relTol=relTol, hev=hev)
            end

    else
        res = Dict{Symbol,AbstractArray{T,1} where T}()
    end

    setDerAsFunction(true)

    if fileStdOut
        #  close(outWrite)
        output = readavailable(outRead)
        close(outRead)
        redirect_stdout(originalSTDOUT)
        print(String(output))
    end

    if logTiming
        println(@sprintf("Total time: %0.3f", (time_ns() - start) * 1E-9), " seconds")
    end
    closeLogModia()
    println("something")
    println("noRes = $noResult")
    println("res = $res")
    #test_expr!("@test", ex, kws...)
    #orig_ex = Expr(:inert, ex)
    #result = get_test_result(ex, __source__)
    #:(do_test($result, $orig_ex))
    if res != noResult
        return res
    else
        throw(DomainError("res = $res"))
    end
    return res
end

function simulateModel(model, t; options...)
    return simulateModelWithOptions(model, t, options=options)
end

function simulate(model, stopTime; startTime=0, options...)
    nSteps = 1000
    @static if VERSION < v"0.7.0-DEV.2005"
        t = linspace(startTime, stopTime, nSteps)
    else
        t = range(startTime, stop=stopTime, length=nSteps)
    end

    return simulateModelWithOptions(model, t, options=options)
end

function simulate_der(model, stopTime, var, var_val, key; startTime=0, options...)
    nSteps = 1000
    @static if VERSION < v"0.7.0-DEV.2005"
        t = linspace(startTime, stopTime, nSteps)
    else
        t = range(startTime, stop=stopTime, length=nSteps)
    end

    return simulateModelWithOptionsDer(model, t, var, var_val, key, options=options)
end

function print_rgb(r, g, b, t)
    println("\e[1m\e[38;2;$r;$g;$b;249m", t)
end

@static if VERSION < v"0.7.0-DEV.2005"
    printstyled(s; bold=false, color=:black) = print_with_color(color, s, bold=bold)
end
"""
    function checkSimulation(mod, stopTime, observer, finalSolution; options...)
Simulates model mod until stopTime and checks that the final value of the observer variable is approximately finalSolution. The following options are available:

* `logTranslation`: determines if logging of the translation is performed
* ...
"""
function checkSimulation(mod, stopTime, observer="", finalSolution=0.0; startTime=0, options...)
    nSteps = 1000
    @static if VERSION < v"0.7.0-DEV.2005"
        t = linspace(startTime, stopTime, nSteps)
    else
        t = range(startTime, stop=stopTime, length=nSteps)
    end
    res = nothing

    try
        res = simulateModelWithOptions(mod, t, options=options)
    catch err
        st = stacktrace(catch_backtrace())
        closeLogModia()

        setTestStatus(false)
        println()
        println("\n----------------------\n")
        println()
            printstyled("Simulation FAILED:", bold=true, color=:red); println()
        if isa(err, ErrorException)
            printstyled(err.msg, bold=true, color=:red); println()
        elseif isa(err, UndefVarError)
            printstyled(err, bold=true, color=:red); println()
            ModiaLogging.increaseLogCategory(:(UndefinedSymbol))
        else
            printstyled(err, bold=true, color=:red); println()
        end
        println()
        println("\n----------------------\n")
        println()
        println("Stack trace: ------------------------------------------------")
        for l in st
            println(l)
        end
        println("End stack trace: --------------- ----------------------------")
        @test false
        return noResult
    end

    final = nothing
    if res != nothing && res != noResult && observer != "" && haskey(res, observer)
        obs = res[observer]
        if length(obs) > 0
            final = obs[end]
        end
    end

    if final != nothing
        println("final $observer = $final")
        ok = finalSolution == nothing || typeof(final) == Float64 && Base.isapprox(final, finalSolution, rtol=1.0E-3) || final == finalSolution
        @test ok

        if !ok
            setTestStatus(false)
            println("final solution $observer = $finalSolution")
            printstyled("Simulation NOT OK", bold=true, color=:red); println()
            println()
#= Replaced PyPlot solution with ModiaMath.plot(..) call
            figure()
            title("Simulation NOT OK in " * string(mod.name))
            plot(res["time"], res[observer])
            grid(true)
            xlabel("time [s]")
            legend([observer],  loc="center right")
=#
            ModiaMath.plot(res, observer, heading="Simulation NOT OK in " * string(mod.name))
            false
        else
            ModiaLogging.increaseLogCategory(:(CalculatedResult))
            setTestStatus(true)
            printstyled("Simulation OK", bold=true, color=:green); println()
            println()
            true
        end

    else
        setTestStatus(true)
        printstyled("Simulation RAN", bold=true, color=:green); println()
        println()
        true
        @test true
    end
    res
end

# ----------------------------

"""
Experimental code for multi-mode handling with impulses.
"""
function simulateMultiModeModel(model, t0, t1; n=1000, m=100, options...)
    opt = Dict(options)
    ModiaLogging.setDefaultLogName(string(model.name))
    ModiaLogging.setOptions(opt)
    StructuralTransform.setOptions(opt)
    BasicStructuralTransform.setOptions(opt)
    Instiantiation.setOptions(opt)
    Execution.setOptions(opt)
    if length(keys(opt)) >= 0
        println("Option(s) not found: ", keys(opt))
    end

    println("\nSimulating model: ", model.name)
    loglnModia("\nSimulating model: ", model.name)

    dt = (t1 - t0) / m
    @static if VERSION < v"0.7.0-DEV.2005"
        times1 = linspace(t0, t0 + dt, 100)
    else
        times1 = range(t0, stop=t0 + dt, length=100)
    end

    model1 = flatten(instantiate(model, t0, Dict()))
    #  model1 = elaborateModel(flatten(instantiate(model, t0, Dict())))
    println("\nSub-simulation 1; startTime=$t0, interval=$dt")
    res1 = simulate_ida(model1, times1, if false; nothing else nothing end, log=false)
    tn = t0 + dt

    for step in 2:m
        println("\nSub-simulation $step; startTime=$tn, interval=$dt")

        # extract final values

        # finals = [name => values[end] for (name,values) in res1]
        finals = Dict(name => if length(values) > 0; values[end] else false end for (name, values) in res1)

        @static if VERSION < v"0.7.0-DEV.2005"
            times2 = linspace(tn, tn + dt, 100)
        else
            times2 = range(tn, stop=tn + dt, length=100)
        end
        model2 = flatten(instantiate(model, tn, Dict()))

        # carry over final values
        vars = vars_of(model2)

        for (name, final) in finals
            name = Symbol(name)
            if haskey(vars, name)
                var = vars[name]
                if isa(var, Variable)
                    var.start = final
#                   @show name final
                end
            end
        end

        res2 = simulate_ida(model2, times2, if false; nothing else nothing end, log=false)
        for (name, values) in res1
            res1[name] = vcat(res1[name], res2[name])
        end
        tn += dt
    end

    return res1
end

end


module Simple

using Purple
using SymbolicUtils: Sym

b(x::Int) = 10 + x
function fn(x::Int, y::Int)
    z = 0
    z += x + y
    return b(z) + 10
end

println("Before:")
src = lift(fn,
           Tuple{Int, Int};
           clean = true,
           jit = false,
           opt = false)
display(src)

println("After:")
src = lift(fn,
           Tuple{Int, Int};
           jit = false,
           opt = false)
display(src)

f = lift(fn,
         Tuple{Int, Int};
         jit = true,
         opt = true)
ret = f(Sym{Int}(:x), Sym{Int}(:y))
display(ret)

end # module


module SIMDIndexing

using SIMD
using Requires

const VecOfIntegers{N, T <: SIMD.IntegerTypes} = Vec{N, T}

abstract type AbstractIndex end
const Index = Union{Int, VecOfIntegers, VecRange, AbstractIndex}

function __init__()
    @require(MappedArrays="dbb5928d-eab1-5f90-85c2-b9b0edb7c900",
             include("mappedarrays.jl"))
    @require(FillArrays="1a297f60-69ca-5386-bcde-b61e274b549b",
             include("fillarrays.jl"))
end

end # module


module TestRules

using  Test
import PlayingCards: Card
import Cribbage.Rules: cardvalue, validhands, validplays

@testset "Test cardvalue" begin

    @test cardvalue(Card("5♥")) == 5
    @test cardvalue(Card("A♠")) == 1
    @test cardvalue(Card("10♣")) == 10
    @test cardvalue(Card("J♦")) == 10
    @test cardvalue(Card("Q♥")) == 10
    @test cardvalue(Card("K♦")) == 10

end

@testset "Test validhands" begin

    originalhand = tuple(
        Card("5♥"), Card("A♠"), Card("10♣"), Card("J♦"), Card("Q♥")
    )
    hands = validhands(originalhand)
    @test length(hands) == 5
    for hand in hands
        @test length(hand) == 4
        for card in hand
            @test card ∈ originalhand
        end
    end

    originalhand = tuple(
        Card("5♥"), Card("A♠"), Card("10♣"), Card("J♦"), Card("Q♥"), Card("K♦")
    )
    hands = validhands(originalhand)
    @test length(hands) == 15
    for hand in hands
        @test length(hand) == 4
        for card in hand
            @test card ∈ originalhand
        end
    end
end

@testset "Test validplays" begin

    prior = tuple()
    hand = tuple(
        Card("5♥"), Card("10♣"), Card("J♦"), Card("Q♥")
    )
    possibleplays = validplays(prior, hand)
    @test length(possibleplays) == 4
    for card in possibleplays
        @test card ∈ hand
    end

    prior = tuple(Card("5♥"), Card("K♦"))
    hand = tuple(
        Card("10♣"), Card("J♦"), Card("Q♥")
    )
    possibleplays = validplays(prior, hand)
    @test length(possibleplays) == 3
    for card in possibleplays
        @test card ∈ hand
    end

    prior = tuple(Card("5♥"), Card("K♦"), Card("10♣"), Card("4♣"))
    hand = tuple(
        Card("J♦"), Card("Q♥")
    )
    possibleplays = validplays(prior, hand)
    @test length(possibleplays) == 0

    prior = tuple(Card("5♥"), Card("K♦"), Card("10♣"), Card("4♣"))
    hand = tuple()
    possibleplays = validplays(prior, hand)
    @test length(possibleplays) == 0
end

end  # module TestRules


const Shift = 1
const ReduceL = 2
const ReduceR = 3

type State
    tokens::Vector{Token}
    step::Int
    top::Int
    right::Int
    left::State
    lc::State
    rc::State

    State(tokens) = new(tokens, 1, 1, 2)
end

Base.done(s::State) = isdefined(s,:left) && s.right == length(s.tokens)+1

function Base.next(s::State, acts::Vector{Int})
    scores = s.scorefun(s, acts)
    states = State[]
    for i = 1:length(acts)
        push!(states, expand(s, acts[i], scores[i]))
    end
    states
end

function Base.next(s::State, act::Int, score::Float64)
    top, lcs, rcs, left, right = begin
        if act == shift
            s.right, Int[], Int[], s, s.right+1
        elseif act == reducel
            _lcs = length(s.lcs) == 0 ? [s.left.top] : [s.left.top,s.lcs[1]]
            s.top, _lcs, s.rcs, s.left.left, s.right
        elseif act == reducer
            _rcs = length(s.left.rcs) == 0 ? [s.top] : [s.top,s.left.rcs[1]]
            s.left.top, s.left.lcs, _rcs, s.left.left, s.right
        else
            throw("Invalid action: $(act)")
        end
    end
    score = s.score + score
    State(top, lcs, rcs, left, right, s.step+1, s, act, s.tokens, s.scorefun, score, nothing)
end

function expand_gold(s::State)
    isfinal(s) && return State[]
    if s.left == nothing
        act = shift
    else
        tokens = s.tokens
        s0, s1 = tokens[s.top], tokens[s.left.top]
        if s1.headid == s.top
            act = reducel
        elseif s0.headid == s.left.top
            reducable = all(i -> tokens[i].headid != s.top, s.right:length(tokens))
            act = reducable ? reducer : shift
        elseif s.right <= length(tokens)
            act = shift
        else
            throw("Invalid")
        end
    end
    expand(s, [act])
end

function expand_pred(s::State)
    isfinal(s) && return State[]
    acts = Int[]
    s.left == nothing || push!(acts, reducel, reducer)
    s.right <= length(s.tokens) && push!(acts, shift)
    expand(s, acts)
end

function toheads(s::State)
    heads = fill(0, length(s.tokens))
    while s != nothing
        length(s.lcs) > 0 && (heads[s.lcs[1]] = s.top)
        length(s.rcs) > 0 && (heads[s.rcs[1]] = s.top)
        s = s.prev
    end
    heads
end


"""
A dense key table stores the data as a single dictionary of row-tuples.
"""
immutable DenseKeyTable{Index,DataTypes,Key,KeyType} <: AbstractTable{Index,Key}
    data::Dict{KeyType,Row{Index,DataTypes}}

    function DenseKeyTable(data_in)
        check_row(Index,DataTypes)
        check_key(Key,KeyType)
        new(data_in)
    end
end