get_test_curve1() = [CurvePt(VecSE2(0.0,0.0,0.0), 0.0),
                     CurvePt(VecSE2(1.0,0.0,0.0), 1.0),
                     CurvePt(VecSE2(3.0,0.0,0.0), 3.0)]

function get_test_roadway()
    roadway = Roadway()

    seg1 = RoadSegment(1,
        [Lane(LaneTag(1,1),
            [CurvePt(VecSE2(-2.0,0.0,0.0), 0.0),
             CurvePt(VecSE2(-1.0,0.0,0.0), 1.0)]),
         Lane(LaneTag(1,2),
            [CurvePt(VecSE2(-2.0,1.0,0.0), 0.0),
             CurvePt(VecSE2(-1.0,1.0,0.0), 1.0)]),
         Lane(LaneTag(1,3),
            [CurvePt(VecSE2(-2.0,2.0,0.0), 0.0),
             CurvePt(VecSE2(-1.0,2.0,0.0), 1.0)])
        ])

    seg2 = RoadSegment(2,
        [Lane(LaneTag(2,1),
            [CurvePt(VecSE2(0.0,0.0,0.0), 0.0),
             CurvePt(VecSE2(1.0,0.0,0.0), 1.0),
             CurvePt(VecSE2(2.0,0.0,0.0), 2.0),
             CurvePt(VecSE2(3.0,0.0,0.0), 3.0)]),
         Lane(LaneTag(2,2),
            [CurvePt(VecSE2(0.0,1.0,0.0), 0.0),
             CurvePt(VecSE2(1.0,1.0,0.0), 1.0),
             CurvePt(VecSE2(2.0,1.0,0.0), 2.0),
             CurvePt(VecSE2(3.0,1.0,0.0), 3.0)]),
         Lane(LaneTag(2,3),
            [CurvePt(VecSE2(0.0,2.0,0.0), 0.0),
             CurvePt(VecSE2(1.0,2.0,0.0), 1.0),
             CurvePt(VecSE2(2.0,2.0,0.0), 2.0),
             CurvePt(VecSE2(3.0,2.0,0.0), 3.0)])
         ])

    seg3 = RoadSegment(3,
        [Lane(LaneTag(3,1),
            [CurvePt(VecSE2(4.0,0.0,0.0), 0.0),
             CurvePt(VecSE2(5.0,0.0,0.0), 1.0)]),
         Lane(LaneTag(3,2),
            [CurvePt(VecSE2(4.0,1.0,0.0), 0.0),
             CurvePt(VecSE2(5.0,1.0,0.0), 1.0)]),
         Lane(LaneTag(3,3),
            [CurvePt(VecSE2(4.0,2.0,0.0), 0.0),
             CurvePt(VecSE2(5.0,2.0,0.0), 1.0)])
        ])

    for i in 1:3
        connect!(seg1.lanes[i], seg2.lanes[i])
        connect!(seg2.lanes[i], seg3.lanes[i])
    end

    push!(roadway.segments, seg1)
    push!(roadway.segments, seg2)
    push!(roadway.segments, seg3)

    roadway
end

@testset "1d roadway" begin
    roadway = StraightRoadway(20.0)
    s = 10.0
    @test mod_position_to_roadway(s, roadway) == s
    s = 25.0
    @test mod_position_to_roadway(s, roadway) == 5.0
    s = 45.0
    @test mod_position_to_roadway(s, roadway) == 5.0
    s = -5.0
    @test mod_position_to_roadway(s, roadway) == 15.0
    s_rear = 10.0
    s_fore = 15.0
    @test get_headway(s_rear, s_fore, roadway) == 5.0
    s_fore = 25.0
    @test get_headway(s_rear, s_fore, roadway) == 15.0
    s_fore = 5.0
    @test get_headway(s_rear, s_fore, roadway) == 15.0
end

@testset "Curves" begin
    p = lerp(CurvePt(VecSE2(0.0,0.0,0.0), 0.0), CurvePt(VecSE2(1.0,2.0,3.0), 4.0), 0.25)
    show(IOBuffer(), p)
    @test isapprox(convert(VecE2, p.pos), VecE2(0.25, 0.5))
    @test isapprox(p.pos.θ, 0.75)
    @test isapprox(p.s, 1.0)

    show(IOBuffer(), CurveIndex(1,0.0))

    @test isapprox(get_lerp_time(VecE2(0.0,0.0), VecE2(1.0,0.0), VecE2(1.0,0.0)), 1.0)
    @test isapprox(get_lerp_time(VecE2(0.0,0.0), VecE2(1.0,0.0), VecE2(0.5,0.0)), 0.5)
    @test isapprox(get_lerp_time(VecE2(0.0,0.0), VecE2(1.0,0.0), VecE2(0.5,0.5)), 0.5)
    @test isapprox(get_lerp_time(VecE2(0.0,0.0), VecE2(1.0,1.0), VecE2(0.5,0.5)), 0.5)
    @test isapprox(get_lerp_time(VecE2(1.0,0.0), VecE2(2.0,0.0), VecE2(1.5,0.5)), 0.5)
    @test isapprox(get_lerp_time(VecE2(0.0,0.0), VecE2(-1.0,0.0), VecE2(1.0,0.0)), 0.0)
    @test isapprox(get_lerp_time(VecE2(0.0,0.0), VecE2(-1.0,0.0), VecE2(-0.75,0.0)), 0.75)

    curve = get_test_curve1()
    @inferred index_closest_to_point(curve, VecE2(0.0,0.0))
    @test index_closest_to_point(curve, VecE2(0.0,0.0)) == 1
    @test index_closest_to_point(curve, VecE2(1.0,0.0)) == 2
    @test index_closest_to_point(curve, VecE2(2.1,0.0)) == 3
    @test index_closest_to_point(curve, VecE2(0.49,0.0)) == 1
    @test index_closest_to_point(curve, VecE2(1.9,-100.0)) == 2
    @test index_closest_to_point(curve, VecSE2(1.9,-100.0,0.0)) == 2
    @test index_closest_to_point(curve, VecSE2(-1.0,0.0,0.0)) == 1
    @test index_closest_to_point([CurvePt(VecSE2(0.0,0.0,0.0),0.0)], VecE2(0.0,0.0)) == 1
    @test get_curve_index(curve, 0.0) == CurveIndex(1, 0.0)
    @test get_curve_index(curve, 1.0) == CurveIndex(2, 0.0)
    @test get_curve_index(curve, 1.5) == CurveIndex(2, 0.25)
    @test get_curve_index(curve, 3.5) == CurveIndex(2, 1.0)
    @test get_curve_index(curve,-0.5) == CurveIndex(1, 0.0)

    p = curve[CurveIndex(1, 0.75)]
    @test isapprox(convert(VecE2, p.pos), VecE2(0.75, 0.0))
    @test isapprox(p.pos.θ, 0.0)
    @test isapprox(p.s, 0.75)

    @test get_curve_index(CurveIndex(1, 0.0), curve,  0.25) == CurveIndex(1,0.25)
    @test get_curve_index(CurveIndex(1, 0.0), curve,  1.25) == CurveIndex(2,0.125)
    @test get_curve_index(CurveIndex(1, 0.5), curve,  1.50) == CurveIndex(2,0.5)
    @test get_curve_index(CurveIndex(1, 0.5), curve, -0.25) == CurveIndex(1,0.25)
    @test get_curve_index(CurveIndex(2, 0.0), curve, -0.50) == CurveIndex(1,0.50)

    res = proj(VecSE2(0.0,0.0,0.0), curve)
    @inferred proj(VecSE2(0.0,0.0,0.0), curve)
    show(IOBuffer(), res)
    @test res.ind == CurveIndex(1, 0.0)
    @test isapprox(res.t, 0.0)
    @test isapprox(res.ϕ, 0.0)
    res = proj(VecSE2(0.25,0.5,0.1), curve)
    @inferred proj(VecSE2(0.25,0.5,0.1), curve)
    @test res.ind == CurveIndex(1, 0.25)
    @test isapprox(res.t, 0.5)
    @test isapprox(res.ϕ, 0.1)
    res = proj(VecSE2(0.25,-0.5,-0.1), curve)
    @test res.ind == CurveIndex(1, 0.25)
    @test isapprox(res.t, -0.5)
    @test isapprox(res.ϕ, -0.1)
    res = proj(VecSE2(1.5,0.5,-0.1), curve)
    @test res.ind == CurveIndex(2, 0.25)
    @test isapprox(res.t,  0.5)
    @test isapprox(res.ϕ, -0.1)
    res = proj(VecSE2(10.0,0.0,0.0), curve)
    @test res.ind == CurveIndex(length(curve)-1, 1.0)
    @test isapprox(res.t, 0.0)
    @test isapprox(res.ϕ, 0.0)

    curve2 = [CurvePt(VecSE2(-1.0,-1.0,π/4), 0.0),
              CurvePt(VecSE2( 1.0, 1.0,π/4), hypot(2.0,2.0)),
              CurvePt(VecSE2( 3.0, 3.0,π/4), hypot(4.0,4.0))]

   res = proj(VecSE2(0.0,0.0,0.0), curve2)
   @test res.ind == CurveIndex(1, 0.5)
   @test isapprox(res.t, 0.0)
   @test isapprox(res.ϕ, -π/4)
end # curve tests

@testset "roadways" begin
    curve = get_test_curve1()

    lanetag = LaneTag(1,1)
    lane = Lane(lanetag, curve)
    show(IOBuffer(), lanetag)

    @test !has_next(lane)
    @test !has_prev(lane)

    show(IOBuffer(), RoadIndex(CurveIndex(1,0.0), lanetag))

    roadway = get_test_roadway()

    io = IOBuffer()
    show(io, roadway)
    close(io)

    lane = roadway[LaneTag(1,1)]
    @test has_next(lane)
    @test !has_prev(lane)
    @test lane.exits[1].target == RoadIndex(CurveIndex(1,0.0), LaneTag(2,1))

    lane = roadway[LaneTag(2,1)]
    @test has_next(lane)
    @test has_prev(lane)
    @test lane.exits[1].target == RoadIndex(CurveIndex(1,0.0), LaneTag(3,1))
    @test lane.entrances[1].target == RoadIndex(CurveIndex(1,1.0), LaneTag(1,1))

    res = proj(VecSE2(1.0,0.0,0.0), lane, roadway)
    @inferred proj(VecSE2(1.0,0.0,0.0), lane, roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(1.0,0.0,0.0), lane, roadway, move_along_curves=false)
    @inferred proj(VecSE2(1.0,0.0,0.0), lane, roadway, move_along_curves=false)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(1.5,0.25,0.1), lane, roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.5)
    @test isapprox(res.curveproj.t, 0.25)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == lane.tag

    res = proj(VecSE2(0.0,0.0,0.0), lane, roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(-0.75,0.0,0.0), lane, roadway)
    @test res.curveproj.ind == CurveIndex(0, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(-1.75,0.0,0.0), lane, roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == prev_lane(lane, roadway).tag

    res = proj(VecSE2(-1.75,0.0,0.0), lane, roadway, move_along_curves=false)
    @test res.curveproj.ind == CurveIndex(0, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(4.25,0.2,0.1), lane, roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == next_lane(lane, roadway).tag

    res = proj(VecSE2(4.25,0.2,0.1), lane, roadway, move_along_curves=false)
    @test res.curveproj.ind == CurveIndex(4, 1.0)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == lane.tag

    res = proj(VecSE2(3.25,0.2,0.1), lane, roadway)
    @test res.curveproj.ind == CurveIndex(4, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == lane.tag

    res = proj(VecSE2(4.25,0.2,0.1), prev_lane(lane, roadway), roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == next_lane(lane, roadway).tag

    res = proj(VecSE2(-0.75,0.0,0.0), next_lane(lane, roadway), roadway)
    @test res.curveproj.ind == CurveIndex(0, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    ####

    seg = roadway[2]

    res = proj(VecSE2(1.0,0.0,0.0), seg, roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(1.5,0.25,0.1), seg, roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.5)
    @test isapprox(res.curveproj.t, 0.25)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == lane.tag

    res = proj(VecSE2(0.0,0.0,0.0), seg, roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(-0.75,0.0,0.0), seg, roadway)
    @test res.curveproj.ind == CurveIndex(0, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(-1.75,0.0,0.0), seg, roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == prev_lane(lane, roadway).tag

    res = proj(VecSE2(4.25,0.2,0.1), seg, roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == next_lane(lane, roadway).tag

    res = proj(VecSE2(3.25,0.2,0.1), seg, roadway)
    @test res.curveproj.ind == CurveIndex(4, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == lane.tag

    res = proj(VecSE2(4.25,0.2,0.1), roadway[1], roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == next_lane(lane, roadway).tag

    res = proj(VecSE2(-0.75,0.0,0.0), roadway[3], roadway)
    @test res.curveproj.ind == CurveIndex(0, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(1.0,1.0,0.0), seg, roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == LaneTag(2,2)

    res = proj(VecSE2(1.0,2.0,0.0), seg, roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == LaneTag(2,3)

    ###

    res = proj(VecSE2(1.0,0.0,0.0), roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(1.5,0.25,0.1), roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.5)
    @test isapprox(res.curveproj.t, 0.25)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == lane.tag

    res = proj(VecSE2(0.0,0.0,0.0), roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(-0.75,0.0,0.0), roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == prev_lane(lane, roadway).tag

    res = proj(VecSE2(-1.75,0.0,0.0), roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == prev_lane(lane, roadway).tag

    res = proj(VecSE2(4.25,0.2,0.1), roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == next_lane(lane, roadway).tag

    res = proj(VecSE2(3.25,0.2,0.1), roadway)
    @test res.curveproj.ind == CurveIndex(4, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == lane.tag

    res = proj(VecSE2(4.25,0.2,0.1), roadway[1], roadway)
    @test res.curveproj.ind == CurveIndex(1, 0.25)
    @test isapprox(res.curveproj.t, 0.2)
    @test isapprox(res.curveproj.ϕ, 0.1)
    @test res.tag == next_lane(lane, roadway).tag

    res = proj(VecSE2(-0.75,0.0,0.0), roadway[3], roadway)
    @test res.curveproj.ind == CurveIndex(0, 0.25)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == lane.tag

    res = proj(VecSE2(1.0,1.0,0.0), roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == LaneTag(2,2)

    res = proj(VecSE2(1.0,2.0,0.0), roadway)
    @test res.curveproj.ind == CurveIndex(2, 0.0)
    @test isapprox(res.curveproj.t, 0.0)
    @test isapprox(res.curveproj.ϕ, 0.0)
    @test res.tag == LaneTag(2,3)

    ###

    roadind_0 = RoadIndex(CurveIndex(1, 0.0), LaneTag(2,1))
    roadind = move_along(roadind_0, roadway, 0.0)
    @inferred move_along(roadind_0, roadway, 0.0)
    @test roadind == roadind_0

    roadind = move_along(roadind_0, roadway, 1.0)
    @test roadind == RoadIndex(CurveIndex(2,0.0), LaneTag(2,1))

    roadind = move_along(roadind_0, roadway, 1.25)
    @test roadind == RoadIndex(CurveIndex(2,0.25), LaneTag(2,1))

    roadind = move_along(roadind_0, roadway, 3.0)
    @test roadind == RoadIndex(CurveIndex(3,1.0), LaneTag(2,1))

    roadind = move_along(roadind_0, roadway, 4.0)
    @test roadind == RoadIndex(CurveIndex(1,0.0), LaneTag(3,1))

    roadind = move_along(roadind_0, roadway, 4.5)
    @test roadind == RoadIndex(CurveIndex(1,0.5), LaneTag(3,1))

    roadind = move_along(roadind_0, roadway, 3.75)
    @test roadind == RoadIndex(CurveIndex(0,0.75), LaneTag(3,1))

    roadind = move_along(roadind_0, roadway, -1.0)
    @test roadind == RoadIndex(CurveIndex(0,0.0), LaneTag(2,1))

    roadind = move_along(roadind_0, roadway, -1.75)
    @test roadind == RoadIndex(CurveIndex(1,0.25), LaneTag(1,1))

    roadind = move_along(roadind_0, roadway, -0.75)
    @test roadind == RoadIndex(CurveIndex(0,0.25), LaneTag(2,1))

    roadind = move_along(roadind_0, roadway, -Inf)
    @test roadind == RoadIndex(CurveIndex(1,0.0), LaneTag(1,1))

    roadind = move_along(roadind_0, roadway, Inf)
    @test roadind == RoadIndex(CurveIndex(1,1.0), LaneTag(3,1))

    ####

    @test n_lanes_right(roadway[LaneTag(2,1)], roadway) == 0
    @test n_lanes_right(roadway[LaneTag(2,2)], roadway) == 1
    @test n_lanes_right(roadway[LaneTag(2,3)], roadway) == 2
    @test n_lanes_left(roadway[LaneTag(2,1)], roadway) == 2
    @test n_lanes_left(roadway[LaneTag(2,2)], roadway) == 1
    @test n_lanes_left(roadway[LaneTag(2,3)], roadway) == 0

    ####
    roadway = get_test_roadway()
    all_lanes = lanes(roadway)
    all_lane_tags = lanetags(roadway)
    @test all_lane_tags == [l.tag for l in all_lanes]
    @test all_lanes == Lane{Float64}[roadway[tag] for tag in all_lane_tags]
    for i=1:2
        for j=1:3
            @test roadway[LaneTag(i,j)] in all_lanes
            @test LaneTag(i,j) in all_lane_tags
        end
    end

end # roadway test

@testset "roadway read/write" begin
    roadway = Roadway()

    seg1 = RoadSegment(1,
        [Lane(LaneTag(1,1),
            [CurvePt(VecSE2(0.0,0.0,0.0), 0.0),
             CurvePt(VecSE2(1.0,0.0,0.0), 1.0)],
             width=1.0,
             boundary_left = LaneBoundary(:solid, :white),
             boundary_right = LaneBoundary(:broken, :yellow)),
         Lane(LaneTag(1,2),
            [CurvePt(VecSE2(0.0,1.0,0.0), 0.0),
             CurvePt(VecSE2(1.0,1.0,0.0), 1.0)],
             width=2.0,
             boundary_left = LaneBoundary(:double, :white))],
         )

    seg2 = RoadSegment(2,
        [Lane(LaneTag(2,1),
            [CurvePt(VecSE2(4.0,0.0,0.0), 0.0),
             CurvePt(VecSE2(5.0,0.0,0.0), 1.0)]),
         Lane(LaneTag(2,2),
            [CurvePt(VecSE2(4.0,1.0,0.0), 0.0),
             CurvePt(VecSE2(5.0,1.0,0.0), 1.0)])],
         )

    connect!(seg1.lanes[1], seg2.lanes[2])

    push!(roadway.segments, seg1)
    push!(roadway.segments, seg2)

    ############

    path, io = mktemp()
    write(io, MIME"text/plain"(), roadway)
    close(io)

    lines = open(readlines, path)

    for (line_orig, line_test) in zip(lines,
            [
             "ROADWAY",
             "2",
             "1",
             "   2",
             "   1",
             "       1.000",
             "       -Inf Inf",
             "       solid white",
             "       broken yellow",
             "       1",
             "           D (1 1.000000) 1 0.000000 2 2",
             "       2",
             "           (0.0000 0.0000 0.000000) 0.0000 NaN NaN",
             "           (1.0000 0.0000 0.000000) 1.0000 NaN NaN",
             "   2",
             "       2.000",
             "       -Inf Inf",
             "       double white",
             "       unknown unknown",
             "       0",
             "       2",
             "           (0.0000 1.0000 0.000000) 0.0000 NaN NaN",
             "           (1.0000 1.0000 0.000000) 1.0000 NaN NaN",
             "2",
             "   2",
             "   1",
             "       3.000",
             "       -Inf Inf",
             "       unknown unknown",
             "       unknown unknown",
             "       0",
             "       2",
             "           (4.0000 0.0000 0.000000) 0.0000 NaN NaN",
             "           (5.0000 0.0000 0.000000) 1.0000 NaN NaN",
             "   2",
             "       3.000",
             "       -Inf Inf",
             "       unknown unknown",
             "       unknown unknown",
             "       1",
             "           U (1 0.000000) 1 1.000000 1 1",
             "       2",
             "           (4.0000 1.0000 0.000000) 0.0000 NaN NaN",
             "           (5.0000 1.0000 0.000000) 1.0000 NaN NaN",
             ]
        )
        @test strip(line_orig) == strip(line_test)
    end

    io = open(path)
    roadway2 = read(io, MIME"text/plain"(), Roadway)
    close(io)
    rm(path)

    @test length(roadway.segments) == length(roadway2.segments)
    for (seg1, seg2) in zip(roadway.segments, roadway2.segments)
        @test seg1.id == seg2.id
        @test length(seg1.lanes) == length(seg2.lanes)
        for (lane1, lane2) in zip(seg1.lanes, seg2.lanes)
            @test lane1.tag == lane2.tag
            @test isapprox(lane1.width, lane2.width, atol=1e-3)
            @test isapprox(lane1.speed_limit.lo, lane2.speed_limit.lo, atol=1e-3)
            @test isapprox(lane1.speed_limit.hi, lane2.speed_limit.hi, atol=1e-3)
            @test lane1.boundary_left == lane2.boundary_left
            @test lane1.boundary_right == lane2.boundary_right
            for (conn1, conn2) in zip(lane1.exits, lane2.exits)
                @test conn1.downstream == conn2.downstream
                @test conn1.mylane.i == conn2.mylane.i
                @test isapprox(conn1.mylane.t, conn2.mylane.t, atol=1e-3)
                @test conn1.target.tag == conn2.target.tag
                @test conn1.target.ind.i == conn2.target.ind.i
                @test isapprox(conn1.target.ind.t, conn2.target.ind.t, atol=1e-3)
            end
            for (conn1, conn2) in zip(lane1.entrances, lane2.entrances)
                @test conn1.downstream == conn2.downstream
                @test conn1.mylane.i == conn2.mylane.i
                @test isapprox(conn1.mylane.t, conn2.mylane.t, atol=1e-3)
                @test conn1.target.tag == conn2.target.tag
                @test conn1.target.ind.i == conn2.target.ind.i
                @test isapprox(conn1.target.ind.t, conn2.target.ind.t, atol=1e-3)
            end
            for (pt1, pt2) in zip(lane1.curve, lane2.curve)
                @test normsquared(convert(VecE2, pt1.pos) - convert(VecE2, pt2.pos)) < 0.01
                @test angledist(pt1.pos.θ, pt2.pos.θ) < 1e-5
                @test isnan(pt1.s) || isapprox(pt1.s, pt2.s, atol=1e-3)
                @test isnan(pt1.k) || isapprox(pt1.k, pt2.k, atol=1e-6)
                @test isnan(pt1.kd) || isapprox(pt1.kd, pt2.kd, atol=1e-6)
            end
        end
    end
end # roadway read/write tests

@testset "Lane connection" begin
    lc = LaneConnection(true, CurveIndex(1,0.0), RoadIndex(CurveIndex(2,0.0), LaneTag(2,2)))
    show(IOBuffer(), lc)
end

@testset "Frenet" begin
    @test isapprox(AutomotiveDrivingModels._mod2pi2(0.0), 0.0)
    @test isapprox(AutomotiveDrivingModels._mod2pi2(0.5), 0.5)
    @test isapprox(AutomotiveDrivingModels._mod2pi2(2pi + 1), 1.0)
    @test isapprox(AutomotiveDrivingModels._mod2pi2(1 - 2pi), 1.0)

    roadway = get_test_roadway()
    roadproj = proj(VecSE2(0.0, 0.0, 0.0), roadway)
    roadind = RoadIndex(roadproj)

    @test roadind.ind.i == 1
    @test isapprox(roadind.ind.t, 0.0)
    @test roadind.tag == LaneTag(2,1)
    f = Frenet(roadind, roadway, t=0.1, ϕ=0.2)
    @test f.roadind === roadind
    @test isapprox(f.s, 0.0)
    @test f.t == 0.1
    @test f.ϕ == 0.2

    f = Frenet(roadproj, roadway)
    @test f.roadind === roadind
    @test isapprox(f.s, 0.0)
    @test f.t == 0.0
    @test f.ϕ == 0.0

    f = Frenet(VecSE2(0.0, 0.0, 0.0), roadway)
    @test f.roadind === roadind
    @test isapprox(f.s, 0.0)
    @test f.t == 0.0
    @test f.ϕ == 0.0
    @test get_posG(f, roadway) == VecSE2(0.0, 0.0, 0.0)

    lane = roadway[LaneTag(1,1)]
    f = Frenet(lane, 1.0, 0.0, 0.0)
    @test f.roadind.tag == lane.tag
    @test isapprox(f.s, 1.0)
    @test f.t == 0.0
    @test f.ϕ == 0.0
end

@testset "straight roadway" begin
    curve = gen_straight_curve(VecE2(25.0, -10.0), VecE2(25.0, 10.0), 2)
    @inferred gen_straight_curve(VecE2(25.0, -10.0), VecE2(25.0, 10.0), 2)
    @test length(curve) == 2
    @test curve[1].pos.x == 25.0

    roadway = gen_straight_roadway(1, 1000.0)
    @test length(roadway.segments) == 1

    seg = roadway[1]
    @test length(seg.lanes) == 1

    lane = seg.lanes[1]
    @test isapprox(lane.width, DEFAULT_LANE_WIDTH)
    @test lane.curve[1] == CurvePt(VecSE2(0.0,0.0,0.0), 0.0)
    @test lane.curve[2] == CurvePt(VecSE2(1000.0,0.0,0.0), 1000.0)

    roadway = gen_straight_roadway(3, 500.0, lane_width=1.0)
    @test length(roadway.segments) == 1

    seg = roadway[1]
    @test length(seg.lanes) == 3

    lane = seg.lanes[1]
    @test isapprox(lane.width, 1.0)
    @test lane.curve[1] == CurvePt(VecSE2(0.0,0.0,0.0), 0.0)
    @test lane.curve[2] == CurvePt(VecSE2(500.0,0.0,0.0), 500.0)

    lane = seg.lanes[2]
    @test isapprox(lane.width, 1.0)
    @test lane.curve[1] == CurvePt(VecSE2(0.0,1.0,0.0), 0.0)
    @test lane.curve[2] == CurvePt(VecSE2(500.0,1.0,0.0), 500.0)

    lane = seg.lanes[3]
    @test isapprox(lane.width, 1.0)
    @test lane.curve[1] == CurvePt(VecSE2(0.0,2.0,0.0), 0.0)
    @test lane.curve[2] == CurvePt(VecSE2(500.0,2.0,0.0), 500.0)

    origin = VecSE2(1.0,2.0,deg2rad(90))
    roadway = gen_straight_roadway(1, 100.0, lane_width=1.0, origin=origin)
    @test length(roadway.segments) == 1
    lane = roadway[1].lanes[1]
    @test lane.curve[1].pos == origin
    @test isapprox(lane.curve[2].pos.x,         1.0, atol=1e-6)
    @test isapprox(lane.curve[2].pos.y,       102.0, atol=1e-6)
    @test isapprox(lane.curve[2].pos.θ, deg2rad(90), atol=1e-6)

    roadway = gen_straight_roadway(2, 100.0, lane_widths=[1.0, 2.0])
    @test length(roadway.segments) == 1
    @test isapprox(roadway[1].lanes[1].curve[1].pos.y, 0.0, atol=1e-6)
    @test isapprox(roadway[1].lanes[2].curve[1].pos.y, 0.5 + 1.0, atol=1e-6)
end

@testset "bezier curve" begin
    # see intersection tutorial for visualization
    roadway = Roadway()
    # Define coordinates of the entry and exit points to the intersection
    r = 5.0 # turn radius
    A = VecSE2(0.0,DEFAULT_LANE_WIDTH,-π)
    B = VecSE2(0.0,0.0,0.0)
    C = VecSE2(r,-r,-π/2)
    D = VecSE2(r+DEFAULT_LANE_WIDTH,-r,π/2)
    E = VecSE2(2r+DEFAULT_LANE_WIDTH,0,0)
    F = VecSE2(2r+DEFAULT_LANE_WIDTH,DEFAULT_LANE_WIDTH,-π)

    # helper function
    function append_to_curve!(target::Curve, newstuff::Curve)
        s_end = target[end].s
        for c in newstuff
            push!(target, CurvePt(c.pos, c.s+s_end, c.k, c.kd))
        end
        return target
    end

    # Append right turn coming from the left
    curve = gen_straight_curve(convert(VecE2, B+VecE2(-100,0)), convert(VecE2, B), 2)
    append_to_curve!(curve, gen_bezier_curve(B, C, 0.6r, 0.6r, 51)[2:end])
    @test length(curve) == 52
    append_to_curve!(curve, gen_straight_curve(convert(VecE2, C), convert(VecE2, C+VecE2(0,-50.0)), 2))
    @test length(curve) == 54
    lane = Lane(LaneTag(length(roadway.segments)+1,1), curve)
    push!(roadway.segments, RoadSegment(lane.tag.segment, [lane]))

end

@testset "stadium roadway" begin
    roadway = gen_stadium_roadway(1, length=100.0, width=10.0,
                                  radius=25.0, ncurvepts_per_turn=2)
    @test length(roadway.segments) == 6

    seg = roadway[1]
    @test length(seg.lanes) == 1

    lane = seg.lanes[1]
    @test lane.curve[1] == CurvePt(VecSE2(100.0,0.0,0.0), 0.0)
    @test lane.curve[2] == CurvePt(VecSE2(125.0,25.0,π/2), 25*π/2)

    @test roadway[LaneTag(2,1)].curve[1] == CurvePt(VecSE2(125.0,35.0,π/2), 0.0)

    roadway = gen_stadium_roadway(4)
end


module Ecosystem

using StatsBase
using EcosystemCore

export Grass, Sheep, Wolf, World, Mushroom
export agent_step!, agent_count, world_step!, simulate!, every_nth

abstract type Mushroom <: PlantSpecies end
Base.show(io::IO,::Type{Mushroom}) = print(io,"🍄")

EcosystemCore.eats(::Animal{Sheep},::Plant{Mushroom}) = true
function EcosystemCore.eat!(s::Animal{Sheep}, m::Plant{Mushroom}, w::World)
    incr_energy!(s, -size(m)*Δenergy(s))
    m.size = 0
end

EcosystemCore.mates(::Animal{S,Female}, ::Animal{S,Male}) where S<:Species = true
EcosystemCore.mates(::Animal{S,Male}, ::Animal{S,Female}) where S<:Species = true
EcosystemCore.mates(a::Agent, b::Agent) = false

function simulate!(world::World, iters::Int; cb=()->())
    for i in 1:iters
        world_step!(world)
        cb()
    end
end

agent_count(p::Plant) = size(p)/EcosystemCore.max_size(p)
agent_count(::Animal) = 1
agent_count(as::Vector{<:Agent}) = sum(agent_count,as)

function agent_count(w::World)
    function op(d::Dict,a::Agent{S}) where S<:Species
        n = nameof(S)
        if n in keys(d)
            d[n] += agent_count(a)
        else
            d[n] = agent_count(a)
        end
        return d
    end
    foldl(op, w.agents |> values |> collect, init=Dict{Symbol,Real}())
end

function every_nth(f::Function, n::Int)
    i = 1
    function callback(args...)
        # display(i) # comment this out to see out the counter increases
        if i == n
            f(args...)
            i = 1
        else
            i += 1
        end
    end
end

end


import .Utils

abstract type AbstractSum{OpT, StringT, CoeffT} end

const ZERO_ATOL = 1e-12

####
#### Constructors
####

## Should we use this ?
op_strings(as::AbstractSum) = as.strings

function Base.similar(ps::AbstractSum{OpT, T, V}, n=0) where {OpT, W, C, V <:Vector{C}, T <: Vector{Vector{W}}}
    m = size(ps, 2)
    strings = [Vector{W}(undef, m) for i in 1:n]
    coeffs = Vector{C}(undef, n)
    return strip_typeof(ps)(strings, coeffs; already_sorted=true)
end

function Base.copy(as::AbstractSum)
    (new_strings, new_coeffs) = (copy.(as.strings), copy(as.coeffs))
    return strip_typeof(as)(new_strings, new_coeffs; already_sorted=true)
end

####
#### IO
####

function Base.show(io::IO, asum::AbstractSum)
    (m, n) = size(asum)
    print(io, m, "x", n, " ", typeof(asum), ":\n")
    for i::Integer in eachindex(asum)
        _show_abstract_term(io, asum[i])
        if i != lastindex(asum)
            print(io, "\n")
        end
    end
end

Base.show(m::MIME{Symbol("text/input")}, asum::AbstractSum) = show(stdout, m, asum)
function Base.show(io::IO, mime::MIME{Symbol("text/input")}, asum::AbstractSum)
    print(io, strip_typeof(asum), "(")
    print(io, "[")
    for term in asum
        show(io, mime, term)
        print(io, ",")
    end
    print(io, "])")
end

####
#### Canonicalization / sorting
####

function sort_and_sum_duplicates!(asum::AbstractSum)
    sort_and_sum_duplicates!(asum.strings, asum.coeffs)
    return asum
end

sum_duplicates!(asum::AbstractSum) = sum_duplicates!(asum.strings, asum.coeffs)

function sort_and_sum_duplicates!(terms, coeffs)
    sort_sums!(terms, coeffs)
    sum_duplicates!(terms, coeffs)
    remove_zeros!(terms, coeffs)
    return nothing
end

## This is expensive. Most time is spent in sortperm.
## There is no ThreadsX.sortperm, only sort.
Base.sort!(asum::AbstractSum; alg=MergeSort) = (sort_sums!(asum.strings, asum.coeffs; alg=alg); asum)

function sort_sums!(strings, coeffs; alg=MergeSort)
    p = sortperm(strings; alg=alg)
    permute!(strings, p)
    permute!(coeffs, p)
    return nothing
end

# TODO: may want to make this a method of zchop
"""
    remove_zeros!(asum::AbstractSum)
    remove_zeros!(terms, coeffs)

Remove terms from `asum` with coefficient (approximately) equal to zero.
If `terms` and `coeffs` are supplied, then elements are deleted from both `terms`
and `coeffs` at indices corresponding to vanishing elements of `coeff`.
"""
function remove_zeros!(asum::AbstractSum, atol=ZERO_ATOL)
    remove_zeros!(asum.strings, asum.coeffs, atol)
    return asum
end

function remove_zeros!(terms::Vector, coeffs::Vector, atol=ZERO_ATOL)
    # ThreadsX is very slow for small arrays
    # The findall(iszero, coeffs) is 500ns for two non-zero floats. What is wrong?
    # Appears to be this: iszero.(array) is taking almost all the time.
    # The following is what Base does, but writing it out is faster. A bug?
    # if length(coeffs) > 10^10
    #     inds = ThreadsX.findall(Utils.isapprox_zero.(coeffs))
    # else
    inds::Vector{Int} = findall(Utils.isapprox_zero.(coeffs, atol))
    #    end
    if ! isempty(inds)
        deleteat!(coeffs::Vector, inds)
        deleteat!(terms::Vector, inds)
    end
    return nothing
end

## Modeled on code in unique! for sorted input
"""
    sum_duplicates!(_op_strings, coeffs)

Find groups of terms whose members differ only in the coefficient.
Replace each group by one term carrying the sum of the coefficients
in that group. This routine assumes that `_op_strings` are sorted.
"""
function sum_duplicates!(_op_strings, coeffs)
    last_pauli::eltype(_op_strings) = first(_op_strings)
    coeff = first(coeffs)
    k = 2
    @inbounds for j in 2:length(_op_strings)
        if _op_strings[j] != last_pauli
            last_pauli = _op_strings[k] = _op_strings[j]
            coeffs[k] = coeffs[j]
            k += 1
        else
            coeffs[k-1] += coeffs[j]
        end
    end
    resize!(_op_strings, k-1)
    resize!(coeffs, k-1)
    return nothing
end

####
#### Container interface
####

## Fails for empty asum
function Base.size(asum::AbstractSum)
    n = isempty(asum) ? 0 : length(first(asum))
    return (length(asum), n)
end
Base.size(asum::AbstractSum, i::Integer) = size(asum)[i]

# Enables using `findall`, for instance.
# Fallback methods for `values` and `pairs` are OK.
Base.keys(asum::AbstractSum) = eachindex(asum)

# Iterate uses getindex to return `AbstractTerm`s.
function Base.iterate(asum::AbstractSum, state=1)
    state > lastindex(asum) && return nothing
    return (asum[state], state + 1)
end

for func in (:length, :eachindex, :lastindex, :firstindex)
    @eval begin
        Base.$func(as::AbstractSum, args...) = return $func(as.coeffs, args...)
    end
end

Base.getindex(asum::AbstractSum, j::Integer) =
    term_type(typeof(asum))(asum.strings[j], asum.coeffs[j])

Base.getindex(asum::AbstractSum, j::Integer, k::Integer) = asum.strings[j][k]
Base.getindex(asum::AbstractSum, i1::Integer, inds2) = asum.strings[i1][inds2]

## TODO: Use already_sorted flag ?
## I think no, because asum[5:-1:1] should sort the reversed result.
## Unless, someone wants to use indexing to get something other than canonical order.
Base.getindex(asum::AbstractSum, inds) = strip_typeof(asum)(asum.strings[inds], asum.coeffs[inds]) # already_sorted=true)

## Should this return a sum or an array of arrays ?
## Currently it returns a sum. Note that selected substrings may be identical, and coeffs combined
Base.getindex(asum::AbstractSum, inds1, inds2) = strip_typeof(asum)([x[inds2] for x in asum.strings[inds1]], asum.coeffs[inds1])

####
#### Compare / predicates
####

# This will fail for an empty `psum`. Use type info instead.
# There no well-defined `one` for `PauliSum`. It depends on the
# width of the string.
function Base.one(asum::AbstractSum)
    t = one(first(asum))
    strip_typeof(asum)([op_string(t)], [t.coeff]; already_sorted=true)
end

function Base.isone(asum::AbstractSum)
    return length(asum) == 1 && isone(first(asum))
end

function Base.:(==)(asum1::AbstractSum, asum2::AbstractSum)
    if length(asum1) != length(asum2)
        return false
    end
    if length(asum1) > 10^3
        # This is 8x-12x faster for large arrays, 10^4 or 5.
        return ThreadsX.all(i -> asum1[i] == asum2[i], eachindex(asum1))
    end
    return all(i -> asum1[i] == asum2[i], eachindex(asum1))
end

"""
    reverse(ps::AbstractSum)

Reverse qubit order in `ps` and sort terms.
"""
Base.reverse(as::AbstractSum) = reverse!(copy(as))

"""
    reverse!(ps::AbstractSum)

Reverse qubit order in `ps` in place and sort terms.
"""
function Base.reverse!(as::AbstractSum)
    strings = as.strings
    @inbounds for i in eachindex(strings)
        reverse!(strings[i])
    end
    return Base.sort!(as)
end

"""
    add!(psum::AbstractSum, pt::AbstractTerm...)

Add `AbstractTerm`s to `psum` in place, assuming `psum` is sorted and has no repeated
strings. Either a new term is inserted, or the coefficient is added to an existing
term. After adding the `pt`, `psum` will be left sorted, with no duplicates, and
no zero coefficients. Use `push!` to insert a term at the end of `psum` with no
simplification performed.
"""
add!(asum::AbstractSum, term::AbstractTerm) = add!(asum, op_string(term), term.coeff)

## TODO: pass higher order function to allow + and - instead of hard coded +=
function add!(asum::AbstractSum, op_string, coeff)
    inds = searchsorted(asum.strings, op_string)
    if length(inds) == 0 # op_string not found, add a new term
        insert_tuple!(asum, first(inds), (op_string, coeff))
    elseif length(inds) == 1 # one element equal to op_string
        i = first(inds) # get the (single) index
        @inbounds asum.coeffs[i] += coeff # add p to existing term
        @inbounds if Utils.isapprox_zero(asum.coeffs[i])
            @inbounds deleteat!(asum, i)
        end
    else
        throw(ErrorException("Duplicate terms found in operator sum."))
    end
    return asum
end

"""
    add!(to::AbstractSum, from::AbstractSum)

Adds the terms in `from` to `to` in place. `to` is mutated. `from` is not.
"""
function add!(to::AbstractSum, from::AbstractSum)
    for p in from
        add!(to, p)
    end
    return to
end

####
#### Updating / adding elements
####

"""
    insert!(ps::AbstractSum, ind, p::AbstractTerm)

Insert `p` into `ps` without sorting resulting `ps`.
"""
Base.insert!(ps::AbstractSum, ind, p::AbstractTerm) = insert_tuple!(ps, ind, (op_string(p), p.coeff))

@inline function insert_tuple!(ps::AbstractSum, ind, (paulis, coeff))
    insert!(ps.strings, ind, paulis)
    insert!(ps.coeffs, ind, coeff)
    return ps
end

function Base.deleteat!(ps::AbstractSum, args...)
    deleteat!(ps.coeffs, args...)
    deleteat!(ps.strings, args...)
    return ps
end

function Base.deleteat!(ps::AbstractSum, ind)
    deleteat!(ps.coeffs, ind)
    deleteat!(ps.strings, ind)
    return ps
end

"""
    push!(psum::AbstractSum, ps::AbstractTerm...)

Push `ps` to the end of `psum` without regard to order or possible duplication.

See `sort_and_sum_duplicates!`.
"""
function Base.push!(psum::AbstractSum, ps::AbstractTerm...)
    for p in ps
        push!(psum.strings, op_string(p))
        push!(psum.coeffs, p.coeff)
    end
    return psum
end

## TODO: This should probably not be a method of push!
function Base.push!(psum::AbstractSum, (string, coeff))
    push!(psum.strings, string)
    push!(psum.coeffs, coeff)
end

function Base.append!(to::AbstractSum, from::AbstractSum)
    for t in from
        push!(to, t)
    end
    return to
end

####
#### Algebra / mathematical operations
####

Base.:+(term::T, terms::T...) where {T <: AbstractTerm} = sum_type(T)([term, terms...])

function Base.:+(ps0::AbstractSum, pss::AbstractSum...)
    ps_out = copy(ps0)
    for ps in pss
        add!(ps_out, ps)
    end
    return ps_out
end

function Base.sum(v::Vector{<:AbstractSum})
    isempty(v) && throw(ArgumentError("Can't sum empty vector of `AbstractSum`."))
    v0, vrest = Iterators.peel(v)
    ps_out = copy(v0)
    for vi in vrest
        add!(ps_out, vi)
    end
    return ps_out
end

## TODO: Do something more efficient here.
function Base.:-(ps1::T, ps2::T) where T <: AbstractSum
    return ps1 + -ps2
end

## TODO: Do something more efficient here.
function Base.:-(pt1::T, pt2::T) where T <: AbstractTerm
    return sum_type(T)([pt1, -one(pt2.coeff) * pt2])
end

function Base.:-(psum::AbstractSum)
    strip_typeof(psum)(psum.strings, -one(eltype(psum.coeffs)) .* psum.coeffs; already_sorted=true)
end

function Base.:*(n::Number, asum::AbstractSum)
    strip_typeof(asum)(asum.strings, n .* asum.coeffs; already_sorted=true)
end

Base.:*(asum::AbstractSum, n::Number) = n * asum

function Base.:/(psum::AbstractSum, n::Number)
    strip_typeof(psum)(psum.strings, psum.coeffs ./ n; already_sorted=true)
end

function mul!(asum_out::T, term::AbstractTerm, asum::T) where T <: AbstractSum
    @inbounds for j in eachindex(asum)
        new_term = term * asum[j]
        asum_out.coeffs[j] = new_term.coeff
        asum_out.strings[j] = op_string(new_term)
    end
    return asum_out
end

function Base.:*(term::AbstractTerm, asum::AbstractSum)
    asum_out = strip_typeof(asum)(similar(asum.strings), similar(asum.coeffs); already_sorted=true)
    return sort!(mul!(asum_out, term, asum))
end

"""
    *(as1::AbstractSum, as2::AbstractSum)

Multiply `as1` and `as2` returning another `AbstractSum`

# Examples
```jldoctest
julia> a = rand_op_sum(Pauli, 5, 3); ma = Matrix(a);

julia> b = rand_op_sum(Pauli, 5, 3); mb = Matrix(b);

julia> Matrix(a * b * a) == ma * mb * ma
true
```
"""
function Base.:*(as1::AbstractSum, as2::AbstractSum)
    ## Using  a buffer and mul! ought to be faster, but it is not.
    ## because mul! takes no time here compared to add!
    asum_out = similar(as1)
    asum_temp = strip_typeof(as2)(similar(as2.strings), similar(as2.coeffs); already_sorted=true)
    @inbounds for i in 1:length(as1)
        mul!(asum_temp, as1[i], as2)
        append!(asum_out, asum_temp)
    end
    return sort_and_sum_duplicates!(asum_out)
end

"""
    filter(f, as::AbstractSum)

Return a copy of `as` keeping only terms for which `f` is true.

# Examples
```julia
julia> filter(x -> count_bodies(x) == 1, fermi_sum)
IIIN * -0.47594871522096355
IINI * -0.47594871522096355
INII * -1.2524635735648981
NIII * -1.2524635735648981
```
"""
function Base.filter(f, as::AbstractSum)
    inds = findall(f, as)
    return strip_typeof(as)(as.strings[inds], as.coeffs[inds])
end


using LightGraphs

@testset "Constructors" begin
   @test @inferred(StorageGraph()) == StorageGraph()
   @test eltype(StorageGraph{UInt8}()) == UInt8
   @test eltype(StorageGraph()) == eltype(SimpleDiGraph())
   @test edgetype(StorageGraph()) == edgetype(SimpleDiGraph())
   @test nv(StorageGraph()) == 0
   @test ne(StorageGraph()) == 0
   @test is_directed(StorageGraph)
   @test is_directed(StorageGraph())
   @test eltype(@inferred(StorageGraph{UInt8}())) == UInt8
   @test @inferred(SimpleDiGraph(StorageGraph())) == SimpleDiGraph()

   g = SimpleDiGraph(3)
   add_edge!(g, 1, 2)
   add_edge!(g, 2, 3)
   data = Dict(1=>(a=1,),2=>(a=2,),3=>(a=3,))
   paths = Dict(Edge(1, 2)=>Set(1), Edge(2, 3)=>Set(1))
   sg = StorageGraph(g, data, paths)
   @test @inferred(StorageGraph(g, data, paths)) == sg
   @test nv(sg) == nv(g)
   @test ne(sg) == ne(g)
   @test sg.data == data
   @test sg.paths == paths
   @test sg.maxid[] == 2
   @test sg.index == Dict((a=1,)=>1,(a=2,)=>2,(a=3,)=>3)

   @test @inferred(StorageGraphs.fadj(sg, 2)) == LightGraphs.SimpleGraphs.fadj(g, 2)
   @test @inferred(StorageGraphs.badj(sg, 2)) == LightGraphs.SimpleGraphs.badj(g, 2)

   @test outneighbors(sg, 1) == outneighbors(sg, (a=1,)) == [2]

   ng = StorageGraph{UInt8}(sg)
   @test eltype(ng) == UInt8
   @test eltype(ng.data) == Pair{UInt8,NamedTuple}
   @test eltype(ng.maxid) == UInt8
end

@testset "Elementary operations" begin
   g = StorageGraph()
   @test add_vertex!(g, (a=1,))
   @test nv(g) == 1
   @test ne(g) == 0
   @test g.data == Dict(1=>(a=1,))
   @test g.index == Dict((a=1,)=>1)
   @test has_prop(g, 1, :a)
   @test sprint(show, g) == "{1, 0} $(eltype(g)) storage graph"

   @test add_vertex!(g, (b=2,))
   @test nv(g) == 2
   @test ne(g) == 0
   @test g.data == Dict(1=>(a=1,), 2=>(b=2,))
   @test g.index == Dict((a=1,)=>1, (b=2,)=>2)
   @test has_prop(g, 2, :b)
   @test !set_prop!(g, 3, (x="doesn't exist",))
   @test !set_prop!(g, 1, 2, 0)

   @test add_edge!(g, 1, 2, 1)
   @test nv(g) == 2
   @test ne(g) == 1
   @test g.paths == Dict(Edge(1,2)=>Set(1))
   @test has_prop(g, 1, 2, 1)

   @test g.maxid[] == get_prop(g) == 1
   set_prop!(g, 2)
   @test get_prop(g) == 2

   @test rem_edge!(g, 1, 2)
   @test ne(g) == 0
   @test !has_prop(g, 1, 2, 1)
   @test rem_vertex!(g, 2)
   @test nv(g) == 1
   @test !has_prop(g, 2, :b)
   @test rem_vertex!(g, 1)
   @test nv(g) == 0
   @test !has_prop(g, 1, :a)
   @test g == StorageGraph()
end


using CFMG, Test
using Pkg

if (in("JLD", keys(Pkg.installed())) == false)
  Pkg.add("JLD")
end

if (in("Statistics", keys(Pkg.installed())) == false)
  Pkg.add("Statistics")
end

using JLD

function CFMG_test1()
  # try
    # generate model with CFMG
    generate_default_project("./default1")
    cp("config/Network.vff", "./default1/Network.vff", force=true)
    make_julia_model("./default1/Network.vff", "./default2")

    # run the model
    println("change pwd to make model run normal")
    println(pwd())
    cd("./default2/")
    println(pwd())
    println("run the model")
    print("__DIR__ at test1.jl ")
    println(@__DIR__)
    include("default2/Static.jl")  # this is interesting!

    # verify simulation results
    println("load std result")
    test1_result_path = normpath(joinpath(@__DIR__, "test1_result.jld"))
    test1_result = load(test1_result_path)["soln_std"]
    println(static_soln_object)
    @assert test1_result.exit_flag == static_soln_object.exit_flag "check exit_flag"
    @assert test1_result.status_flag == static_soln_object.status_flag "check status_flag"
    @assert norm(test1_result.objective_value - static_soln_object.objective_value) < 1e-8 "Check objective_value"
    @assert norm(test1_result.flux - static_soln_object.flux) < 1e-8 "Check flux"

    # remove generated files
    # sleep(60)
    cd("../")
    rm("./default1", recursive=true)
    rm("./default2", recursive=true)
    # @assert 0 == 1
    # println(pwd())
    # default1 = normpath(joinpath(@__DIR__, "default1/"))
    # default2 = normpath(joinpath(@__DIR__, "default2/"))
    # rm(default1, recursive=true)
    # rm(default2, recursive=true)

  # catch e
  #   return false
  # end

  return true
end

# CFMG_test1()


# using CSV
# using PyPlot
# using Dates

path_45 = "/home/sarah/Master/Thesis/Data/Projektionen/new_station_data_rcp45/rcp45/"
Name_Projections_45 = readdir(path_45)
path_85 = "/home/sarah/Master/Thesis/Data/Projektionen/new_station_data_rcp85/rcp85/"
Name_Projections_85 = readdir(path_85)

include("loadfunctions.jl")
include("compare_Present_Future.jl")
include("compare_Present_Future_High_Flows.jl")
include("compare_Present_Future_snow.jl")
include("compare_Present_Future_low_flows.jl")

Area_Catchment_Gailtal = sum([98227533.0, 184294158.0, 83478138.0, 220613195.0])
Area_Catchment_Palten = sum([198175943.0, 56544073.0, 115284451.3])
Area_Catchment_Pitten = 115496400.
Area_Catchment_Silbertal = 100139168.
Area_Catchment_Defreggental = sum([235811198.0, 31497403.0])
Area_Catchment_Pitztal = sum([20651736.0, 145191864.0])

Discharge_Feistritz = CSV.read("/home/sarah/HBVModel/Feistritz/Q-Tagesmittel-214353.csv", header= false, skipto=388, decimal=',', delim = ';', types=[String, Float64])
Discharge_Palten = CSV.read("/home/sarah/HBVModel/Palten/Q-Tagesmittel-210815.csv", header= false, skipto=21, decimal=',', delim = ';', types=[String, Float64])
Discharge_Gailtal = CSV.read("/home/sarah/HBVModel/Gailtal/Q-Tagesmittel-212670.csv", header= false, skipto=23, decimal=',', delim = ';', types=[String, Float64])
Discharge_Silbertal = CSV.read("/home/sarah/HBVModel/Silbertal/Q-Tagesmittel-200048.csv", header= false, skipto=24, decimal=',', delim = ';', types=[String, Float64])
Discharge_Defreggental = CSV.read("/home/sarah/HBVModel/Defreggental/Q-Tagesmittel-212100.csv", header= false, skipto=26, decimal=',', delim = ';', types=[String, Float64])
Discharge_Pitztal = CSV.read("/home/sarah/HBVModel/Pitztal/Q-Tagesmittel-201335.csv", header= false, skipto=23, decimal=',', delim = ';', types=[String, Float64])

Catchment_Name = "Pitztal"
Area_Catchment = Area_Catchment_Pitztal
Discharge_Catchment = Discharge_Pitztal
nr_runs = 300
scaling_factor_discharge = 1
# make folder Projections/Catchment_Name/PastvsFuture/ - (Inputs, Monthly_Discharge, Annual_Discharge, FDC, Budyko, Annual_Max_Discharge/Circular, LowFlows, Snow_Cover, Drought (/none, /summer /winter)

# make plots of tmeperature and precipitation in past and future
#Prec_past, Prec_Future,Temp_past, Temp_Future, Months = plot_Monthly_Temperature_Precipitation(path_45, Catchment_Name)
#Prec_past, Prec_Future,Temp_past, Temp_Future, Months = plot_Monthly_Temperature_Precipitation(path_85, Catchment_Name)

# Prec_statistics_past_45, Prec_statistics_future_45 = monthly_prec_statistics(path_45, Catchment_Name)
# Prec_statistics_past_85, Prec_statistics_future_85 = monthly_prec_statistics(path_85, Catchment_Name)

# plot HYDROGRAPHS ------------
# @time begin
# plot_hydrographs_proj(path_45, Catchment_Name, Area_Catchment, nr_runs)
# plot_hydrographs_proj(path_85, Catchment_Name, Area_Catchment, nr_runs)
# end
# plot_hydrographs_proj(path_45, Catchment_Name, Area_Catchment, 1987, 2087)
# plot_hydrographs_proj(path_85, Catchment_Name, Area_Catchment, 1987, 2087)
# plot_hydrographs_proj(path_45, Catchment_Name, Area_Catchment, 1990, 2090)
# plot_hydrographs_proj(path_85, Catchment_Name, Area_Catchment, 1990, 2090)
# plot_hydrographs_proj(path_45, Catchment_Name, Area_Catchment, 1995, 2095)
# plot_hydrographs_proj(path_85, Catchment_Name, Area_Catchment, 1995, 2095)
# plot_hydrographs_proj(path_45, Catchment_Name, Area_Catchment, 1983, 2083)
# plot_hydrographs_proj(path_85, Catchment_Name, Area_Catchment, 1983, 2083)
# plot_hydrographs_proj(path_45, Catchment_Name, Area_Catchment, 1987, 2087)
# plot_hydrographs_proj(path_85, Catchment_Name, Area_Catchment, 1987, 2087)
# plot_hydrographs_proj(path_45, Catchment_Name, Area_Catchment, 1992, 2092)
# plot_hydrographs_proj(path_85, Catchment_Name, Area_Catchment, 1992, 2092)
# # # ----------------- MONTHLY DISCHARGES ------------------------
# # saves the results to a textfile
# @time begin
# Monthly_Discharge_Change_85, monthly_Discharge_past_85, monthly_Discharge_future_85, months_85 = change_monthly_Discharge(path_85, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/discharge_months_8.5.txt",hcat(months_85, monthly_Discharge_past_85, monthly_Discharge_future_85, Monthly_Discharge_Change_85),',')
# end
# #
# @time begin
# Monthly_Discharge_Change_45, monthly_Discharge_past_45, monthly_Discharge_future_45, months_45 = change_monthly_Discharge(path_45, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/discharge_months_4.5.txt",hcat(months_45, monthly_Discharge_past_45, monthly_Discharge_future_45, Monthly_Discharge_Change_45),',')
# end
# #
# monthly_changes_85 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/discharge_months_8.5.txt", ',')
# months_85 = monthly_changes_85[:,1]
# monthly_Discharge_past_85 = monthly_changes_85[:,2]
# monthly_Discharge_future_85  = monthly_changes_85[:,3]
# Monthly_Discharge_Change_85  = monthly_changes_85[:,4]
# monthly_changes_45 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/discharge_months_4.5.txt", ',')
# months_45 = monthly_changes_45[:,1]
# monthly_Discharge_past_45 = monthly_changes_45[:,2]
# monthly_Discharge_future_45  = monthly_changes_45[:,3]
# Monthly_Discharge_Change_45  = monthly_changes_45[:,4]
# #
# plot_changes_monthly_discharge(monthly_Discharge_past_45, monthly_Discharge_future_45, monthly_Discharge_past_85, monthly_Discharge_future_85, months_45, Catchment_Name, nr_runs)
# #
# plot_changes_monthly_discharge_mm(monthly_Discharge_past_45, monthly_Discharge_future_45, monthly_Discharge_past_85, monthly_Discharge_future_85, months_45, Area_Catchment, Catchment_Name)

## -------------- PLOT MONTHLY DISCHARGE PAST REAL DATA ---------------------------------
# Timeseries_Past = collect(Date(1981,1,1):Day(1):Date(2010,12,31))
# Discharge = convert(Matrix, Discharge_Catchment)
# startindex = findfirst(isequal("01.01."*string(1981)*" 00:00:00"), Discharge)
# endindex = findfirst(isequal("31.12."*string(2010)*" 00:00:00"), Discharge)
# Observed_Discharge = Array{Float64,1}[]
# push!(Observed_Discharge, Discharge[startindex[1]:endindex[1],2])
# Observed_Discharge_m3s = Observed_Discharge[1]
# Observed_Discharge = Observed_Discharge_m3s * 1000 / Area_Catchment * (3600 * 24)
# Monthly_Discharge_Observed, Months_Past = monthly_discharge(Observed_Discharge, Timeseries_Past)
# plot()
# for month in 1:12
#     boxplot!(Monthly_Discharge_Observed[findall(x-> x == month, Months_Past)], size=(2000,800), leg=false, color=["red"], left_margin = [5mm 0mm])
# end
# ylabel!("Averaged Monthly Discharge [mm/d]")
# title!("Averaged Measured Monthly Discharge 1981-2010")
# #ylims!((0,40))
# #hline!([0], color=["grey"], linestyle = :dash)
# xticks!([1:12;], ["Jan", "Feb", "Mar", "Apr", "May","Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"])
# #xticks!([2:2:24;], ["Jan 8.5", "Feb 8.5", "Mar 8.5", "Apr 8.5", "May 8.5","Jun 8.5", "Jul 8.5", "Aug 8.5", "Sep 8.5", "Oct 8.5", "Nov 8.5", "Dec 8.5"])
# savefig("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/"*Catchment_Name*"_measured_monthly_discharge_mm.png")
#
# # ---------------- ANNUAL DISCHARGE --------------------
#
# relative_change_45, Total_Discharge_Past_45, Total_Discharge_Future_45 = annual_discharge_new(monthly_Discharge_past_45, monthly_Discharge_future_45, Area_Catchment, nr_runs)
# relative_change_85, Total_Discharge_Past_85, Total_Discharge_Future_85 = annual_discharge_new(monthly_Discharge_past_85, monthly_Discharge_future_85, Area_Catchment, nr_runs)

# #can also be calculate directly from discharge data but takes more time
#relative_change_45, Total_Discharge_Past_45, Total_Discharge_Future_45, relative_change_85, Total_Discharge_Past_85, Total_Discharge_Future_85 = change_annual_discharge(path_45, path_85, Catchment_Name, Area_Catchment)
# plot_change_annual_discharge(relative_change_45, Total_Discharge_Past_45, Total_Discharge_Future_45, relative_change_85, Total_Discharge_Past_85, Total_Discharge_Future_85, Area_Catchment, Catchment_Name)

# ----------------------- FDC ------------------------

## the change in discharge of a certian percentile can be calculated
# percentile = 0.1
#discharge_past_45, discharge_future_45= FDC_compare_percentile(path_45, percentile, Catchment_Name)
#discharge_past_85, discharge_future_85= FDC_compare_percentile(path_85, percentile, Catchment_Name)
## or directly plotted
#plot_FDC_Percentile(path_45, path_85, Catchment_Name, percentile)

## or the relative change for a range of percentiles can be plotted
#plot_FDC_Percentile_relative_change(path_45, path_85, Catchment_Name, collect(0.1:0.1:0.9))

# --------------------------- BUDYKO -------------------------

## calculates the evaporative and aridity index
# aridity_past45, aridity_future_45, evaporative_past_45, evaporative_future_45 = aridity_evaporative_index(path_45, Area_Catchment, Catchment_Name)
# aridity_past85, aridity_future_85, evaporative_past_85, evaporative_future_85,  prec_past85, prec_future_85 = aridity_evaporative_index(path_85, Area_Catchment, Catchment_Name)
# plot_Budyko(aridity_past45, aridity_future_45, evaporative_past_45, evaporative_future_45, aridity_past85, aridity_future_85, evaporative_past_85, evaporative_future_85, path_45, Catchment_Name, nr_runs)
# plot_changes_Budyko(aridity_past45, aridity_future_45, evaporative_past_45, evaporative_future_45, aridity_past85, aridity_future_85, evaporative_past_85, evaporative_future_85, Area_Catchment, Catchment_Name, nr_runs)
#

# ----------- RUNOFF COEFFICIENT ----------------
# @time begin
#monthly_runoff_coef_past, monthly_runoff_coef_future, months, monthly_discharge_past, monthly_discharge_future = monthly_runoff_coefficient(path_45, Catchment_Name, Area_Catchment, nr_runs)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Budyko/monthly_runoff_coefficient_4.5_new.txt",hcat(monthly_runoff_coef_past, monthly_runoff_coef_future, months),',')
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/monthly_discharge_all_years_past_4.5.txt", monthly_discharge_past)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/monthly_discharge_all_years_future_4.5.txt", monthly_discharge_future)
# end
#
# @time begin
# monthly_runoff_coef_past, monthly_runoff_coef_future, months, monthly_discharge_past, monthly_discharge_future = monthly_runoff_coefficient(path_85, Catchment_Name, Area_Catchment, nr_runs)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Budyko/monthly_runoff_coefficient_8.5_new.txt",hcat(monthly_runoff_coef_past, monthly_runoff_coef_future, months),',')
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/monthly_discharge_all_years_past_8.5.txt", monthly_discharge_past)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Monthly_Discharge/monthly_discharge_all_years_future_8.5.txt", monthly_discharge_future)
# end

# @time begin
#seasonal_runoff_coef_past, seasonal_runoff_coef_future, all_seasons = seasonal_runoff_coefficient(path_45, Catchment_Name, nr_runs)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Budyko/seasonal_runoff_coefficient_4.5.txt",hcat(seasonal_runoff_coef_past, seasonal_runoff_coef_future, all_seasons),',')
# end

# @time begin
# seasonal_runoff_coef_past_85, seasonal_runoff_coef_future_85, all_seasons_85 = seasonal_runoff_coefficient(path_85, Catchment_Name, nr_runs)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Budyko/seasonal_runoff_coefficient_8.5.txt",hcat(seasonal_runoff_coef_past_85, seasonal_runoff_coef_future_85, all_seasons_85 ),',')
# end
# # # --------------------------- Maximum Annual Discharge -------------------
# #
# @time begin
# average_max_Discharge_past_45, average_max_Discharge_future_45, Timing_max_Discharge_past_45, Timing_max_Discharge_future_45, All_Concentration_past_45, All_Concentration_future_45 =  change_max_Annual_Discharge(path_45, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_4.5.txt",hcat(average_max_Discharge_past_45, average_max_Discharge_future_45, Timing_max_Discharge_past_45, Timing_max_Discharge_future_45, All_Concentration_past_45, All_Concentration_future_45),',')
# end
#
# @time begin
# average_max_Discharge_past_85, average_max_Discharge_future_85, Timing_max_Discharge_past_85, Timing_max_Discharge_future_85, All_Concentration_past_85, All_Concentration_future_85 =  change_max_Annual_Discharge(path_85, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_8.5.txt",hcat(average_max_Discharge_past_85, average_max_Discharge_future_85, Timing_max_Discharge_past_85, Timing_max_Discharge_future_85, All_Concentration_past_85, All_Concentration_future_85),',')
# end
#
# annual_max_flow_45 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_4.5.txt",',')
# annual_max_flow_85 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_8.5.txt",',')
# average_max_Discharge_past_45 = annual_max_flow_45[:,1]
# average_max_Discharge_future_45 = annual_max_flow_45[:,2]
# Timing_max_Discharge_past_45 = annual_max_flow_45[:,3]
# Timing_max_Discharge_future_45 = annual_max_flow_45[:,4]
# All_Concentration_past_45 = annual_max_flow_45[:,5]
# All_Concentration_future_45 = annual_max_flow_45[:,6]
# average_max_Discharge_past_85 = annual_max_flow_85[:,1]
# average_max_Discharge_future_85 = annual_max_flow_85[:,2]
# Timing_max_Discharge_past_85 = annual_max_flow_85[:,3]
# Timing_max_Discharge_future_85 = annual_max_flow_85[:,4]
# All_Concentration_past_85 = annual_max_flow_85[:,5]
# All_Concentration_future_85 = annual_max_flow_85[:,6]
# plot_Max_Flows(average_max_Discharge_past_45, average_max_Discharge_future_45, average_max_Discharge_past_85, average_max_Discharge_future_85, Timing_max_Discharge_past_45, Timing_max_Discharge_future_45, Timing_max_Discharge_past_85, Timing_max_Discharge_future_85, Catchment_Name, Area_Catchment, nr_runs)
#
# std_timing(All_Concentration_past_45, All_Concentration_future_45, All_Concentration_past_85, All_Concentration_future_85, "Annual_Max_Discharge")
# # # # #--------------using probability distribution
# @time begin
# max_Discharge_Past_45, Max_Discharge_Future_45, Exceedance_Probability_45, Date_Past_45, Date_Future_45 =  change_max_Annual_Discharge_Prob_Distribution(path_45, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_prob_distr_4.5.txt",hcat(max_Discharge_Past_45, Max_Discharge_Future_45, Exceedance_Probability_45, Date_Past_45, Date_Future_45), ',')
# end
# @time begin
# max_Discharge_Past_85, Max_Discharge_Future_85, Exceedance_Probability_85, Date_Past_85, Date_Future_85 =  change_max_Annual_Discharge_Prob_Distribution(path_85, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_prob_distr_8.5.txt",hcat(max_Discharge_Past_85, Max_Discharge_Future_85, Exceedance_Probability_85, Date_Past_85, Date_Future_85), ',')
# end
# max_discharge_prob_45 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_prob_distr_4.5.txt", ',')
# max_Discharge_Past_45 = max_discharge_prob_45[:,1]
# Max_Discharge_Future_45 = max_discharge_prob_45[:,2]
# Exceedance_Probability_45 = max_discharge_prob_45[:,3]
# Date_Past_45 = max_discharge_prob_45[:,4]
# Date_Future_45 = max_discharge_prob_45[:,5]
#
# max_discharge_prob_85 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Annual_Max_Discharge/change_max_Annual_Discharge_prob_distr_8.5.txt", ',')
# max_Discharge_Past_85 = max_discharge_prob_85[:,1]
# Max_Discharge_Future_85 = max_discharge_prob_85[:,2]
# Exceedance_Probability_85 = max_discharge_prob_85[:,3]
# Date_Past_85 = max_discharge_prob_85[:,4]
# Date_Future_85 = max_discharge_prob_85[:,5]
#
#
# plot_change_timing_AMF_over_year(Date_Past_45, Date_Future_45, Catchment_Name, nr_runs, "45")
# plot_change_timing_AMF_over_year(Date_Past_85, Date_Future_85, Catchment_Name, nr_runs, "85")
# mean_change, min_change, max_change, std_change = plot_Max_Flows_Prob_Distribution(max_Discharge_Past_45, Max_Discharge_Future_45, max_Discharge_Past_85, Max_Discharge_Future_85, Exceedance_Probability_45, Catchment_Name)
# #
# # # ---------------- SUMMER LOW FLOWS 7 days ----------------------
# #
# @time begin
# Summer_Low_Flows_past45, Summer_Low_Flows_future45, Timing_Summer_Low_Flows_Past_45, Timing_Summer_Low_Flows_Future_45, concentration_past_45, concentration_future_45 = analyse_low_flows(path_45, Catchment_Name, "summer")
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_summer4.5.txt",hcat(Summer_Low_Flows_past45, Summer_Low_Flows_future45, Timing_Summer_Low_Flows_Past_45, Timing_Summer_Low_Flows_Future_45, concentration_past_45, concentration_future_45),',')
# end
# @time begin
# Summer_Low_Flows_past85, Summer_Low_Flows_future85, Timing_Summer_Low_Flows_Past_85, Timing_Summer_Low_Flows_Future_85, concentration_past_85, concentration_future_85  = analyse_low_flows(path_85, Catchment_Name, "summer")
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_summer8.5.txt",hcat(Summer_Low_Flows_past85, Summer_Low_Flows_future85, Timing_Summer_Low_Flows_Past_85, Timing_Summer_Low_Flows_Future_85, concentration_past_85, concentration_future_85),',')
# end
#
# low_flows_summer_85 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_summer8.5.txt", ',')
# Summer_Low_Flows_past85 = low_flows_summer_85[:,1]
# Summer_Low_Flows_future85 = low_flows_summer_85[:,2]
# Timing_Summer_Low_Flows_Past_85 = low_flows_summer_85[:,3]
# Timing_Summer_Low_Flows_Future_85 = low_flows_summer_85[:,4]
# concentration_past_85 = low_flows_summer_85[:,5]
# concentration_future_85 = low_flows_summer_85[:,6]
# low_flows_summer_45 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_summer4.5.txt", ',')
# Summer_Low_Flows_past45 = low_flows_summer_45[:,1]
# Summer_Low_Flows_future45 = low_flows_summer_45[:,2]
# Timing_Summer_Low_Flows_Past_45 = low_flows_summer_45[:,3]
# Timing_Summer_Low_Flows_Future_45 = low_flows_summer_45[:,4]
# concentration_past_45 = low_flows_summer_45[:,5]
# concentration_future_45 = low_flows_summer_45[:,6]
#
# plot_low_flows_summer(Summer_Low_Flows_past45, Summer_Low_Flows_future45, Summer_Low_Flows_past85, Summer_Low_Flows_future85,  Timing_Summer_Low_Flows_Past_45, Timing_Summer_Low_Flows_Future_45,  Timing_Summer_Low_Flows_Past_85, Timing_Summer_Low_Flows_Future_85, Catchment_Name, Area_Catchment, nr_runs)
#
# std_timing(concentration_past_45, concentration_future_45, concentration_past_85, concentration_future_85, "LowFlows/Summer")
# # # # # ----------------------- WINTER LOW FLOWS 7 DAYS ------------------------
# @time begin
# Winter_Low_Flows_past45, Winter_Low_Flows_future45, Timing_Winter_Low_Flows_Past_45, Timing_Winter_Low_Flows_Future_45, concentration_past_45, concentration_future_45  = analyse_low_flows(path_45, Catchment_Name, "winter")
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_winter4.5.txt",hcat(Winter_Low_Flows_past45, Winter_Low_Flows_future45, Timing_Winter_Low_Flows_Past_45, Timing_Winter_Low_Flows_Future_45, concentration_past_45, concentration_future_45),',')
# end
# @time begin
# Winter_Low_Flows_past85, Winter_Low_Flows_future85, Timing_Winter_Low_Flows_Past_85, Timing_Winter_Low_Flows_Future_85, concentration_past_85, concentration_future_85  = analyse_low_flows(path_85, Catchment_Name, "winter")
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_winter8.5.txt",hcat(Winter_Low_Flows_past85, Winter_Low_Flows_future85, Timing_Winter_Low_Flows_Past_85, Timing_Winter_Low_Flows_Future_85, concentration_past_85, concentration_future_85),',')
# end
#
# low_flows_winter_85 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_winter8.5.txt", ',')
# Winter_Low_Flows_past85 = low_flows_winter_85[:,1]
# Winter_Low_Flows_future85 = low_flows_winter_85[:,2]
# Timing_Winter_Low_Flows_Past_85 = low_flows_winter_85[:,3]
# Timing_Winter_Low_Flows_Future_85 = low_flows_winter_85[:,4]
# concentration_past_85 = low_flows_winter_85[:,5]
# concentration_future_85 = low_flows_winter_85[:,6]
# low_flows_winter_45 = readdlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_winter4.5.txt", ',')
# Winter_Low_Flows_past45 = low_flows_winter_45[:,1]
# Winter_Low_Flows_future45 = low_flows_winter_45[:,2]
# Timing_Winter_Low_Flows_Past_45 = low_flows_winter_45[:,3]
# Timing_Winter_Low_Flows_Future_45 = low_flows_winter_45[:,4]
# concentration_past_45 = low_flows_winter_45[:,5]
# concentration_future_45 = low_flows_winter_45[:,6]
#
# plot_low_flows_winter(Winter_Low_Flows_past45, Winter_Low_Flows_future45, Winter_Low_Flows_past85, Winter_Low_Flows_future85,  Timing_Winter_Low_Flows_Past_45, Timing_Winter_Low_Flows_Future_45,  Timing_Winter_Low_Flows_Past_85, Timing_Winter_Low_Flows_Future_85, Catchment_Name, Area_Catchment, nr_runs)
#
# std_timing(concentration_past_45, concentration_future_45, concentration_past_85, concentration_future_85, "LowFlows/Winter")
#
# # ------------------- LOW FLOWS ALL YEAR --------------------
# @time begin
# Summer_Low_Flows_past45, Summer_Low_Flows_future45, Timing_Summer_Low_Flows_Past_45, Timing_Summer_Low_Flows_Future_45, concentration_past_45, concentration_future_45 = analyse_low_flows(path_45, Catchment_Name, "none")
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_whole_year4.5.txt",hcat(Summer_Low_Flows_past45, Summer_Low_Flows_future45, Timing_Summer_Low_Flows_Past_45, Timing_Summer_Low_Flows_Future_45, concentration_past_45, concentration_future_45),',')
# end
# @time begin
# Summer_Low_Flows_past85, Summer_Low_Flows_future85, Timing_Summer_Low_Flows_Past_85, Timing_Summer_Low_Flows_Future_85, concentration_past_85, concentration_future_85  = analyse_low_flows(path_85, Catchment_Name, "none")
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_whole_year8.5.txt",hcat(Summer_Low_Flows_past85, Summer_Low_Flows_future85, Timing_Summer_Low_Flows_Past_85, Timing_Summer_Low_Flows_Future_85, concentration_past_85, concentration_future_85),',')
# end
#
# @time begin
# max_Discharge_Past_45, Max_Discharge_Future_45, Exceedance_Probability_45, Date_Past_45, Date_Future_45 =  analyse_low_flows_prob_distr(path_45, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_prob_distr_4.5.txt",hcat(max_Discharge_Past_45, Max_Discharge_Future_45, Exceedance_Probability_45, Date_Past_45, Date_Future_45), ',')
# end
# @time begin
# max_Discharge_Past_85, Max_Discharge_Future_85, Exceedance_Probability_85, Date_Past_85, Date_Future_85 =  analyse_low_flows_prob_distr(path_85, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/LowFlows/low_flows_prob_distr_8.5.txt",hcat(max_Discharge_Past_85, Max_Discharge_Future_85, Exceedance_Probability_85, Date_Past_85, Date_Future_85), ',')
# end

## --------------- HYDROLOGICAL DROUGHT ----------------------------

#threshold = get_threshold_hydrological_drought(Discharge_Catchment, 1981,2010, 0.9, scaling_factor_discharge)
# "whole" means over whole timeseries
# season = "winter" # "summer", "winter", "none"
# @time begin
# Drought_45 = compare_hydrological_drought(path_45, threshold, season, Area_Catchment, Catchment_Name, "yearly")
# end
# @time begin
# Drought_85 = compare_hydrological_drought(path_85, threshold, season, Area_Catchment, Catchment_Name, "yearly")
# end
# plot_drought_total_deficit(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
# plot_drought_statistics_change(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
# plot_drought_statistics_rel_change(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
# plot_drought_statistics(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
# #
# season = "none" # "summer", "winter", "none"
# @time begin
# Drought_45 = compare_hydrological_drought(path_45, threshold, season, Area_Catchment, Catchment_Name, "yearly")
# end
# @time begin
# Drought_85 = compare_hydrological_drought(path_85, threshold, season, Area_Catchment, Catchment_Name, "yearly")
# end
# plot_drought_total_deficit(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
# plot_drought_statistics_change(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
# plot_drought_statistics_rel_change(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
# plot_drought_statistics(Drought_45, Drought_85, threshold, Catchment_Name, Area_Catchment, season)
#
# @time begin
# Drought_Extremes_45 = compare_hydrological_drought_extremes(path_45, Area_Catchment, Catchment_Name)
# Drought_Extremes_85 = compare_hydrological_drought_extremes(path_85, Area_Catchment, Catchment_Name)
# # plot_drought_extremes_statistics(Drought_Extremes_45, Drought_Extremes_85, Catchment_Name)
# plot_drought_extremes_statistics_change(Drought_Extremes_45, Drought_Extremes_85, Catchment_Name)
# end

# @time begin
Nr_Days_Drought_monthly_past_45, Nr_Days_Drought_monthly_future_45, Deficit_monthly_past_45, Deficit_monthly_future_45, Threshold_45 = monthly_days_Q90(path_45, Area_Catchment, Catchment_Name)
Nr_Days_Drought_monthly_past_85, Nr_Days_Drought_monthly_future_85, Deficit_monthly_past_85, Deficit_monthly_future_85, Threshold_85 = monthly_days_Q90(path_85, Area_Catchment, Catchment_Name)
writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Drought/monthly_days_deficit_Q90_4.5.txt",hcat(Nr_Days_Drought_monthly_past_45, Nr_Days_Drought_monthly_future_45, Deficit_monthly_past_45, Deficit_monthly_future_45, Threshold_45),',')
writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Drought/monthly_days_deficit_Q90_8.5.txt",hcat(Nr_Days_Drought_monthly_past_85, Nr_Days_Drought_monthly_future_85, Deficit_monthly_past_85, Deficit_monthly_future_85, Threshold_85),',')
# plot_monthly_low_flows(Nr_Days_Drought_monthly_past_45, Nr_Days_Drought_monthly_future_45, Nr_Days_Drought_monthly_past_85, Nr_Days_Drought_monthly_future_85, Catchment_Name, nr_runs)
# plot_monthly_deficit(Deficit_monthly_past_45, Deficit_monthly_future_45, Deficit_monthly_past_85, Deficit_monthly_future_85, Catchment_Name, nr_runs)
# end
# # --------------- SNOW MELT ----------------
#
# @time begin
# Snow_Storage_Past85, Snow_Storage_Future85 = snow_contribution_yearly_projections(path_85, Catchment_Name)
# Snow_Storage_Past45, Snow_Storage_Future45 = snow_contribution_yearly_projections(path_45, Catchment_Name)
# end
#plot_snow_storage_contribution(Snow_Storage_Past45, Snow_Storage_Future45, Snow_Storage_Past85, Snow_Storage_Future85, Catchment_Name)
#
# @time begin
# Monthly_Discharge_Change_85, monthly_Discharge_past_85, monthly_Discharge_future_85, months_85 = change_monthly_snowmelt(path_85, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Snow_Cover/snow_melt_months_8.5.txt",hcat(months_85, monthly_Discharge_past_85, monthly_Discharge_future_85, Monthly_Discharge_Change_85),',')
# end
# #
# @time begin
# Monthly_Discharge_Change_45, monthly_Discharge_past_45, monthly_Discharge_future_45, months_45 = change_monthly_snowmelt(path_45, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Snow_Cover/snow_melt_months_4.5.txt",hcat(months_45, monthly_Discharge_past_45, monthly_Discharge_future_45, Monthly_Discharge_Change_45),',')
# end
#
# @time begin
# Monthly_Discharge_Change_85, monthly_Discharge_past_85, monthly_Discharge_future_85, months_85 = change_monthly_snowstorage(path_85, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Snow_Cover/snow_storage_months_8.5.txt",hcat(months_85, monthly_Discharge_past_85, monthly_Discharge_future_85, Monthly_Discharge_Change_85),',')
# end
# #
# @time begin
# Monthly_Discharge_Change_45, monthly_Discharge_past_45, monthly_Discharge_future_45, months_45 = change_monthly_snowstorage(path_45, Catchment_Name)
# writedlm("/home/sarah/Master/Thesis/Results/Projektionen/"*Catchment_Name*"/PastvsFuture/Snow_Cover/snow_storage_months_4.5.txt",hcat(months_45, monthly_Discharge_past_45, monthly_Discharge_future_45, Monthly_Discharge_Change_45),',')
# end


##############optimsamin
mutable struct IterBBState{TΘ⁺<:VolumePartsIter, T} <: AbstractIterState{T}
  Θ⁺::TΘ⁺
  λ⁺::T

  starttime::Float64
  lasttime::Float64
  besttime::Float64
  maxelapsedsincebest::Float64
  maxelapsed::Float64

  steps::Int
  beststeps::Int
  maxstepssincebest::Int
  maxsteps::Int

  #lastreporttime::Float64
  #minreporttime::Float64
  stop::Union{Nothing,Symbol}
end

IterBBState(Θ::TΘ, ::Type{T};
  Θ⁺::TΘ = Θ |> deepcopy,
  λ⁺::T = T(Inf),

  starttime::Float64=time(),
  lasttime::Float64=time(),
  besttime::Float64=time(),
  maxelapsedsincebest::Float64=PARAM[:bbmaxelapsedsincebest],
  maxelapsed::Float64=PARAM[:bbmaxelapsed],

  #NOTE: using steps as a limit only makes sense within a particular method
  steps::Int=0,
  beststeps::Int=0,
  maxstepssincebest::Int=PARAM[:bbmaxstepsincebest],
  maxsteps::Int=PARAM[:bbmaxsteps],

  #lastreporttime::Float64
  #minreporttime::Float64
  stop::Union{Nothing,Symbol} = nothing,
  ) where {TΘ,T} = IterBBState{TΘ,T}(Θ⁺, λ⁺,
    starttime, lasttime, besttime, maxelapsedsincebest, maxelapsed,
    steps, beststeps, maxstepssincebest, maxsteps,
    #lastreporttime, minreporttime,
    stop)

#updates the problem state given the solution for an iteration
function callbackbb!(bb, Θ::AbstractVolumeParts{<:Any, <:Any, elTΘ},
  rΨ::Ref{IterBBState{TΘ,elTΨ}}) where {elTΘ, TΘ, elTΨ}

  #get the current time
  t::Float64 = time()

  #this next code is useful for comparing methods
  if (bb.start_time > rΨ[].lasttime) &&  (rΨ[].steps>0)#reset the state if the op solver itself reset
    rΨ[] = IterBBState(Θ, elTΨ, starttime=bb.start_time)
  end

  #now update the state
  Ψ = rΨ[]
  elapsed::Float64 = t - Ψ.starttime
  Ψ.lasttime = t

  Ψ.steps::Int = BlackBoxOptim.num_steps(bb)
  stepsperssecond::Float64 = round(Ψ.steps/elapsed, digits=1)
  stepssincelast::Int = bb.num_steps_since_last_report
  Δnumbetter::Int = bb.num_better_since_last_report
  improvementpercent::Int = Δnumbetter ÷ stepssincelast*100

  Ψ.steps += stepssincelast

  λ::elTΨ = bb |> BlackBoxOptim.best_fitness |> elTΨ
  if λ < Ψ.λ⁺ #keep track of the best solution
    copyto!(Ψ.Θ⁺.G, bb |> best_candidate |> Vector{elTΘ})
    Ψ.λ⁺ = λ
    Ψ.besttime=t
    Ψ.beststeps=Ψ.steps
  end

  #use the solver for stopping criteria
  #run through the stopping criteria
  elapsedsincebest::Int = round(t - Ψ.besttime)
  stepssincebest::Int = Ψ.steps - Ψ.beststeps

  println("bestloss=$(Ψ.λ⁺)," *
    "time=$(round(elapsed))/$(Ψ.maxelapsed), " *
    "timesincebest=$elapsedsincebest/$(Ψ.maxelapsedsincebest), " *
    "steps=$(Ψ.steps), " *
    "improvements=$(Δnumbetter), " *
    "stepssincebest≈$stepssincebest, " *
    "steps/s=$stepsperssecond")
  bb.num_better_since_last_report = 0
  bb.num_steps_since_last_report = 0

  if !(Ψ.stop===nothing)
    throw("Previous shutdown call failed to cut the optimizer. Manual abort!")
  elseif elapsed > Ψ.maxelapsed
    @info "STOP: max time $(round(elapsed))/$(Ψ.maxelapsed) reached"
    Ψ.stop=:maxtime
  elseif elapsedsincebest>Ψ.maxelapsedsincebest
    @info "STOP: max time with minimal improvement $elapsedsincebest/$(Ψ.maxelapsedsincebest) reached"
    Ψ.stop=:maxtimesincebest
  elseif Ψ.steps ≥ Ψ.maxsteps
    @info "STOP: max steps $(Ψ.steps)/$(Ψ.maxsteps) reached." *
      "(NOTE- the meaning of this depends on the model)"
    Ψ.stop=:maxiter
  elseif stepssincebest  ≥ Ψ.maxstepssincebest
    @info "STOP: max steps since best $stepssincebest/$(Ψ.maxstepssincebest) reached. " *
      "(NOTE- the meaning of this depends on the model)"
    Ψ.stop=:maxstepssincebest
  end

  Ψ.stop===nothing || BlackBoxOptim.shutdown_optimizer!(bb)

  return nothing
end

function BBOptions(callback, Θ::VolumePartsIter;
    bbadditionaloptions = [:PopulationSize, :TraceMode])

  dims = (K=size(Θ.A,1), T=size(Θ.A,2))

  #construct the required parameters
  method = PARAM[:bbmethod]
  searchrange = (PARAM[:bblowerbound], PARAM[:bbupperbound])
  numdimensions=(dims.T-1)*dims.K
  callbackinterval=PARAM[:iterminreporttime]

  #make sure we only exit via the callback, setting others to be ignored
  maxfuncevals = 0
  maxsteps = 0 #some large number

  #construct the
  bboptionsdict = Dict(
    :Method=>method,
    :SearchRange=>searchrange,
    :NumDimensions=>numdimensions,
    :MaxFuncEvals=>maxfuncevals,
    :MaxSteps=>maxsteps,
    :CallbackFunction=>callback,
    :CallbackInterval=>callbackinterval) # we willl run the callback function each loop

  #load additional optional parameters
  loadedoptionalparams::Vector{Symbol} = Vector{Symbol}()
  for k ∈ bbadditionaloptions
    #parse the option name into its PARAM name
    paramname::Symbol = k |> string |> lowercase |> (s)->"bb$s" |> Symbol
    if !(PARAM[paramname] === nothing)
      bboptionsdict[k] = PARAM[paramname]
      push!(loadedoptionalparams, k)
    end
  end
  @info "Loaded bb optional params $loadedoptionalparams"

  return (; bboptionsdict...)
end

function solveA!(loss, Θ::TΘ,
    Xv::AbstractXYIter{TM, TV, T}, _Xv::XYIterAlloc{TM, TV, T},
    SolveType::Val{:bb};
    RegressionType::Val = Val(PARAM[:iterregressiontype]),
    GrowthType::Val = Val(PARAM[:itergrowthtype])) where {TΘ, TM, TV, T}

    dims = (K=size(Θ.A,1), T=size(Θ.A,2))

    rΨ::Ref{IterBBState{TΘ,Float64}} = Ref(IterBBState(Θ,Float64))
    callback(opt) = callbackbb!(opt,Θ,rΨ)

    bboptions = BBOptions(callback, Θ)

    x::Vector{T} = initializeΘgvector(Θ.g, T, GrowthType) #provides an initial vector for the optimizer

    #create an objective stub

    function obj(x::Vector)
      λ=updateA₁!(x, loss, Θ, Xv, _Xv, RegressionType, GrowthType) |> Float64
      return isfinite(λ) ? λ : Inf
    end

    #test the objective
    obj(x)
    local res
    try
      res = bboptimize(obj; bboptions...)
    catch err
      @warn "Aborted optimization with error $err. "
      Ψ.stop = :aborted
    finally

      t=time()
      Ψ=rΨ[]
      elapsed::Int = round(time() - Ψ.starttime)
      elapsedsincebest::Int = round(t - Ψ.besttime)
      stepssincebest::Int = Ψ.steps - Ψ.beststeps
      Θ.G .= Ψ.Θ⁺.G
      updateA₁!(loss, Θ, Xv, _Xv, RegressionType)
      updateAfromGAₜ!(Θ,Xv, _Xv,1) #update to the final values of A

      b = IOBuffer()
      write(b, "***Results info (BlackBoxOptim):\n" *
        "stop_reason: $(Ψ.stop)\n" *
        "iterations: $(Ψ.steps)/$(Ψ.maxsteps)\n" *
        "elapsed_at_completion: $(elapsed)/$(Ψ.maxelapsed)\n" *
        "elapsed since best: ~$(elapsedsincebest)/$(Ψ.maxelapsedsincebest)\n" *
        "iter since best: ~$(stepssincebest)/$(Ψ.maxstepssincebest)\n" *
        "Best resid: $(Ψ.λ⁺)\n")

      write(b, "\n\n***Other Simulation parameters:\n" *
        "growth type: $GrowthType\n" *
        "regression method: $(RegressionType)\n" *
        "report interval: $(PARAM[:iterminreporttime])\n" *
        "Types (Matrix, Vector, T): $TM, $TV, $T\n" *
        "bboptions:\n")
      for n ∈ propertynames(bboptions)
        occursin("callback", n |> string |> lowercase) && continue #callback options covered elsewhere
        write(b, "..$n=>$(bboptions[n])\n")
      end

      iterinfo = String(take!(b))
      println(iterinfo)
      open("$(PARAM[:outputpath])\\$(iterresultsname(Ψ.λ⁺))_info.txt", "w+") do f
        write(f, iterinfo)
      end

      return Ψ.λ⁺
    end

    @assert false #should never get here

  end

  #######BBOptim


using RecipesBase

@recipe function f(g::Function, cols::Vector{DataColumn}; xtransform=identity, ztransform=identity, contrast=8)
    xs, ys, zs = datamap(g, cols)
    xs, ys = xs[1, :], ys[:, 1]
    xguide := get(cols[1].metadata, :xsymbol, nothing)
    yguide := cols[1].ysymbol
    xs = xtransform.(xs)
    zs = ztransform.(zs)

    xlims --> get(cols[1].metadata, :xlims, (xs[1], xs[end]))
    ylims --> get(cols[1].metadata, :ylims, length(ys) >= 2 ? (ys[1], ys[end]) : nothing)

    fzs = filter(!isnan, zs)
    maxz = maximum(abs.(fzs))
    stdz = std(fzs)
    clim = max(min(maxz, contrast*stdz), maxz / contrast)

    if all(x -> x >= 0, zs)
        clims --> (0.0, clim)
        seriescolor --> :bilbao
    else
        clims --> (-clim, clim)
        seriescolor --> :vik
    end

    # @series begin
    #     linewidth --> 1
    #     linecolor --> :black
    #     seriesalpha --> 0.3
    #     seriestype := :path
    #     primary := false
    #     markershape := :none

    #     (xs, mapreduce(col -> col.metadata.sim.eig.values, hcat, cols)')
    # end

    (xs, ys, zs)
end

@recipe function f(cols::Vector{DataColumn}; xtransform=identity, markerfunction=x->:black, step=1)
    xs, ys = ranges(cols)
    xguide --> get(cols[1].metadata, :xsymbol, nothing)
    xs = xtransform.(xs)
    eigvalues, energy_range = energy_bands(cols, step=step)

    xlims --> get(cols[1].metadata, :xlims, (xs[1], xs[end]))
    ylims --> get(cols[1].metadata, :ylims, length(ys) >= 2 ? (ys[1], ys[end]) : nothing)

    yticks --> energy_range

    linewidth --> 1
    linecolor --> :black
    seriesalpha --> 0.3
    legend --> false
    # markercolor --> mapreduce(col -> map(markerfunction, col.data), hcat, cols)
    # seriestype := :path
    # markershape := :none

    @series begin
        μs = mapreduce(col -> col.metadata.sim.μ, vcat, cols)
        linewidth --> 1
        linecolor := :red
        linestyle --> :dash
        (xs, μs)
    end

    (xs, eigvalues)
end

@recipe function f(g::Function, slices::Vector{DataSlice}; xtransform=identity)
    xs, ys = datamap(g, slices)
    xs = xtransform.(xs)
    xlims --> (xs[1], xs[end])
    ylim = maximum(abs.(ys))
    if all(y -> y >= 0, ys)
        ylims --> (0.0, ylim)
    else
        ylims --> (-ylim, ylim)
    end

    #linewidth --> 1

    (xs, ys)
end

@recipe function f(col::DataColumn; xtransform=identity, markerfunction=x->:black)
    xs = col.metadata.sim.eig.values
    xticks --> range(floor(Int, minimum(xs)), ceil(Int, maximum(xs)), step=1)

    xlims --> get(col.metadata, :xlims, (xs[1], xs[end]))

    linewidth --> 1
    linecolor --> :black
    legend --> false

    @series begin
        μ = col.metadata.sim.μ
        linewidth --> 1
        linecolor := :red
        linestyle --> :dash
        [μ]
    end

    xs
end

@recipe function f(g::Function, col::DataColumn; xtransform=identity, markerfunction=x->:black)
    xs = col.ys
    #xticks --> range(floor(Int, minimum(xs)), ceil(Int, maximum(xs)), step=1)

    ys = map(g, col.data)
    xguide := col.ysymbol

    xlims --> get(col.metadata, :ylims, length(ys) >= 2 ? (xs[1], xs[end]) : nothing)

    linewidth --> 1
    linecolor --> :black
    legend --> false

    xs, ys
end

@recipe function f(T::Type{DataScatter}, g::Function, cols::Vector{DataColumn}; xtransform=identity, ztransform=identity, contrast=8)
    xs, ys, zs = T(g, cols)
    xguide := get(cols[1].metadata, :xsymbol, nothing)
    yguide := cols[1].ysymbol
    xs = xtransform.(xs)
    zs = ztransform.(zs)

    xlims --> get(cols[1].metadata, :xlims, (xs[1], xs[end]))
    ylims --> get(cols[1].metadata, :ylims, length(ys) >= 2 ? (ys[1], ys[end]) : nothing)

    fzs = filter(!isnan, zs)
    maxz = maximum(abs.(fzs))
    stdz = std(fzs)
    clim = max(min(maxz, contrast*stdz), maxz / contrast)

    if all(x -> x >= 0, zs)
        clims --> (0.0, clim)
        color --> :bilbao
    else
        clims --> (-clim, clim)
        color --> :vik
    end

    zcolor --> zs
    ms --> max.(1.0, 20 .* sqrt.(abs.(zs)))
    α --> sqrt(0.5)

    (xs, ys)
end


using MathOptFormat, Test

const MOI = MathOptFormat.MOI
const MOIU = MOI.Utilities

@testset "MathOptFormat tests" begin
    @testset "$(file)" for file in ["CBF", "LP", "MOF", "MPS"]
        include("$(file)/$(file).jl")
    end

    @testset "Copying options" begin
        models = [
            MathOptFormat.CBF.Model,
            MathOptFormat.LP.Model,
            MathOptFormat.MOF.Model,
            MathOptFormat.MPS.Model
        ]
        for src in models
            model_src = src()
            for dest in models
                model_dest = dest()
                MOI.copy_to(model_dest, model_src)
                @test !isempty(sprint(io -> MOI.write_to_file(model_dest, io)))
            end
            model_dest = MOIU.MockOptimizer(MOIU.Model{Float64}())
            MOI.copy_to(model_dest, model_src)
        end
    end

    @testset "Calling gzip_open" begin
        @test_throws ArgumentError MathOptFormat.gzip_open((x) -> nothing,
                                                           "dummy.gz", "a")
        @test_throws ArgumentError MathOptFormat.gzip_open((x) -> nothing,
                                                           "dummy.gz", "r+")
        @test_throws ArgumentError MathOptFormat.gzip_open((x) -> nothing,
                                                           "dummy.gz", "w+")
        @test_throws ArgumentError MathOptFormat.gzip_open((x) -> nothing,
                                                           "dummy.gz", "a+")

    end
end


function SQ._collect{T<:AbstractTable}(
    g_tbl::Grouped{T}, g::SQ.SummarizeNode
)::Grouped
    ngroupbys = length(g_tbl.groupbys)
    return group_summarize(Val{ngroupbys}(), g_tbl, g)
end

new_groupby_column(groupby_field, tbl) = similar(tbl[groupby_field], 0)

function build_groupby_resources{T<:AbstractTable}(g_tbl::Grouped{T})
    groupby_columns = Dict{Symbol, Vector}()
    groupby_fields = map(
        x -> isa(x, Symbol) ? x : g_tbl.predicate_aliases[x],
        g_tbl.groupbys
    )
    for groupby_field in groupby_fields
        groupby_column = new_groupby_column(groupby_field, g_tbl.source)
        groupby_columns[groupby_field] = groupby_column
    end
    return groupby_fields, groupby_columns
end

@generated function group_summarize{N}(::Val{N}, g_tbl, q)
    # generate expression for (group_1, ..., group_n) inner loop variables
    group_tuple_ex = Expr(:tuple)
    for i in 1:N
        push!(group_tuple_ex.args, Symbol("group_$i"))
    end

    return quote
        groupby_fields, groupby_columns = build_groupby_resources(g_tbl)
        res_columns = Dict{Symbol, Vector}()
        group_levels = g_tbl.group_levels

        @nloops $N group group_levels begin
            if !haskey(g_tbl.group_indices, $group_tuple_ex)
                continue
            end

            # push group levels to appropriate groupby columns
            @nexprs $N j-> push!(groupby_columns[groupby_fields[j]], group_j)

            # compute each summarization and insert into appropriate
            # result column
            for helper in q.helpers
                res_field, f, g, arg_fields = SQ.parts(helper)
                if haskey(res_columns, res_field)
                    push!(
                        res_columns[res_field],
                        rhs_summarize(f, g, arg_fields, g_tbl, $group_tuple_ex)
                    )
                else
                    res_columns[res_field] =
                        [rhs_summarize(f, g, arg_fields, g_tbl, $group_tuple_ex)]
                end
            end
        end

        new_tbl = default(g_tbl.source)
        for groupby_field in groupby_fields
            new_tbl[groupby_field] = groupby_columns[groupby_field]
        end
        for helper in q.helpers
            res_field, f, g, arg_fields = SQ.parts(helper)
            new_tbl[res_field] = res_columns[res_field]
        end
        return groupby(new_tbl, g_tbl.groupbys, g_tbl.predicate_aliases)
    end
end

function rhs_summarize(f, g, arg_fields, g_tbl, key)
    T, row_itr = _preprocess(f, arg_fields, g_tbl, key)

    # Allocate a temporary column.
    temp = Array(T, 0)

    # Fill the new column in row-by-row, skipping nulls.
    grow_nonnull_output!(temp, f, row_itr)

    # Return the summarization function applied to the temporary.
    return g(temp)
end


# save to disk/read from disk
printstyled("  Native file I/O\n", color=:light_green)
savfile1 = "test.dat"
fs = 100.0
nc = 10
nx = 2^12

D = randSeisData(nc, nx=nx, s=1.0)
t = mk_t(nx, D.t[1][1,2])
for i in 1:D.n
  D.fs[i] = fs
  D.t[i] = copy(t)
end
S = convert(NodalData, D)
S.ox = (rand()-0.5) * 360.0
S.oy = (rand()-0.5) * 90.0
S.oz = rand()*1000.0
S.info["foo"] = "bar"
pop_nodal_dict!(S.info)

printstyled("    NodalChannel\n", color=:light_green)
C = getindex(S, 1)
wseis(savfile1, C)
R = rseis(savfile1)[1]
@test R == C

printstyled("    NodalData\n", color=:light_green)
wseis(savfile1, S)
R = rseis(savfile1)[1]
@test R == S
rm(savfile1)


using GnuplotRecipes
using Test
using Gnuplot, DataFrames, Measurements, PNGFiles, ColorTypes, FixedPointNumbers

function compare_plot(name)
    fn = "img/$name.png"
    fn_expected = "img/$name.expect.png"
    fn_diff = "img/$name.diff.png"
    isdir("img") || mkdir("img")
    save(term="pngcairo size 640,360 fontscale 0.8", output=fn)
    cur = PNGFiles.load(fn);
    if isfile(fn_expected)
        exp = PNGFiles.load(fn_expected);
    else
        exp = cur;
        cp(fn, fn_expected)
    end
    if exp != cur
        diff = RGB{N0f8}(1,1,1) .- (RGB{Float16}.(cur) .- RGB{Float16}.(exp) .|> abs .|> RGB{N0f8})
        PNGFiles.save(fn_diff, diff);
    end

    return exp == cur
end

macro gpok(args...)
    gp_call = :(@gp $(esc.(args)...))
    if ! Gnuplot.options.gpviewer
        gp_call = :(display($gp_call))
    end
    return quote
        $gp_call
        compare_plot($("gp_" * join(string.(args))))
    end
end

orig_term = Gnuplot.options.term
if !isinteractive()
    Gnuplot.options.term = "unknown"
end

@testset "GnuplotRecipes.jl" begin
    # Plain numbers
    dfnum = DataFrame(names=["a", "b", "c"], temp=10:12, speed=4:-1:2)
    @test @gpok bars(dfnum)
    # Measurement values
    df = DataFrame(names=["very long label", "b", "c"], temp=10:12, speed=collect(5:-1.5:2) .± 1)
    @test @gpok bars(df)
    @test @gpok bars(df, errorbars="lw 5 lc rgb '#ff0000'")
    @test @gpok bars(df, fill_style="pattern 1")
    @test @gpok bars(df, gap=2)
    @test @gpok bars(df, box_width=1)
    @test @gpok bars(df, label_rot=0)
    @test @gpok bars(df, label_rot=90) # visually check that labels are top-aligned
    @test @gpok bars(df, y2cols=[:speed])
    @test @gpok bars(df, y2cols=[:speed, :temp])
    @test @gpok bars(df, hist_style = "rowstacked")
    @test_logs (:warn, r"columnstacked") @gp bars(df, hist_style = "columnstacked")
    @test_throws AssertionError @gp bars(df, y2cols=[:bad])
    # Symbols as lables
    @test @gpok bars(DataFrame(l=[:a, :b], v=[2, 1]))
    # xlabelcol
    @test @gpok bars(DataFrame(v=[2, 1, 0.5], l=[:a, :b, :c]), xlabelcol=2)
    @test @gpok bars(DataFrame(v=[2, 1, 0.5], l=[:a, :b, :c]), xlabelcol=:l)
end

Gnuplot.options.term = orig_term


using ElectrodermalActivity

# Specify the folder containing the raw Empatica data.
input_directory = "/home/zygmunt/Downloads/edadata/EDA LAB DATA/107/experiment"
output_directory = "/home/zygmunt/Downloads/edadata/EDA LAB DATA/107/experiment/processed"
#input_directory = "C:/Users/Spock/Documents/Empatica Data/"
#output_directory = "C:/Users/Spock/Documents/Empatica Data/processed"

# Create the output_directory if it doesn't already exist.
mkpath(output_directory)

#=
 The marked interval starts a specified number of seconds prior to the
 first marker and ends at the first marker.
=#
interval_name_1 = "Interval 1"
seconds_prior_marker_1 = 60

#=
  The second marked interval starts at the first marker and continues for a specified
  number of seconds.
=#
interval_name_2 = "Interval 2"
seconds_after_marker_1 = 60

#=
 The third marked interval starts a specified number of seconds prior to the
 second marker and ends at the second marker.
=#
interval_name_3 = "Interval 3"
seconds_prior_marker_2 = 60

#=
  The fourth marked interval starts at the second marker and continues for a specified
  number of seconds.
=#
interval_name_4 = "Interval 4"
seconds_after_marker_2 = 60

interval_names = (name_1 = interval_name_1,
                  name_2 = interval_name_2,
                  name_3 = interval_name_3,
                  name_4 = interval_name_4)
offsets = (offset_1 = seconds_prior_marker_1,
           offset_2 = seconds_after_marker_1,
           offset_3 = seconds_prior_marker_2,
           offset_4 = seconds_after_marker_2)

a = prepare_empatica(input_directory, output_directory, interval_names, offsets)


using Showoff
using Test
using Dates
using Printf

const drops0s = !isdefined(Base, :Ryu)

@testset "Internals" begin
    @test Showoff.@grisu_ccall(1, 2, 3) === nothing
    if isdefined(Showoff, :Grisu)
        @test Showoff.grisu(1.0, Showoff.Grisu.SHORTEST, 2) == (1, 1, false, Showoff.Grisu.DIGITS)
    end

    let x = [1.0, Inf, 2.0, NaN]
        @test Showoff.concrete_minimum(x) == 1.0
        @test Showoff.concrete_maximum(x) == 2.0
    end

    @test_throws ArgumentError Showoff.concrete_minimum([])
    @test_throws ArgumentError Showoff.concrete_maximum([])

    let x = [1.12345, 4.5678]
        @test Showoff.plain_precision_heuristic(x) == 5
        @test Showoff.scientific_precision_heuristic(x) == 6
    end
end

@testset "Formatting" begin
    @test Showoff.format_fixed(-10.0, 0) == "-10"
    @test Showoff.format_fixed(0.012345, 3) == "0.012"
    @test Showoff.format_fixed(Inf, 1) == "∞"
    @test Showoff.format_fixed(-Inf, 1) == "-∞"
    @test Showoff.format_fixed(NaN, 1) == "NaN"
    @test Showoff.format_fixed_scientific(0.0, 1, false) == "0"
    @test Showoff.format_fixed_scientific(Inf, 1, false) == "∞"
    @test Showoff.format_fixed_scientific(-Inf, 1, false) == "-∞"
    @test Showoff.format_fixed_scientific(NaN, 1, false) == "NaN"
    @test Showoff.format_fixed_scientific(0.012345678, 4, true) == "12.346×10⁻³"
    @test Showoff.format_fixed_scientific(0.012345678, 4, false) == "1.2346×10⁻²"
    @test Showoff.format_fixed_scientific(-10.0, 4, false) == (drops0s ? "-1.000×10¹" : "-1.0000×10¹")
    @test Showoff.format_fixed_scientific(-10.0, 4, true) == "-10.000×10⁰"
    @test Showoff.format_fixed_scientific(-10.0, 4, false)[1:end-5] == (drops0s ? @sprintf("%0.4e", -10.0)[1:end-5] : @sprintf("%0.4e", -10.0)[1:end-4])
    @test Showoff.format_fixed_scientific(1.23456e7, 3, false)[1:end-5] == @sprintf("%0.3e", 1.23456e7)[1:end-4]
    @test Showoff.format_fixed_scientific(2.99999999999999956E-16, 2, false) == (drops0s ? "3.0×10⁻¹⁶" : "3.00×10⁻¹⁶")
end

@testset "Showoff" begin
    x = [1.12345, 4.5678]
    @test showoff(x) == ["1.12345", "4.56780"]
    @test showoff([0.0, 50000.0]) == (drops0s ? ["0", "5×10⁴"] : ["0", "5.0×10⁴"])
    @test showoff(x, :plain) == ["1.12345", "4.56780"]
    @test showoff(x, :scientific) == (drops0s ? ["1.12345×10⁰", "4.56780×10⁰"] : ["1.123450×10⁰", "4.567800×10⁰"])
    @test showoff(x, :engineering) == showoff(x, :scientific)
    @test showoff([DateTime("2017-04-11", "yyyy-mm-dd")]) == ["Apr 11, 2017"]
    @test showoff(["a", "b"]) == ["\"a\"", "\"b\""]
    @test showoff([1, 1e39]) == (drops0s ? ["1×10⁰", "1×10³⁹"] : ["1.0×10⁰", "1.0×10³⁹"])
    @test_throws ArgumentError showoff(x, :nevergonnagiveyouup)
    @test showoff([Inf, Inf, NaN]) == ["Inf", "Inf", "NaN"]
end


using Agents
using Serialization

function restore_snapshot(filename = "snapshot100kr1.dat")
    space = GridSpace((k1,k2))
    covidmodel = ABM(CovidAgent.GenericAgent,space; scheduler = fastest)
    (agents, lochh, locf, unemplist, shorttimelist, empcount, unempcount, shorttimecount, oldcount, unemp, firms, hh, tau, divperhh, weeklyconsumption) = open(filename,"r") do snapshot_file
        deserialize(snapshot_file)
    end
    for (id, agent) in agents
        add_agent_pos!(agent,covidmodel)
    end
    return covidmodel, lochh, locf, unemplist, shorttimelist, empcount, unempcount, shorttimecount, oldcount, unemp, firms, hh, tau, divperhh, weeklyconsumption
end


# infinit
rstudy001,cstudy001=simpleprover("study/study001.cnf",12,3)
rstudy001n,cstudy001n=simpleprover("study/study001n.cnf",12,3)
rstudy001n2,cstudy001n2=simpleprover("study/study001n2.cnf",12,3)
rstudy001n3,cstudy001n3=simpleprover("study/study001n3.cnf",12,3)

# finite
rstudy002,cstudy002=simpleprover("study/study002.cnf",4,3)
rstudy003,cstudy003=simpleprover("study/study003.cnf",4,3)

#
rstudy004,cstudy004=simpleprover("study/study004.cnf",10,3)
rstudy005,cstudy005=simpleprover("study/study005.cnf",5,5)
#rstudy005,cstudy005=simpleprover("study/study005.cnf",20,15)