module LE1NAFEMS_examples
using FinEtools
using FinEtoolsDeforLinear
using FinEtoolsDeforLinear.AlgoDeforLinearModule
using FinEtools.MeshExportModule
using FinEtools.MeshImportModule: import_ABAQUS
using Statistics: mean
using LinearAlgebra: Symmetric, cholesky, norm

# function LE1NAFEMS_compare_meshes()
#     Thick0 = 0.1*phun("m")/2.0 # to account for the symmetry reduction

#     ref  = 0
#     Thickness = Thick0
#     tolerance = Thickness/2^ref/300.; # Geometrical tolerance

#     fens,fes = T10block(1.0, pi/2, Thickness, 2^ref*5, 2^ref*6, 1; orientation = :b)
#     for i=1:count(fens)
#         t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
#         fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
#     end
#     println("$((count(fens), count(fes)))")

#     output = import_ABAQUS("LE1AbaqusExport-C3D10HS-5-6-1.inp")
#     fens1, fes1 = output["fens"], output["fesets"][1]
#     println("$((count(fens1), count(fes1[1])))")

#     fens, newfes1, fes2 = mergemeshes(fens,fes, fens1,fes1[1], tolerance)
#     # fes = cat(fes2, newfes1)
#     # println("$((count(fens), count(fes)))")

#     File =  "a1.vtk"
#     vtkexportmesh(File, newfes1.conn, fens.xyz,
#     FinEtools.MeshExportModule.T10)
#     @async run(`"paraview.exe" $File`)
#     File =  "a2.vtk"
#     vtkexportmesh(File, fes2.conn, fens.xyz,
#     FinEtools.MeshExportModule.T10)
#     @async run(`"paraview.exe" $File`)

#     #
#     # fens,fes = H8block(1.0, pi/2, Thickness, 2^ref*5, 2^ref*6, 1)
#     # for i=1:count(fens)
#     #     t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
#     #     fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
#     # end
#     # println("$((count(fens), count(fes)))")
#     #
#     # fens1,fes1 = import_ABAQUS("LE1AbaqusExport-C3D8S-80-96-1.inp")
#     # println("$((count(fens1), count(fes1[1])))")
#     #
#     #  fens, newfes1, fes2 = mergemeshes(fens,fes, fens1,fes1[1], tolerance)
#     #  fes = cat(fes2, newfes1)
#     #  println("$((count(fens), count(fes)))")
#     #
#     # File =  "a.vtk"
#     # vtkexportmesh(File, fes.conn, fens.xyz,
#     #                FinEtools.MeshExportModule.H8)
#     # @async run(`"paraview.exe" $File`)

#     true

# end # LE1NAFEMS_compare_meshes

function LE1NAFEMS_MSH8()
    println("LE1NAFEMS, 3D version."        )
    t0 = time()

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Radius = 1.0*phun("m")
    Thickness = 0.1*phun("m")
    n = 2; # number of elements per side
    tolerance = 1.0/n/1000.; # Geometrical tolerance

    fens,fes = Q4block(1.0, pi/2, n, n*2)
    fens,fes  = H8extrudeQ4(fens, fes,
    1, (xyz, layer)->[xyz[1], xyz[2], (layer)*Thickness]);

    bdryfes = meshboundary(fes);
    icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
    for i=1:count(fens)
        t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
        fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
    end

    geom = NodalField(fens.xyz)
    u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

    l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
    setebc!(u,l1,true, 2, 0.0)
    l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
    setebc!(u,l1,true, 1, 0.0)
    l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
    setebc!(u,l1,true, 3, 0.0)

    applyebc!(u)
    numberdofs!(u)

    el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), GaussRule(2, 2)))
    function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
        pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
        forceout .=    vec(p*pt/norm(pt));
        return forceout
    end
    fi = ForceIntensity(FFlt, 3, pfun);
    F2 = distribloads(el1femm, geom, u, fi, 2);

    # Note that the material object needs to be created with the proper
    # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
    MR = DeforModelRed3D

    material = MatDeforElastIso(MR, E, nu)

    femm = FEMMDeforLinearMSH8(MR, IntegDomain(fes, GaussRule(3, 2)), material)

    # The geometry field now needs to be associated with the FEMM
    femm = associategeometry!(femm, geom)

    K = stiffness(femm, geom, u)
    K = cholesky(K)
    U = K\(F2)
    scattersysvec!(u, U[:])

    nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, 0.0],inflate=tolerance);
    thecorneru = zeros(FFlt,1,3)
    gathervalues_asmat!(u, thecorneru, nl);
    thecorneru = thecorneru/phun("mm")
    println("$(time()-t0) [s];  displacement =$(thecorneru) [MM] as compared to reference [-0.10215,0] [MM]")

    fld= fieldfromintegpoints(femm, geom, u, :Cauchy, 2;
    nodevalmethod = :averaging, reportat = :meanonly)
    println("Sigma_y =$(fld.values[nl,1][1]/phun("MPa")) as compared to reference sigma_yD = $(sigma_yD/phun("MPa")) [MPa]")
    fld= fieldfromintegpoints(femm, geom, u, :Cauchy, 2;
    nodevalmethod = :averaging, reportat = :extrapmean)
    println("Sigma_y =$(fld.values[nl,1][1]/phun("MPa")) as compared to reference sigma_yD = $(sigma_yD/phun("MPa")) [MPa]")
    fld= fieldfromintegpoints(femm, geom, u, :Cauchy, 2;
    nodevalmethod = :averaging, reportat = :extraptrend)
    println("Sigma_y =$(fld.values[nl,1][1]/phun("MPa")) as compared to reference sigma_yD = $(sigma_yD/phun("MPa")) [MPa]")

    fld= fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :invdistance, reportat = :meanonly)
    println("Sigma_y =$(fld.values[nl,1][1]/phun("MPa")) as compared to reference sigma_yD = $(sigma_yD/phun("MPa")) [MPa]")
    fld= fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :averaging, reportat = :extrapmean)
    println("Sigma_y =$(fld.values[nl,1][1]/phun("MPa")) as compared to reference sigma_yD = $(sigma_yD/phun("MPa")) [MPa]")
    fld= fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :averaging, reportat = :extraptrend)
    println("Sigma_y =$(fld.values[nl,1][1]/phun("MPa")) as compared to reference sigma_yD = $(sigma_yD/phun("MPa")) [MPa]")

    println("$(n), $(fld.values[nl,1][1]/phun("MPa"))")

    File =  "a.vtk"
    vtkexportmesh(File, fes.conn, geom.values,
    FinEtools.MeshExportModule.H8; vectors=[("u", u.values)],
    scalars=[("sigmay", fld.values)])
    @async run(`"paraview.exe" $File`)
    true

end # LE1NAFEMS_MSH8

function LE1NAFEMS_MSH8_convergence()
    # Example from the "Improved Stress Recovery for Mean-strain Finite Elements" paper by Sivapuram and Krysl, 2017
    println("LE1NAFEMS, 3D version. MSH8"        )
    t0 = time()

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Thick0 = 0.1*phun("m")/2.0 # to account for the symmetry reduction

    sigyderrs = Dict{Symbol, FFltVec}()

    nnodes = []
    for extrapolation in [:extraptrend :extrapmean]
        sigyderrs[extrapolation] = FFltVec[]
        nnodes = []
        for ref in 0:1:4
            # Thickness = Thick0
            Thickness = Thick0/2^ref
            tolerance = Thickness/2^ref/1000.; # Geometrical tolerance

            fens,fes = H8block(1.0, pi/2, Thickness, 2^ref*5, 2^ref*6, 1)

            bdryfes = meshboundary(fes);
            icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
            for i=1:count(fens)
                t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
                fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
            end

            geom = NodalField(fens.xyz)
            u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

            l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 2, 0.0)
            l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 1, 0.0)
            l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
            setebc!(u,l1,true, 3, 0.0)

            applyebc!(u)
            numberdofs!(u)

            el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), GaussRule(2, 2)))
            function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
                pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
                forceout .=    vec(p*pt/norm(pt));
                return forceout
            end
            fi = ForceIntensity(FFlt, 3, pfun);
            F2 = distribloads(el1femm, geom, u, fi, 2);

            # Note that the material object needs to be created with the proper
            # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
            MR = DeforModelRed3D

            material = MatDeforElastIso(MR, E, nu)

            femm = FEMMDeforLinearMSH8(MR, IntegDomain(fes, GaussRule(3, 2)), material)

            # The geometry field now needs to be associated with the FEMM
            femm = associategeometry!(femm, geom)

            K = stiffness(femm, geom, u)
            K = cholesky(K)
            U = K\(F2)
            scattersysvec!(u, U[:])

            nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, Thickness], inflate=tolerance);
            thecorneru = zeros(FFlt,length(nl),3)
            gathervalues_asmat!(u, thecorneru, nl);
            thecorneru = mean(thecorneru, dims=1)[1]/phun("mm")
            println("displacement =$(thecorneru) vs -0.10215 [MM]")

            fld = fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :averaging, reportat = extrapolation)
            sigyd = mean(fld.values[nl,1], dims=1)[1]
            println("Sigma_y =$(sigyd/phun("MPa")) vs $(sigma_yD/phun("MPa")) [MPa]")

            println("$extrapolation, $(count(fes)), $(sigyd/phun("MPa"))")
            push!(nnodes, count(fes))
            push!(sigyderrs[extrapolation], (sigyd/sigma_yD - 1.0))
            # File =  "a.vtk"
            # vtkexportmesh(File, fes.conn, geom.values,
            # FinEtools.MeshExportModule.H8; vectors=[("u", u.values)],
            # scalars=[("sigmay", fld.values)])
            # @async run(`"paraview.exe" $File`)
        end
    end

    File = "LE1NAFEMS_MSH8_convergence.CSV"
    savecsv(File, nnodes=vec(nnodes), sigyderrtrend=vec(sigyderrs[:extraptrend]), sigyderrmean=vec(sigyderrs[:extrapmean]))

end # LE1NAFEMS_MSH8_convergence

function LE1NAFEMS_MSH8_export()
    println("LE1NAFEMS, 3D version."        )
    t0 = time()

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Radius = 1.0*phun("m")
    Thickness = 0.1*phun("m")
    n = 64; # number of elements per side
    tolerance = 1.0/n/1000.; # Geometrical tolerance

    fens,fes = Q4block(1.0, pi/2, n, n*2)
    fens,fes  = H8extrudeQ4(fens, fes,
    1, (xyz, layer)->[xyz[1], xyz[2], (layer)*Thickness]);

    bdryfes = meshboundary(fes);
    icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
    for i=1:count(fens)
        t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
        fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
    end

    geom = NodalField(fens.xyz)
    u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

    l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
    setebc!(u,l1,true, 2, 0.0)
    l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
    setebc!(u,l1,true, 1, 0.0)
    l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
    setebc!(u,l1,true, 3, 0.0)

    applyebc!(u)
    numberdofs!(u)

    el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), GaussRule(2, 2)))
    function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
        pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
        forceout .=    vec(p*pt/norm(pt));
        return forceout
    end
    fi = ForceIntensity(FFlt, 3, pfun);
    F2 = distribloads(el1femm, geom, u, fi, 2);

    # Note that the material object needs to be created with the proper
    # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
    MR = DeforModelRed3D

    material = MatDeforElastIso(MR, E, nu)

    femm = FEMMDeforLinearMSH8(MR, IntegDomain(fes, GaussRule(3, 2)), material)

    # The geometry field now needs to be associated with the FEMM
    femm = associategeometry!(femm, geom)

    K = stiffness(femm, geom, u)
    K = cholesky(K)
    U = K\(F2)
    scattersysvec!(u, U[:])

    nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, 0.0],inflate=tolerance);
    thecorneru = zeros(FFlt,1,3)
    gathervalues_asmat!(u, thecorneru, nl);
    thecorneru = thecorneru/phun("mm")
    println("$(time()-t0) [s];  displacement =$(thecorneru) [MM] as compared to reference [-0.10215,0] [MM]")

    fld= fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :averaging, reportat = :extraptrend)
    println("Sigma_y =$(fld.values[nl,1][1]/phun("MPa")) as compared to reference sigma_yD = $(sigma_yD/phun("MPa")) [MPa]")

    println("$(n), $(fld.values[nl,1][1]/phun("MPa"))")

    File =  "a.vtk"
    vtkexportmesh(File, fes.conn, geom.values,
    FinEtools.MeshExportModule.H8; vectors=[("u", u.values)],
    scalars=[("sigmay", fld.values)])
    @async run(`"paraview.exe" $File`)
    true

end # LE1NAFEMS_MSH8_export

function LE1NAFEMS_MST10_convergence()
    # Example from the "Improved Stress Recovery for Mean-strain Finite Elements" paper by Sivapuram and Krysl, 2017
    println("LE1NAFEMS, 3D version. MST10"        )
    t0 = time()

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Thick0 = 0.1*phun("m")/2.0 # to account for the symmetry reduction

    sigyderrs = Dict{Symbol, FFltVec}()

    nnodes = []
    for extrapolation in [:extraptrend :extrapmean]
        sigyderrs[extrapolation] = FFltVec[]
        nnodes = []
        for ref in 0:1:4
            # Thickness = Thick0
            Thickness = Thick0/2^ref
            tolerance = Thickness/2^ref/1000.; # Geometrical tolerance

            fens,fes = T10block(1.0, pi/2, Thickness, 2^ref*5, 2^ref*6, 1; orientation = :b)

            bdryfes = meshboundary(fes);
            icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
            for i=1:count(fens)
                t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
                fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
            end

            geom = NodalField(fens.xyz)
            u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

            l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 2, 0.0)
            l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 1, 0.0)
            l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
            setebc!(u,l1,true, 3, 0.0)

            applyebc!(u)
            numberdofs!(u)

            el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), TriRule(3)))
            function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
                pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
                forceout .=    vec(p*pt/norm(pt));
                return forceout
            end
            fi = ForceIntensity(FFlt, 3, pfun);
            F2 = distribloads(el1femm, geom, u, fi, 2);

            # Note that the material object needs to be created with the proper
            # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
            MR = DeforModelRed3D

            material = MatDeforElastIso(MR, E, nu)

            femm = FEMMDeforLinearMST10(MR, IntegDomain(fes, TetRule(4)), material)

            # The geometry field now needs to be associated with the FEMM
            femm = associategeometry!(femm, geom)

            K = stiffness(femm, geom, u)
            K = cholesky(K)
            U = K\(F2)
            scattersysvec!(u, U[:])

            nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, Thickness], inflate=tolerance);
            thecorneru = zeros(FFlt,length(nl),3)
            gathervalues_asmat!(u, thecorneru, nl);
            thecorneru = mean(thecorneru, dims=1)[1]/phun("mm")
            println("displacement =$(thecorneru) vs -0.10215 [MM]")

            fld = fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :averaging, reportat = extrapolation)
            sigyd = mean(fld.values[nl,1], dims=1)[1]
            println("Sigma_y =$(sigyd/phun("MPa")) vs $(sigma_yD/phun("MPa")) [MPa]")

            println("$extrapolation, $(count(fes)), $(sigyd/phun("MPa"))")
            push!(nnodes, count(fens))
            push!(sigyderrs[extrapolation], abs(sigyd/sigma_yD - 1.0))
            # File =  "a.vtk"
            # vtkexportmesh(File, fes.conn, geom.values,
            # FinEtools.MeshExportModule.H8; vectors=[("u", u.values)],
            # scalars=[("sigmay", fld.values)])
            # @async run(`"paraview.exe" $File`)
        end
    end

    File = "LE1NAFEMS_MST10_convergence.CSV"
    savecsv(File, nnodes=vec(nnodes), sigyderrtrend=vec(sigyderrs[:extraptrend]), sigyderrdefault=vec(sigyderrs[:extrapmean]))

end # LE1NAFEMS_MST10_convergence

function LE1NAFEMS_MST10_one()
    println("LE1NAFEMS, 3D version."        )
    t0 = time()

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Thick0 = 0.1*phun("m")/2.0 # to account for the symmetry reduction

    sigyderrs = Dict{Symbol, FFltVec}()

    for extrapolation in [:extraptrend]
        sigyderrs[extrapolation] = FFltVec[]
        nnodes = []
        for ref in 1:1
            Thickness = Thick0
            # Thickness = Thick0/2^ref
            tolerance = Thickness/2^ref/1000.; # Geometrical tolerance

            fens,fes = T10block(1.0, pi/2, Thickness, 2^ref*5, 2^ref*6, 1; orientation = :b)

            bdryfes = meshboundary(fes);
            icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
            for i=1:count(fens)
                t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
                fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
            end

            geom = NodalField(fens.xyz)
            u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

            l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 2, 0.0)
            l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 1, 0.0)
            l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
            setebc!(u,l1,true, 3, 0.0)

            applyebc!(u)
            numberdofs!(u)

            el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), TriRule(3)))
            function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
                pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
                forceout .=    vec(p*pt/norm(pt));
                return forceout
            end
            fi = ForceIntensity(FFlt, 3, pfun);
            F2 = distribloads(el1femm, geom, u, fi, 2);

            # Note that the material object needs to be created with the proper
            # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
            MR = DeforModelRed3D

            material = MatDeforElastIso(MR, E, nu)

            femm = FEMMDeforLinearMST10(MR, IntegDomain(fes, TetRule(4)), material)

            # The geometry field now needs to be associated with the FEMM
            femm = associategeometry!(femm, geom)

            K = stiffness(femm, geom, u)
            K = cholesky(K)
            U = K\(F2)
            scattersysvec!(u, U[:])

            nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, Thickness], inflate=tolerance);
            thecorneru = zeros(FFlt,length(nl),3)
            gathervalues_asmat!(u, thecorneru, nl);
            thecorneru = mean(thecorneru, dims=1)[1]/phun("mm")
            println("displacement =$(thecorneru) vs -0.10215 [MM]")

            fld = fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :averaging, reportat = extrapolation)
            sigyd = mean(fld.values[nl,1], dims=1)[1]
            println("Sigma_y =$(sigyd/phun("MPa")) vs $(sigma_yD/phun("MPa")) [MPa]")

            println("$extrapolation, $(count(fes)), $(sigyd/phun("MPa"))")
            push!(nnodes, count(fens))
            push!(sigyderrs[extrapolation], abs(sigyd/sigma_yD - 1.0))
            File =  "LE1NAFEMS_MST10_a.vtk"
            vtkexportmesh(File, fes.conn, geom.values,
            FinEtools.MeshExportModule.T10; vectors=[("u", u.values)],
            scalars=[("sigmay", fld.values)])
            @async run(`"paraview.exe" $File`)
        end
    end

    #
    #
    # using DataFrames
    # using CSV
    #
    # df = DataFrame(nnodes=vec(nnodes),
    #     sigyderrtrend=vec(sigyderrs[:extraptrend]),
    #     sigyderrdefault=vec(sigyderrs[:extrapmean]))
    # File = "LE1NAFEMS_MST10_convergence.CSV"
    # CSV.write(File, df)
    # @async run(`"paraview.exe" $File`)

end # LE1NAFEMS_MST10_one

# using SymPy
# function LE1NAFEMS_MST10_single()

#     @vars ua ub uc ud
#     @vars va vb vc vd
#     @vars wa wb wc wd
#     @vars x y z
#     u(x, y, z) = ua*x +  ub*y +  uc*z +  ud
#     v(x, y, z) = va*x +  vb*y +  vc*z +  vd
#     w(x, y, z) = wa*x +  wb*y +  wc*z +  wd

#     ex = diff(u(x, y, z), x)
#     ey = diff(v(x, y, z), y)
#     ez = diff(w(x, y, z), z)
#     gxy = diff(v(x, y, z), x) + diff(u(x, y, z), y)
#     gxz = diff(w(x, y, z), x) + diff(u(x, y, z), z)
#     gyz = diff(w(x, y, z), y) + diff(v(x, y, z), z)

#     display(ex)
#     display(ey)
#     display(ez)
#     display(gxy)
#     display(gxz)
#     display(gyz)

# end # LE1NAFEMS_MST10_single

function LE1NAFEMS_MST10_stresses_nodal()
    elementtag = "MST10"
    println("LE1NAFEMS, 3D version. Element: $(elementtag)")

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Thick0 = 0.1*phun("m")/2.0 # to account for the symmetry reduction

    for extrapolation in [:extraptrend :extrapmean]
        convergencestudy = FDataDict[]
        for ref in  [0, 1, 2, 3, 4]
            Thickness = Thick0
            tolerance = Thickness/2^ref/1000.; # Geometrical tolerance

            fens,fes = T10block(1.0, pi/2, Thickness, 2^ref*2, 2^ref*3, 1; orientation = :b)

            bdryfes = meshboundary(fes);
            icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
            for i=1:count(fens)
                t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
                fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
            end

            geom = NodalField(fens.xyz)
            u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

            l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 2, 0.0)
            l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 1, 0.0)
            l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
            setebc!(u,l1,true, 3, 0.0)

            applyebc!(u)
            numberdofs!(u)

            el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), TriRule(3)))
            function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
                pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
                forceout .=    vec(p*pt/norm(pt));
                return forceout
            end
            fi = ForceIntensity(FFlt, 3, pfun);
            F2 = distribloads(el1femm, geom, u, fi, 2);

            # Note that the material object needs to be created with the proper
            # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
            MR = DeforModelRed3D

            material = MatDeforElastIso(MR, E, nu)

            femm = FEMMDeforLinearMST10(MR, IntegDomain(fes, TetRule(4)), material)

            # The geometry field now needs to be associated with the FEMM
            femm = associategeometry!(femm, geom)

            K = stiffness(femm, geom, u)
            K = cholesky(K)
            U = K\(F2)
            scattersysvec!(u, U[:])

            nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, Thickness], inflate=tolerance);
            thecorneru = zeros(FFlt,length(nl),3)
            gathervalues_asmat!(u, thecorneru, nl);
            thecorneru = mean(thecorneru, dims=1)[1]/phun("mm")
            println("displacement =$(thecorneru) vs -0.10215 [MM]")

            fld = fieldfromintegpoints(femm, geom, u, :Cauchy, 2;
            nodevalmethod = :averaging, reportat = extrapolation)
            sigyd = mean(fld.values[nl,1], dims=1)[1]
            println("Sigma_y =$(sigyd/phun("MPa")) vs $(sigma_yD/phun("MPa")) [MPa]")

            stressfield = fieldfromintegpoints(femm, geom, u, :Cauchy, collect(1:6);
            nodevalmethod = :averaging, reportat = extrapolation)

            # File =  "LE1NAFEMS_MST10_sigma.vtk"
            # vtkexportmesh(File, fes.conn, geom.values,
            #     FinEtools.MeshExportModule.T10; vectors=[("u", u.values)],
            #     scalars=[("sig", stressfield.values)])
            # @async run(`"paraview.exe" $File`)

            push!(convergencestudy, FDataDict(
            "elementsize"=> 1.0 / (2^ref),
            "fens"=>fens,
            "fes"=>fes,
            "geom"=>geom,
            "u"=>u,
            "femm"=>femm,
            "integrationrule"=>femm.integdomain.integration_rule,
            "stressfields"=>[stressfield],
            "tolerance"=>tolerance)
            )
        end # for ref

        # File = "LE1NAFEMS_MST10_stresses_nodal_convergence_$(elementtag)_$(extrapolation)"
        # open(File * ".jls", "w") do file
        #     serialize(file, convergencestudy)
        # end

    end # for extrapolation

end # LE1NAFEMS_MST10_stresses_nodal

function LE1NAFEMS_MST10_S_convergence()
    println("LE1NAFEMS, 3D version."        )
    t0 = time()

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Thick0 = 0.1*phun("m")/2.0 # to account for the symmetry reduction

    sigyderrs = Dict{Symbol, FFltVec}()

    nnodes = []
    for extrapolation in [:extraptrend :extrapmean]
        sigyderrs[extrapolation] = FFltVec[]
        nnodes = []
        for ref in 0:1:4
            Thickness = Thick0
            # Thickness = Thick0/2^ref
            tolerance = Thickness/2^ref/1000.; # Geometrical tolerance

            fens,fes = T4block(1.0, pi/2, Thickness, 2^ref*5, 2^ref*6, 1)

            bdryfes = meshboundary(fes);
            icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
            for i=1:count(fens)
                t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
                fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
            end
            fens,fes = T4toT10(fens,fes)

            geom = NodalField(fens.xyz)
            u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

            l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 2, 0.0)
            l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
            setebc!(u,l1,true, 1, 0.0)
            l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
            setebc!(u,l1,true, 3, 0.0)

            applyebc!(u)
            numberdofs!(u)

            el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), TriRule(3)))
            function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
                pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
                forceout .=    vec(p*pt/norm(pt));
                return forceout
            end
            fi = ForceIntensity(FFlt, 3, pfun);
            F2 = distribloads(el1femm, geom, u, fi, 2);

            # Note that the material object needs to be created with the proper
            # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
            MR = DeforModelRed3D

            material = MatDeforElastIso(MR, E, nu)

            femm = FEMMDeforLinearMST10(MR, IntegDomain(fes, TetRule(4)), material)

            # The geometry field now needs to be associated with the FEMM
            femm = associategeometry!(femm, geom)

            K = stiffness(femm, geom, u)
            K = cholesky(K)
            U = K\(F2)
            scattersysvec!(u, U[:])

            nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, Thickness], inflate=tolerance);
            thecorneru = zeros(FFlt,length(nl),3)
            gathervalues_asmat!(u, thecorneru, nl);
            thecorneru = mean(thecorneru, dims=1)[1]/phun("mm")
            println("displacement =$(thecorneru) vs -0.10215 [MM]")

            fld = fieldfromintegpoints(femm, geom, u, :Cauchy, 2; nodevalmethod = :averaging, reportat = extrapolation)
            sigyd = mean(fld.values[nl,1], dims=1)[1]
            println("Sigma_y =$(sigyd/phun("MPa")) vs $(sigma_yD/phun("MPa")) [MPa]")

            println("$extrapolation, $(count(fes)), $(sigyd/phun("MPa"))")
            push!(nnodes, count(fens))
            push!(sigyderrs[extrapolation], abs(sigyd/sigma_yD - 1.0))
            # File =  "a.vtk"
            # vtkexportmesh(File, fes.conn, geom.values,
            # FinEtools.MeshExportModule.H8; vectors=[("u", u.values)],
            # scalars=[("sigmay", fld.values)])
            # @async run(`"paraview.exe" $File`)
        end
    end

    File = "LE1NAFEMS_MST10_S_convergence.CSV"
    savecsv(File, nnodes=vec(nnodes), sigyderrtrend=vec(sigyderrs[:extraptrend]), sigyderrdefault=vec(sigyderrs[:extrapmean]))

end # LE1NAFEMS_MST10_S_convergence

function LE1NAFEMS_T10_stresses_nodal()
    elementtag = "T10"
    println("LE1NAFEMS, 3D version. Element: $(elementtag)")

    E = 210e3*phun("MEGA*PA");# 210e3 MPa
    nu = 0.3;
    p = 10*phun("MEGA*PA");# 10 MPA Outward pressure on the outside ellipse
    sigma_yD = 92.7*phun("MEGA*PA");# tensile stress at [2.0, 0.0] meters
    Thick0 = 0.1*phun("m")/2.0 # to account for the symmetry reduction

    convergencestudy = FDataDict[]
    for ref in  [0, 1, 2, 3, 4]
        Thickness = Thick0
        tolerance = Thickness/2^ref/1000.; # Geometrical tolerance

        fens,fes = T10block(1.0, pi/2, Thickness, 2^ref*2, 2^ref*3, 1; orientation = :b)

        bdryfes = meshboundary(fes);
        icl = selectelem(fens, bdryfes, box=[1.0, 1.0, 0.0, pi/2, 0.0, Thickness], inflate=tolerance);
        for i=1:count(fens)
            t=fens.xyz[i,1]; a=fens.xyz[i,2]; z=fens.xyz[i,3]
            fens.xyz[i,:]=[(t*3.25+(1-t)*2)*cos(a), (t*2.75+(1-t)*1)*sin(a), z];
        end

        geom = NodalField(fens.xyz)
        u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

        l1 =selectnode(fens; box=[0.0, Inf, 0.0, 0.0, 0.0, Thickness], inflate = tolerance)
        setebc!(u,l1,true, 2, 0.0)
        l1 =selectnode(fens; box=[0.0, 0.0, 0.0, Inf, 0.0, Thickness], inflate = tolerance)
        setebc!(u,l1,true, 1, 0.0)
        l1 =selectnode(fens; box=[0.0, Inf, 0.0, Inf, 0.0, 0.0], inflate = tolerance)
        setebc!(u,l1,true, 3, 0.0)

        applyebc!(u)
        numberdofs!(u)

        el1femm =  FEMMBase(IntegDomain(subset(bdryfes,icl), TriRule(3)))
        function pfun(forceout::FVec{T}, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt) where {T}
            pt= [2.75/3.25*XYZ[1], 3.25/2.75*XYZ[2], 0.0]
            forceout .=    vec(p*pt/norm(pt));
            return forceout
        end
        fi = ForceIntensity(FFlt, 3, pfun);
        F2 = distribloads(el1femm, geom, u, fi, 2);

        # Note that the material object needs to be created with the proper
        # model-dimension reduction in mind.  In this case that is the fully three-dimensional solid.
        MR = DeforModelRed3D

        material = MatDeforElastIso(MR, E, nu)

        femm = FEMMDeforLinear(MR, IntegDomain(fes, TetRule(4)), material)

        # The geometry field now needs to be associated with the FEMM
        femm = associategeometry!(femm, geom)

        K = stiffness(femm, geom, u)
        K = cholesky(K)
        U = K\(F2)
        scattersysvec!(u, U[:])

        nl = selectnode(fens, box=[2.0, 2.0, 0.0, 0.0, 0.0, Thickness], inflate=tolerance);
        thecorneru = zeros(FFlt,length(nl),3)
        gathervalues_asmat!(u, thecorneru, nl);
        thecorneru = mean(thecorneru, dims=1)[1]/phun("mm")
        println("displacement =$(thecorneru) vs -0.10215 [MM]")

        fld = fieldfromintegpoints(femm, geom, u, :Cauchy, 2)
        sigyd = mean(fld.values[nl,1], dims=1)[1]
        println("Sigma_y =$(sigyd/phun("MPa")) vs $(sigma_yD/phun("MPa")) [MPa]")

        stressfield = fieldfromintegpoints(femm, geom, u, :Cauchy, collect(1:6))

        # File =  "LE1NAFEMS_T10_sigma.vtk"
        # vtkexportmesh(File, fes.conn, geom.values,
        #     FinEtools.MeshExportModule.T10; vectors=[("u", u.values)],
        #     scalars=[("sig", stressfield.values)])
        # @async run(`"paraview.exe" $File`)

        push!(convergencestudy, FDataDict(
        "elementsize"=> 1.0 / (2^ref),
        "fens"=>fens,
        "fes"=>fes,
        "geom"=>geom,
        "u"=>u,
        "femm"=>femm,
        "integrationrule"=>femm.integdomain.integration_rule,
        "stressfields"=>[stressfield],
        "tolerance"=>tolerance)
        )
    end # for ref

    # File = "LE1NAFEMS_T10_stresses_nodal_convergence_$(elementtag)"
    # open(File * ".jls", "w") do file
    #     serialize(file, convergencestudy)
    # end

end # LE1NAFEMS_T10_stresses_nodal

function allrun()
    println("#####################################################")
    println("# LE1NAFEMS_MSH8 ")
    LE1NAFEMS_MSH8()
    println("#####################################################")
    println("# LE1NAFEMS_MSH8_convergence ")
    LE1NAFEMS_MSH8_convergence()
    println("#####################################################")
    println("# LE1NAFEMS_MSH8_export ")
    LE1NAFEMS_MSH8_export()
    println("#####################################################")
    println("# LE1NAFEMS_MST10_convergence ")
    LE1NAFEMS_MST10_convergence()
    println("#####################################################")
    println("# LE1NAFEMS_MST10_one ")
    LE1NAFEMS_MST10_one()
    # println("#####################################################")
    # println("# LE1NAFEMS_MST10_single ")
    # LE1NAFEMS_MST10_single()
    println("#####################################################")
    println("# LE1NAFEMS_MST10_stresses_nodal ")
    LE1NAFEMS_MST10_stresses_nodal()
    println("#####################################################")
    println("# LE1NAFEMS_MST10_S_convergence ")
    LE1NAFEMS_MST10_S_convergence()
    println("#####################################################")
    println("# LE1NAFEMS_T10_stresses_nodal ")
    LE1NAFEMS_T10_stresses_nodal()
    return true
end # function allrun

end # module LE1NAFEMS_examples


mutable struct SymataAutoload
    triggers::Dict
    isloaded::Bool
end

const autoload = SymataAutoload(Dict{Symbol,Bool}(),false)

addtriggers(args...) = foreach( t -> (autoload.triggers[t] = true; protect(t)), args)

check_autoload(x) = nothing

"""
    check_autoload(s::Symbol)

if `s` is in a list of triggers check if the Symata language code has
been loaded. If it has not been, load it. If it has been, do nothing.

`check_autoload` is called in: in `AST_translation.jl`, `AST_translation_tables.jl`,
and `doc.jl`. It is called in most instances of reading a symbol when reading Symata
code.
"""
function check_autoload(s::Symbol)
    global autoload
    if (! autoload.isloaded) && haskey(autoload.triggers, s)
        autoload.isloaded = true
        load_symata_code_now()
    end
end

function load_symata_code_now()
    fqpath = joinpath(dirname(@__FILE__), "../symsrc/autoloaded.sj")
    read_Symata_file(fqpath)
end

####

addtriggers(:Accumulate, :Array, :ArrayDepth, :Divide, :ExpToTrig, :ExpandCos,
            :ExpandSin, :ExpandSinCos, :FixedPoint, :FlattenAt, :ListConvolve,
            :ListCorrelate, :MapAt, :NestWhile, :NestWhileList, :NextPrime,
            :Normal, :Operate, :Subdivide, :TakeDrop, :TensorRank, :Through)


@doc """
Module Utils:
  This module holds miscellaneous functions used throughout the code
"""->
module Utils

push!(LOAD_PATH, joinpath(Pkg.dir("PDESolver"), "src/Utils"))
using ODLCommonTools
import ODLCommonTools.sview
using ArrayViews
import ArrayViews.view
using MPI
using SummationByParts
using PdePumiInterface     # common mesh interface - pumi
using Input

include("output.jl")
include("parallel.jl")
include("io.jl")
include("logging.jl")
include("projections.jl")
include("complexify.jl")
include("mass_matrix.jl")
include("curvilinear.jl")
include("area.jl")
include("checkpoint.jl")
include("interpolation.jl")
include("identity_array.jl")
include("abstract_binning.jl")
include("uniform_binning.jl")

export free
export array1DTo3D, writeQ, array3DTo1D, removeComplex, assembleArray
export calcNorm, calcL2InnerProduct, calcMeshH, calcEuclidianNorm
#export initMPIStructures, exchangeFaceData, verifyCommunication, getSendData
#export startDataExchange
#export exchangeElementData
export  Timings, write_timings
export sharedFaceLogging
export calcBCNormal, calcBCNormal_revm, max_deriv_rev
export applyPermRow, applyPermRowInplace, applyPermColumn
export applyPermColumnInplace, inversePerm, permMatrix, permMatrix!
export arrToVecAssign
export fastzero!, fastscale!, removeComplex
export @verbose1, @verbose2, @verbose3, @verbose4, @verbose5, @unpack, @pack,
       @printit, assertArraysUnique, assertFieldsConcrete
# projections.jl functions
export getProjectionMatrix, projectToXY, projectToNT, calcLength, normalize_vec,
       calcLength_rev

# complexify.jl functions
export absvalue, absvalue_deriv, absvalue_rev, absvalue2, absvalue2_deriv,
       absvalue3, absvalue3_deriv, sign_c, atan2_rev

# output.jl
export printSolution, printCoordinates, printMatrix
export print_qvec_coords

# mass_matrix.jl
export calcMassMatrixInverse, calcMassMatrix, calcMassMatrixInverse3D,
       applyMassMatrixInverse

# curvilinear.jl
export calcSCurvilinear, calcSCurvilinear_rev, calcECurvilinear, calcDCurvilinear

# area.jl
export calcVolumeContribution!, calcVolumeContribution_rev!, calcProjectedAreaContribution!, calcProjectedAreaContribution_rev!, crossProd, crossProd_rev

# io.jl
export BufferedIO, BSTDOUT, BSTDERR, printArray

# parallel_types.jl
export SharedFaceData, getSharedFaceData, setParallelData, getParallelData,
       setNewTag

# parallel.jl
export startSolutionExchange, startSolutionExchange_rev,
       startSolutionExchange_rev2, exchangeData,
       exchangeData_rev,
       finishExchangeData, finishExchangeData_rev, finishExchangeData_rev2,
       finishSolutionBarExchange,
       @mpi_master, @time_all, print_time_all, verifyReceiveCommunication,
       MPITagManager, getNextTag, markTagUsed, freeTag, assertReceivesWaited,
       assertSendsWaited, TagManager

# checkpoint.jl
export Checkpointer, AbstractCheckpointData, readCheckpointData,
       readLastCheckpointData, saveNextFreeCheckpoint, loadLastCheckpoint,
       countFreeCheckpoints,
       getLastCheckpoint, getOldestCheckpoint, freeOldestCheckpoint,
       freeCheckpoint, getNextFreeCheckpoint

# interpolation.jl
export interpField

# identity_array.jl
export IdentityArray, FullFace

# abstract_binning.jl and unifrom_binning.jl
export AbstractBinning, calcBin, sumBins, UniformBinning

"""
  Generic function to free any memory belonging to other libraries
"""
function free(x)

  error("generic free() reached")

  return nothing
end

@doc """
### Utils.array1DTo3D

  This takes a solution vector, and disassembles it into the 3D array form.
  This function uses mesh.dofs to speed the process.

  This function also calls writeQ to do any requested output.

  **Inputs**

   * mesh
   * sbp
   * eqn
   * opts
   * q_vec: vector, same shape as `eqn.q_vec

  **Inputs/Outputs**

   * q_arr: array, same shape as `eqn.q`, to be overwritten

  This is a mid level function, and does the right thing regardless of equation
  dimension.

  This function is optimized for DG meshes where `q_vec` and `q` might
  refer to the same array.

  Aliasing restrictions: none
"""->
# mid level function (although it doesn't need Tdim)
function array1DTo3D(mesh::AbstractCGMesh, sbp,
                          eqn::AbstractSolutionData, opts,
                          q_vec::AbstractVector,
                          q_arr::AbstractArray{T, 3}) where T
  # disassemble q_vec into eqn.
  for i=1:mesh.numEl  # loop over elements
    for j = 1:mesh.numNodesPerElement
      for k=1:size(q_arr, 1)
        dofnum_k = mesh.dofs[k, j, i]
        q_arr[k, j, i] = q_vec[dofnum_k]
      end
    end
  end

  writeQ(mesh, sbp, eqn, opts)

  return nothing
end

function array1DTo3D(mesh::AbstractDGMesh, sbp,
                          eqn::AbstractSolutionData, opts,
                          q_vec::AbstractVector,
                          q_arr::AbstractArray{T, 3}) where T

  # we assume the memory layouts of q_arr and q_vec are the same
  if pointer(q_arr) != pointer(q_vec)
    for i = 1:length(q_vec)
      q_arr[i] = q_vec[i]
    end
  end

  # no need to do any disassembly for DG
  writeQ(mesh, sbp, eqn, opts)

end

@doc """
### Utils.writeQ

  This function writes the real part of the solution variables eqn.q to a space
  delimited file called q.dat, controlled by the input options 'writeq', of type bool

  This is a high level function.
"""->
function writeQ(mesh, sbp, eqn, opts)

  if !opts["writeq"]
    return nothing
  end

  fname = "q.dat"
  rmfile(fname)
  writedlm(fname, eqn.q)

  return nothing
end

@doc """
### Utils.array3DTo1D

  This function takes the 3D array of variables and
  reassembles it into the vector form by summing into the output array.
  Note that
  this is a reduction operation and zeros res_vec before performing the
  operation, unless zero_resvec is set to false

  This is a mid level function, and does the right thing regardless of
  equation dimension

  **Inputs**

   * mesh
   * sbp
   * eqn
   * opts
   * res_array: 3D array, same shape as `eqn.res`

  **Inputs/Outputs**

   * res_vec: vector, same shape as `eqn.res_vec`, to sum into

  **Keyword Arguments**

   * zero_resvec: if true (default), zeros `res_vec` before summing into it

"""->
# mid level function (although it doesn't need Tdim)
function array3DTo1D(mesh::AbstractCGMesh{Tmsh},
       sbp, eqn::AbstractSolutionData{Tsol}, opts,
       res_arr::Abstract3DArray, res_vec::AbstractVector;
       zero_resvec=true) where {Tmsh, Tsol}
# arr is the array to be assembled into res_vec

#  println("in array3DTo1D")

  if zero_resvec
    @assert pointer(res_vec) != pointer(res_arr)
    fill!(res_vec, 0)
  end

  # sum into output
  for i=1:mesh.numEl  # loop over elements
    for j=1:mesh.numNodesPerElement
      @simd for k=1:size(res_arr, 1)  # loop over dofs on the node
        dofnum_k = mesh.dofs[k, j, i]
        res_vec[dofnum_k] += res_arr[k,j,i]
      end
    end
  end

  return nothing
end

function array3DTo1D(mesh::AbstractDGMesh{Tmsh},
       sbp, eqn::AbstractSolutionData{Tsol}, opts,
       res_arr::Abstract3DArray, res_vec::AbstractVector;
       zero_resvec=true) where {Tmsh, Tsol}

  # we assume the memory layouts of q_arr and q_vec are the same
  if pointer(res_arr) != pointer(res_vec)
    if zero_resvec
      @simd for i = 1:length(res_vec)
        res_vec[i] = res_arr[i]
      end
    else
      @simd for i = 1:length(res_vec)
        res_vec[i] += res_arr[i]
      end
    end

  # else: nothing to do
  end

  return nothing

end

function arrToVecAssign(mesh::AbstractMesh{Tmsh},
       sbp, eqn::AbstractSolutionData{Tsol}, opts,
       arr::Abstract3DArray, dest_vec::AbstractArray{Tres,1},
       zero_resvec=true) where {Tmsh, Tsol, Tres}

  # This was created so a q -> q_vec operation could be performed, but it is sufficiently
  #   generic to operate on other things.
  # It is the inverse function of array1DTo3D

  # arr is the array to be assembled into dest_vec, using an assignment reduction

  # removed zeroing out of dest vec, as all of it will be overwritten below

  if pointer(arr) == pointer(dest_vec)
    return nothing
  end

  for i=1:mesh.numEl  # loop over elements
    for j=1:mesh.numNodesPerElement
      for k=1:mesh.numDofPerNode  # loop over dofs on the node

        # mesh.dofs indexing:
        #   [dof ix on the node, node ix on the el, el ix]
        dofnum_k = mesh.dofs[k, j, i]

        dest_vec[dofnum_k] = arr[k,j,i]

      end
    end
  end

  return nothing

end

"""
  This function removes the complex part of the following fields of `eqn`:
  `q`, `q_vec`, `res`, `res_vec`

  **Inputs**

   * eqn: an `AbstractSolutionData`
"""
function removeComplex(eqn::AbstractSolutionData)

  @simd for i=1:length(eqn.q_vec)
    eqn.q_vec[i] = real(eqn.q_vec[i])
    eqn.res_vec[i] = real(eqn.res_vec[i])
  end

  if pointer(eqn.q_vec) != pointer(eqn.q)
    @simd for i=1:length(eqn.q)
      eqn.q[i] = real(eqn.q[i])
    end
  end

  if pointer(eqn.res_vec) != pointer(eqn.res)
    @simd for i=1:length(eqn.res)
      eqn.res[i] = real(eqn.res[i])
    end
  end

  # send and receive buffers
  for i=1:length(eqn.shared_data)
    arr = eqn.shared_data[i].q_send
    @simd for j=1:length(arr)
      arr[j] = real(arr[j])
    end

    arr = eqn.shared_data[i].q_recv
    @simd for j=1:length(arr)
      arr[j] = real(arr[j])
    end
  end

  return nothing
end

# mid level function (although it doesn't need Tdim)
function assembleArray(mesh::AbstractMesh{Tmsh},
       sbp, eqn::AbstractSolutionData{Tsol}, opts,
       arr::Abstract3DArray, res_vec::AbstractArray{Tres,1},
       zero_resvec=true) where {Tmsh, Tsol, Tres}
# arr is the array to be assembled into res_vec, using an assignment reduction
# the length of res_vec is mesh.numDof/mesh.numDofPerNode, only the last
# dof on the node is placed into res_vec

#  println("in array3DTo1D")

  if zero_resvec
    fill!(res_vec, 0.0)
  end

  if mesh.numDofPerNode == 1
    offset = 0
  else
    offset = 1
  end

  for i=1:mesh.numEl  # loop over elements
    for j=1:mesh.numNodesPerElement
      for k=1:size(arr, 1)  # loop over dofs on the node

        dofnum_k = mesh.dofs[k, j, i]
        dofnum_k1 = div(dofnum_k, mesh.numDofPerNode) + offset # get node number

        res_vec[dofnum_k1] = arr[k,j,i]
      end
    end
  end

  return nothing
end

@doc """
### Utils.calcNorm

  This function calculates the norm of a vector (of length numDof) using the
    SBP norm.

    Inputs:
      eqn:  an AbstractSolutionData
      res_vec:  vector to calculate the norm of

    Keyword arguments:
      strongres: if res_vec is the residual of the weak form, then
                 strongres=true computes (efficiently) the norm of the strong
                 form residual.  Default false
      globalnrm: compute the norm over all processes or not. Default true

    Returns:
      val:  norm of solution using SBP norm (Float64)

    There are no restrctions on the datatype of res_vec (ie. it can be complex)

    Aliasing restrictions: none

"""->
function calcNorm(eqn::AbstractSolutionData, res_vec::AbstractArray{T}; strongres=false, globalnrm=true) where T
# calculates the norm of a vector using the mass matrix

  val = zero(real(res_vec[1]))

  #TODO: make this better for complex number: conj(z)Hz
  if !strongres
    for i=1:length(res_vec)
      val += real(res_vec[i])*eqn.M[i]*real(res_vec[i])   # res^T M res
    end
  else  # strongres
    for i=1:length(res_vec)
      val += real(res_vec[i])*eqn.Minv[i]*real(res_vec[i])   # res^T M res
    end
  end

  eqn.params.time.t_allreduce = @elapsed if globalnrm
    val = MPI.Allreduce(val, MPI.SUM, eqn.comm)
  end

  val = sqrt(val)
  return val
end     # end of calcNorm function

"""
  This function calculate the SBP approximation to the L2 inner product of
  two vectors of length mesh.numDof.

  L2_product = u^T H conj(v)

  Note that this function does not take a square root like [`calcNorm`](@ref)
  does.

  **Inputs**

   * eqn: AbstractSolutionData
   * u: first vector, of length mesh.numDof
   * v: second vector, of length mesh.numDof.  If complex, this vector gets
        conjugated

  **Outputs**

   * val: the value of the inner product

  **Keyword Arguments**

   * globalnrm: if true, computes the norm of the entire (parallel) vector,
                if false, computes the norm of only the local part
"""
function calcL2InnerProduct(eqn::AbstractSolutionData, u::AbstractArray{T}, v::AbstractArray{T2}; globalnrm=true) where {T, T2}

  # TODO: generalize this to 1, mesh.numDofPerNode vectors
  @assert length(u) == length(eqn.M)
  @assert length(v) == length(eqn.M)

  val = zero(real(promote_type(T, T2)))

  for i=1:length(u)
    val += real( u[i] * eqn.M[i] * conj(v[i]) )
  end

  eqn.params.time.t_allreduce = @elapsed if globalnrm
    val = MPI.Allreduce(val, MPI.SUM, eqn.comm)
  end

  return val
end

"""
  This function computes the Euclidian norm of a vector where each MPI
  process owns part of the vector

  Inputs:
    comm: an MPI communicator
    vec: the local part of the vector

  Outputs:
    val: the Euclidian norm of the entire vector

  Note that unlike calcNorm, the time spent in the Allreduce is not logged
  for this function.

"""
function calcEuclidianNorm(comm::MPI.Comm, vec::AbstractVector{T}) where T

  Tnorm = real(T)
  val = zero(Tnorm)
  for i=1:length(vec)
    val += real(conj(vec[i])*vec[i])
  end

  val = MPI.Allreduce(val, MPI.SUM, comm)

  return sqrt(val)
end

@doc """
### Utils.calcMeshH

  This function calculates the average distance between nodes over the entire
  mesh.  This function allocates a bunch of temporary memory, so don't call
  it too often.

  Inputs:
    mesh
    eqn
    opts
"""->
function calcMeshH(mesh::AbstractMesh{Tmsh}, sbp,  eqn, opts) where Tmsh
  jac_3d = reshape(mesh.jac, 1, mesh.numNodesPerElement, mesh.numEl)
  jac_vec = zeros(Tmsh, mesh.numNodes)
  assembleArray(mesh, sbp, eqn, opts, jac_3d, jac_vec)
  dim = mesh.dim
  # scale by the minimum distance between nodes on a reference element
  # this is a bit of an assumption, because for distorted elements this
  # might not be entirely accurate
  h_avg = sum(1./(jac_vec.^(1/dim)))
  h_avg = MPI.Allreduce(h_avg, MPI.SUM, mesh.comm)
  # caution: overflow
  numnodes = MPI.Allreduce(length(jac_vec), MPI.SUM, mesh.comm)
  h_avg *= mesh.min_node_dist/numnodes
  return h_avg
end

#=
import Base.flush
function flush(f::IOBuffer)

end
=#

@doc """
### Utils.Timings

  This type accumulates the time spent in each part of the code.
"""->
mutable struct Timings
  # timings
  t_volume::Float64  # time for volume integrals
  t_face::Float64 # time for surface integrals (interior)
  t_source::Float64  # time spent doing source term
  t_shock::Float64   # time spent doing shock capturing
  t_sharedface::Float64  # time for shared face integrals
  t_bndry::Float64  # time spent doing boundary integrals
  t_dataprep::Float64  # time spent preparing data
  t_stab::Float64  # time spent adding stabilization
  t_send::Float64  # time spent sending data

  t_volume_diff::Float64  # time for volume integrals
  t_face_diff::Float64 # time for surface integrals (interior)
  t_shock_diff::Float64  # time for shock capturing
  t_source_diff::Float64  # time spent doing source term
  t_sharedface_diff::Float64  # time for shared face integrals
  t_bndry_diff::Float64  # time spent doing boundary integrals
  t_dataprep_diff::Float64  # time spent preparing data
  t_stab_diff::Float64  # time spent adding stabilization

  t_wait::Float64  # time spent in MPI_Wait
  t_allreduce::Float64 # time spent in allreduce
  t_pert::Float64  # time spent applying peraturbation
  t_alloc::Float64  # time spent allocating the Jacobian
  t_insert::Float64  # time spent inserting values into matrix
  t_func::Float64  # time spent evaluating the residual
  t_color::Float64  # time spent evaluating the colors
  t_jacobian::Float64  # time spent computing jacobian
  t_solve::Float64  # linear solve time
  t_newton::Float64  # time spent in newton loop
  t_timemarch::Float64 # time spent in time marching loop
  t_callback::Float64  # time spent performing callbacks
  t_nlsolve::Float64  # time spent in the nonlinear solver
  t_meshinit::Float64  # time spent initializing the mesh object
  t_barrier::Float64  # time spent in MPI_Barrier
  t_barrier2::Float64
  t_barrier3::Float64
  t_barriers::Array{Float64, 1}

  function Timings()
    nbarriers = 7
    barriers = zeros(Float64, nbarriers)
    return new(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, barriers)
  end
end

@doc """
### Utils.write_timings

  Write the values in a Timings object to a file.  Also writes the names of
  fields to a separate file.

  Inputs:
    t: a  Timings object
    fname: the file name, without extension

  The values are written to the file fname.dat, and the names are written to
  fname_names.dat
"""->
function write_timings(t::Timings, fname::AbstractString)
  timing_names = fieldnames(t)
  nbarriers = length(t.t_barriers)
  nvals = length(timing_names) + nbarriers - 1
  vals = Array{Float64}(nvals)
  val_names = Array{String}(nvals)

  # put all values except those from t_barriers into array
  pos = 1
  for i=1:length(timing_names)
    tname_i = timing_names[i]
    tname_i_str = string(tname_i)
    if tname_i_str != "t_barriers"
      vals[pos] = getfield(t, tname_i)
      val_names[pos] = tname_i_str
      pos += 1
    end
  end

  for i=1:length(t.t_barriers)
    vals[pos] = t.t_barriers[i]
    val_names[pos] = string("t_barriers_", i)
    pos += 1
  end

  fname_ext = string(fname, ".dat")
  writedlm(fname_ext, vals)

  fname2_ext = string(fname, "_names.dat")
  writedlm(fname2_ext, val_names)
end

"""
  Calculate the scaled normal vector in parametric coordinates from the
  face normal and scaled mapping jacobian.  `nrm2` is overwritten with
  the result.
"""
function calcBCNormal(params::AbstractParamType{2}, dxidx::AbstractMatrix,
                    nrm::AbstractVector, nrm2::AbstractVector)

  nrm2[1] = dxidx[1,1]*nrm[1] + dxidx[2,1]*nrm[2]
  nrm2[2] = dxidx[1,2]*nrm[1] + dxidx[2,2]*nrm[2]
  return nothing
end

function calcBCNormal(params::AbstractParamType{3}, dxidx::AbstractMatrix,
                    nrm::AbstractVector, nrm2::AbstractVector)

  n1 = nrm[1]; n2 = nrm[2]; n3 = nrm[3]
  nrm2[1] = dxidx[1,1]*n1 + dxidx[2,1]*n2 + dxidx[3,1]*n3
  nrm2[2] = dxidx[1,2]*n1 + dxidx[2,2]*n2 + dxidx[3,2]*n3
  nrm2[3] = dxidx[1,3]*n1 + dxidx[2,3]*n2 + dxidx[3,3]*n3

  return nothing
end

function calcBCNormal_revm(params::AbstractParamType{2}, dxidx::AbstractMatrix,
                           nrm::AbstractVector, nrm2_bar::AbstractVector,
                           dxidx_bar::AbstractMatrix)

  dxidx_bar[1,1] += nrm2_bar[1]*nrm[1]
  dxidx_bar[2,1] += nrm2_bar[1]*nrm[2]
  dxidx_bar[1,2] += nrm2_bar[2]*nrm[1]
  dxidx_bar[2,2] += nrm2_bar[2]*nrm[2]

  return nothing
end

function calcBCNormal_revm(params::AbstractParamType{3}, dxidx::AbstractMatrix,
                           nrm::AbstractVector, nrm2_bar::AbstractVector,
                           dxidx_bar::AbstractMatrix)

  dxidx_bar[1,1] += nrm2_bar[1]*nrm[1]
  dxidx_bar[2,1] += nrm2_bar[1]*nrm[2]
  dxidx_bar[3,1] += nrm2_bar[1]*nrm[3]
  dxidx_bar[1,2] += nrm2_bar[2]*nrm[1]
  dxidx_bar[2,2] += nrm2_bar[2]*nrm[2]
  dxidx_bar[3,2] += nrm2_bar[2]*nrm[3]
  dxidx_bar[1,3] += nrm2_bar[3]*nrm[1]
  dxidx_bar[2,3] += nrm2_bar[3]*nrm[2]
  dxidx_bar[3,3] += nrm2_bar[3]*nrm[3]

  return nothing
end

#------------------------------------------------------------------------------
# permutation functions
#------------------------------------------------------------------------------
"""
  Permute the rows of A according to the permvec, storing the result in B
  The permvec contains the source indices for each entry in B, ie.
  B[i] comes from A[permvec[i]].  This is consistent with the mathematical
  definition of a permutation that pre-multiplication by a permutation
  matrix (obtained from permMatrix) is a row permutation, ie.
  B = P*A

  Aliasing: no aliasing allowed
"""
function applyPermRow(permvec::AbstractVector, A::AbstractMatrix,
                            B::AbstractMatrix)

  m, n = size(A)
  for i=1:m
    idx = permvec[i]
    for j=1:n
      B[ i, j] = A[idx, j]
    end
  end

  return nothing
end

"""
  Like applyPermRow, but the result is returned in A.  Both A and B
  are overwritten.

  Aliasing: no aliasing allowed
"""
function applyPermRowInplace(permvec::AbstractVector, A::AbstractMatrix,
                            B::AbstractMatrix)

  applyPermRow(permvec, A, B)

  # copy back
  for i=1:length(A)
    A[i] = B[i]
  end

  return nothing
end

"""
  Permute the columns of A according to permvec.  See applyPermRow for the
  definition of a permvec.  Note that for column permutation the
  interpretation is that destination column permvec[i] comes from
  source column i.  This is consistent with the notion that post-multiplying
  by a permutation matrix (obtained from permMatrix) is a column
  permutation, i.e B = A*P

  Aliasing: no aliasing allowed
"""
function applyPermColumn(permvec::AbstractVector, A::AbstractMatrix,
                            B::AbstractMatrix)

  m, n = size(A)
  for i=1:m
    for j=1:n
      B[i, permvec[j]] = A[i, j]
    end
  end

  return nothing
end

"""
  Like applyPermColumn, but the result is returned in A.  Both A and B
  are overwritten

  Aliasing: no aliasing allowed
"""
function applyPermColumnInplace(permvec::AbstractVector,
                            A::AbstractMatrix, B::AbstractMatrix)

  applyPermColumn(permvec, A, B)

  for i=1:length(A)
    A[i] = B[i]
  end

  return nothing
end

"""
  Create a permutation matrix from a permutation vector.  Only select
  entries of A are overwritten.

"""
function permMatrix!(permvec::AbstractVector, A::AbstractMatrix)

  m, n = size(A)

  for i=1:m
    A[i, permvec[i]] = 1
  end

  return nothing
end

"""
  Create a permutation matrix from a permutation vector.  The element type
  of the returned matrix is Int.
"""
function permMatrix(permvec::AbstractVector)
  m = length(permvec)
  A = zeros(Int, m, m)
  permMatrix!(permvec, A)

  return A
end

"""
  Compute the permutation vector that corresponds to the inverse permutation

  Inputs:
    permvec: the original permutation vector

  Inputs/Outputs:
    invperm: the inverse permutation vector

  Aliasing: no aliasing allowed
"""
function inversePerm(permvec::AbstractVector, invperm::AbstractVector)

  n = length(permvec)
  for i=1:n
    idx = permvec[i]
    invperm[idx] = i
  end

  return nothing
end

# TODO: write functions to apply inverse permutation from permvec, without
#       needing to explicetly compute the inverse permutation vector

"""
  This function zeros out an array, and should be faster than fill! (branch
  free)

  Inputs/Outputs:
    x: an array
"""
@inline function fastzero!(x::AbstractArray)

  @inbounds @simd for i=1:length(x)
    x[i] = 0
  end

  return nothing
end

"""
  This function scales an array by a constant, and should be faster than scale!
  because it is branch free
"""
@inline function fastscale!(x::AbstractArray, c::Number)

  @inbounds @simd for i=1:length(x)
    x[i] *= c
  end

  return nothing
end

@doc """
### Utils.verbose1

  This macro introduces an if statement that causes the expression to be
  executed only if the variable verbose is greater than or equal to 1.
  verbose must exist in the scope of the caller

"""->
macro verbose1(ex)
  return quote
#    println("myrank = ", esc(myrank))
    if $(esc(:(verbose >= 1)))
      $(esc(ex))
    end
  end
end

@doc """
### Utils.verbose2

  This macro introduces an if statement that causes the expression to be
  executed only if the variable verbose is greater than or equal to 2.
  verbose must exist in the scope of the caller

"""->
macro verbose2(ex)
  return quote
#    println("myrank = ", esc(myrank))
    if $(esc(:(verbose >= 2)))
      $(esc(ex))
    end
  end
end

@doc """
### Utils.verbose3

  This macro introduces an if statement that causes the expression to be
  executed only if the variable verbose is greater than or equal to 3.
  verbose must exist in the scope of the caller

"""->
macro verbose3(ex)
  return quote
#    println("myrank = ", esc(myrank))
    if $(esc(:(verbose >= 3)))
      $(esc(ex))
    end
  end
end

@doc """
### Utils.verbose4

  This macro introduces an if statement that causes the expression to be
  executed only if the variable verbose is greater than or equal to 4.
  verbose must exist in the scope of the caller

"""->
macro verbose4(ex)
  return quote
#    println("myrank = ", esc(myrank))
    if $(esc(:(verbose >= 4)))
      $(esc(ex))
    end
  end
end

@doc """
### Utils.verbose5

  This macro introduces an if statement that causes the expression to be
  executed only if the variable verbose is greater than or equal to 5.
  verbose must exist in the scope of the caller

"""->
macro verbose5(ex)
  return quote
#    println("myrank = ", esc(myrank))
    if $(esc(:(verbose >= 5)))
      $(esc(ex))
    end
  end
end

"""
  This macro enables easy unpacking of the members of a type to local variables
  of the same name.  For example:

  ```
    @unpack foo a b c
  ```

  expands into

  ```
    a = foo.a; b = foo.b; c = foo.c
  ```

  **Inputs**

   * obj: the name of a variable (must be a type with fields)
   * varnames...: 1 or more names to unpack.  An error is thrown if a name is
                  duplicated, presuming that is not what the user intended
"""
macro unpack(obj, varnames...)

  @assert length(varnames) > 0

  # check varnames are unique
  for i=1:length(varnames)
    for j=1:(i-1)
      if varnames[i] == varnames[j]
        error("attempting to unpack duplicate variable name: $(varnames[i])")
      end
    end
  end

  # build the expression
  ex = :($(varnames[1]) = $(obj).$(varnames[1]))
  for i=2:length(varnames)
    ex = :($ex; $(varnames[i])= $(obj).$(varnames[i]))
  end

  return esc(ex)
end

macro pack(obj, varnames...)

  @assert length(varnames) > 0

  # check varnames are unique
  for i=1:length(varnames)
    for j=1:(i-1)
      if varnames[i] == varnames[j]
        error("attempting to unpack duplicate variable name: $(varnames[i])")
      end
    end
  end

  # build the expression
  ex = :($(obj).$(varnames[1]) = $(varnames[1]))
  for i=2:length(varnames)
    ex = :($ex; $(obj).$(varnames[i]) = $(varnames[i]))
  end

  return esc(ex)
end

"""
  Given a list of variables, prints them out, one per line

  Ex.  The code

  ```
    a = 2
    b = 3
    @printit a b
  ```
    will print
  ```
    a = 2
    b = 3
  ```

  Variables can also be more general expressions:

  ```
    a = [1 2 3]
    @printit a[1:2]
  ```

  will print

  ```
    a[1:2] = [1, 2]
  ```
"""
macro printit(v...)

  ex = :()  # empty expression
  for s in v
    str = string(s)
#    println("s = ", s)
    ex = :($ex, println($str, " = ", $(esc(s))); )
  end

  return ex
end

"""
  This function asserts that every array field of the given object is a
  unique array.  Note that this is less strict than a non-aliasing check,
  becuase views of an array are not the same as the array.
"""
function assertArraysUnique(obj::T) where {T}
# verify a struct does not have any fields that alias each other

  fnames = fieldnames(T)

  for i=1:length(fnames)
    f_i = getfield(obj, fnames[i])
    if typeof(f_i) <: AbstractArray
      for j=(i+1):length(fnames)
        f_j = getfield(obj, fnames[j])
        if typeof(f_j) <: AbstractArray
          @assert !(f_i === f_j)
        end
      end
    end
  end

  return nothing
end

"""
  Asserts that all fields of the given object have declared field types that
  are concrete.  Note that this only considers the types of the field as
  written in the type declaration, not the runtime type of the fields.

  **Inputs**

   * obj: an object
"""
function assertFieldsConcrete(obj::T) where {T}

  for i=1:nfields(T)
    type_i = fieldtype(T, i)
    @assert isleaftype(type_i)
  end

  return nothing
end

end  # end module


using FunctionalCollections

struct JuliaBackend <: ArrayBackend
end

const ObjectID = UInt64

function reify_value(val::Complex)
    if real(val) == 0
        :( $(imag(val)) * im ), Nullable{Symbol}()
    elseif imag(val) == 0
        real(val), Nullable{Symbol}()
    else
        :( $(real(val)) + $(imag(val)) * im ), Nullable{Symbol}()
    end
end

function reify_value(val::AbstractArray)
    if ndims(val) == 0
        return val[1], Nullable{Symbol}()
    else
        name = gensym()
        ast_node = Symbol(name)
        return ast_node, Nullable{Symbol}(ast_node)
    end
end

function reify_value(val::Number)
    val, Nullable{Symbol}()
end

function reify_value(val)
    name = gensym()
    ast_node = Symbol(name)
    return ast_node, Nullable{Symbol}(ast_node)
end

function reify_leaf(identifiers, val)

    val_id = object_id(val)
    if haskey(identifiers, val_id)
        ast_node, _ = identifiers[val_id]
    else
        ast_node, name = reify_value(val)
        if !isnull(name)
            identifiers = assoc(identifiers, val_id, get(name) => val)
        end
    end

    identifiers, ast_node
end

function reify(identifiers, aexpr::LazyArray)
    reify_leaf(identifiers, aexpr.value)
end

function reify(identifiers, aexpr::BackendArray)
    reify_leaf(identifiers, from_backend(aexpr))
end

function reify(identifiers, aexpr::Unknown)
    reify_leaf(identifiers, aexpr)
end

function unpack(x, i)
    x[i]
end

function repeat_outer(x, outer)
    repeat(x, outer=outer)
end

# Julia's reshape only accepts tuples
function reshape(x::AbstractArray, dims::AbstractArray)
    reshape(x, dims...)
end

# Zero-dimensional arrays are reified as scalars, and Julia doesn't have a reshape() method
# for scalars, so we have to provide our own.
function reshape(x::Number, dims...)
    reshape([x], dims...)
end

function random_uniform(state, dims...)
    state_copy = copy(state)
    res = rand(state_copy, Float64, dims...)
    res, state_copy
end

function random_normal(state, dims...)
    state_copy = copy(state)
    res = randn(state_copy, dims...)
    res, state_copy
end

function reify(identifiers, aexpr::CallNode)
    head = aexpr.func
    args = aexpr.args
    identifiers, arg_asts = reify(identifiers, args)
    identifiers, :( $head($(arg_asts...)) )
end

function reify{T}(identifiers, arr::Array{T, 1})
    result = Array{Any, 1}(length(arr))
    for (i, elem) in enumerate(arr)
        identifiers, elem_ast = reify(identifiers, elem)
        result[i] = elem_ast
    end
    identifiers, result
end

function reify{K, V}(identifiers, pair::Pair{K, V})
    fst, snd = pair
    identifiers, fst = reify(identifiers, fst)
    identifiers, snd = reify(identifiers, snd)
    identifiers, (fst => snd)
end

function reify{K, V}(identifiers, dict::Dict{K, V})
    result = Dict{Any, Any}()
    for (i, pair) in enumerate(dict)
        identifiers, pair_ast = reify(identifiers, pair)
        k, v = pair_ast
        result[k] = v
    end
    identifiers, result
end

struct JuliaArrayFunction <: BackendArrayFunction
    func :: Function
    known_arguments :: Array{AbstractArray, 1}
    mutated_parameters_indices :: Set{Int64}
end

function (cafunc::JuliaArrayFunction)(args...)

    # The parameters being mutated must be backend arrays.
    # Technically, this check should be done in a compiled array function in any backend
    if length(cafunc.mutated_parameters_indices) > 0
        for (i, arg) in enumerate(args)
            if in(i, cafunc.mutated_parameters_indices)
                @assert isa(arg, BackendValue) && array_backend(arg) == JuliaBackend()
            end
        end
    end

    unwrapped_args = map(unwrap, args)

    # Cannot call `cafunc.func` directly because of Julia's restrictions
    # on dynamic method definition.
    # See "applicable method may be too new" error, and
    # https://docs.julialang.org/en/latest/manual/methods.html#Redefining-Methods-1
    results = Base.invokelatest(cafunc.func, cafunc.known_arguments..., unwrapped_args...)

    wrapped_results = map(wrap, results)
    length(wrapped_results) == 1 ? wrapped_results[1] : wrapped_results
end

function compile_array_function(::JuliaBackend, afunc::ArrayFunction)

    serialized_aliases = serialize(afunc)

    # Replace known/unknown arrays in the expressions to be returned with temporary identifiers
    body_identifiers = PersistentHashMap{ObjectID, Any}()
    body_identifiers, reified_aliases = reify(body_identifiers, serialized_aliases)
    identifiers, reified_parameters = reify(body_identifiers, afunc.parameters)
    identifiers, reified_mutates = reify(identifiers, afunc.mutates)
    identifiers, reified_returns = reify(identifiers, afunc.returns)

    # At this point we do not need object IDs anymore,
    # a mapping from symbols to values will suffice
    body_identifiers = Dict(pair[2] for pair in body_identifiers)
    identifiers = Dict(pair[2] for pair in identifiers)

    # Since we cannot create a context for the compiled function,
    # known arrays will have to be passed explicitly
    known_parameter_map = filter((sym, val) -> !isa(val, Unknown), identifiers)
    # According to docs, `keys` and `values` will have the same order
    known_parameter_symbols = collect(keys(known_parameter_map))
    known_arguments = collect(values(known_parameter_map))

    # Identify unused parameters and replace them with placeholders
    used_in_expressions = Set(
        filter(unknown_sym -> haskey(body_identifiers, unknown_sym), reified_parameters))
    used_in_mutations = Set(values(reified_mutates))
    used_unknown_parameters = union(used_in_expressions, used_in_mutations)
    unknown_parameter_symbols = [
        in(unknown_sym, used_unknown_parameters) ? unknown_sym : :_
        for unknown_sym in reified_parameters]

    function_name = gensym("array_function")

    # Construct the function
    expr = quote
        function $function_name(
                $(known_parameter_symbols...),
                $(unknown_parameter_symbols...))

            # Assignment to aliases
            #$(((if length(targets) == 1
            #        :( $(targets[1]) = $expr )
            #    else
            #        :( ($(targets...),) = $expr )
            #    end)
            #    for (targets, expr) in reified_aliases)...)
            $((:( const $target = $expr )
                for (target, expr) in reified_aliases)...)

            # Copying to mutated parameters
            $((:( copy!($target, $alias) )
                for (alias, target) in reified_mutates)...)

            # Returns
            $(if length(reified_returns) > 0
                :( ($(reified_returns...),) )
            end)
        end
    end

    #println("Function to compile:")
    #println("-----------------------------------")
    #println(expr)
    #println("-----------------------------------")

    func = eval(expr)

    JuliaArrayFunction(func, map(unwrap, known_arguments), mutated_parameters_indices(afunc))
end

struct JuliaBackendRandomState <: BackendRandomState
    random_state :: MersenneTwister
end

struct JuliaBackendArray <: BackendArray
    array :: AbstractArray
end

import Base: size, ndims, eltype

eltype(arr::JuliaBackendArray) = eltype(arr.array)
size(arr::JuliaBackendArray) = size(arr.array)
size(arr::JuliaBackendArray, dim) = size(arr.array, dim)
ndims(arr::JuliaBackendArray) = ndims(arr.array)

array_backend(::JuliaBackendRandomState) = JuliaBackend()
array_backend(::JuliaBackendArray) = JuliaBackend()
array_backend(::JuliaArrayFunction) = JuliaBackend()

unwrap(value::Number) = value
unwrap(value::AbstractArray) = value
unwrap(value::JuliaBackendRandomState) = value.random_state
unwrap(value::JuliaBackendArray) = value.array
unwrap(value::LazyArray) = value.value

wrap(value::Number) = value
wrap(value::AbstractArray) = JuliaBackendArray(value)
wrap(value::MersenneTwister) = JuliaBackendRandomState(value)

function random_state(::JuliaBackend; seed=nothing)
    if seed === nothing
        JuliaBackendRandomState(MersenneTwister())
    else
        JuliaBackendRandomState(MersenneTwister(seed))
    end
end

empty_array(::JuliaBackend, dtype, shape...) = JuliaBackendArray(Array(dtype, shape...))

to_backend(::JuliaBackend, arr::JuliaBackendArray) = arr
to_backend(::JuliaBackend, arr::AbstractArray) = JuliaBackendArray(arr)
from_backend(arr::JuliaBackendArray) = arr.array


struct DiagonalStateWrapper{T} <: FiniteBasisState where {T<:DoubleState}
    parent::T
end

# accessor
Base.parent(σ::DiagonalStateWrapper) = σ.parent

# Property Accessors
spacedimension(s::DiagonalStateWrapper) = spacedimension(row(s.parent))
nsites(s::DiagonalStateWrapper) = nsites(row(s.parent))
toint(s::DiagonalStateWrapper) = toint(row(s.parent))
index(s::DiagonalStateWrapper) = index(row(s.parent))
index_to_int(s::DiagonalStateWrapper, id) = index_to_int(row(s.parent), id)
flipped(a::DiagonalStateWrapper, b::DiagonalStateWrapper) =
    flipped(a.parent, b.parent)
@inline eltype(state::DiagonalStateWrapper) = eltype(state.parent)
@inline config(state::DiagonalStateWrapper) = config(state.parent)

zero!(s::DiagonalStateWrapper) = zero!(s.parent)
base_state(s::DiagonalStateWrapper) = s.parent

# Operations
function flipat!(rng::AbstractRNG, s::DiagonalStateWrapper, i)
    old, new = flipat!(rng, row(s.parent), i)
    setat!(col(s.parent), i, new)
    return (old, new)
end

function setat!(s::DiagonalStateWrapper, i, val)
    setat!(row(s.parent), i, val)
    setat!(col(s.parent), i, val)
    return s
end

set_index!(v::DiagonalStateWrapper, i::Integer) = set!(v, index_to_int(v, i))
function set!(s::DiagonalStateWrapper, i)
    set!(row(s.parent), i)
    set!(col(s.parent), i)
    return s
end

function add!(s::DiagonalStateWrapper, i)
    add!(row(s.parent), i)
    add!(col(s.parent), i)
    return s
end

function rand!(rng::AbstractRNG, s::DiagonalStateWrapper)
    rand!(rng, row(s.parent))
    set!(col(s.parent), toint(row(s.parent)))
    return s
end

# -------------- Base.show extension for nice printing -------------- #
Base.show(io::IO, mm::MIME"text/plain", bs::DiagonalStateWrapper) =
    print(io, "Diagonal State: $(bs.parent)\n")

Base.show(io::IO, bs::DiagonalStateWrapper) = print(io, "DiagSt: ", bs.parent)


save_output = false

using DSGE, ModelConstructors, Test, JLD2, FileIO, OrderedCollections, CSV, Random, DataFrames
path = dirname(@__FILE__)

m = AnSchorfheide()

m <= Setting(:use_population_forecast, false)
m <= Setting(:cond_id, 0)
m <= Setting(:date_forecast_start, quartertodate("2015-Q4"))
m <= Setting(:date_conditional_end, quartertodate("2015-Q4"))
m <= Setting(:forecast_block_size, 5)
m <= Setting(:data_vintage, "REF", true, "vint", "Date of data")
m <= Setting(:dataroot, "$path/../reference/input_data")

estroot = normpath(joinpath(dirname(@__FILE__), "..", "reference"))
overrides = forecast_input_file_overrides(m)
overrides[:mode] = joinpath(estroot, "optimize.h5")
overrides[:full] = joinpath(estroot, "metropolis_hastings.h5")

output_vars = add_requisite_output_vars([:histpseudo, :histobs, :histstdshocks,
                                         :histutpseudo, :histutobs,
                                         :hist4qpseudo, :hist4qobs,
                                         :forecaststates, :forecastpseudo, :forecastobs, :forecaststdshocks,
                                         :forecastutpseudo, :forecastutobs,
                                         :forecast4qpseudo, :forecast4qobs,
                                         :bddforecaststates, :bddforecastshocks, :bddforecastpseudo, :bddforecastobs,
                                         :shockdecpseudo, :shockdecobs,
                                         :trendstates, :trendobs, :trendpseudo,
                                         :dettrendstates, :dettrendobs, :dettrendpseudo,
                                         :irfstates, :irfpseudo, :irfobs])

Random.seed!(47)
df = VERSION >= v"1.3" ? CSV.read("$path/../reference/input_data/data/data_dsid=00_vint=REF.csv", DataFrame) :
    CSV.read("$path/../reference/input_data/data/data_dsid=00_vint=REF.csv")
forecast_one(m, :full, :none, output_vars, df = df, verbose = :none)
compute_meansbands(m, :full, :none, [:histobs, :forecastobs, :histpseudo, :forecastpseudo,
                                     :hist4qobs, :forecast4qobs, :hist4qpseudo, :forecast4qpseudo, :shockdecobs, :irfobs], df = df, verbose = :none)

hist, fore = construct_fcast_and_hist_dfs(m, :none, [:obs_gdp, :obs_cpi, :obs_nominalrate])
hist_noT, fore_noT = construct_fcast_and_hist_dfs(m, :none, [:obs_gdp, :obs_cpi, :obs_nominalrate], include_T_in_df_forecast = false)
hist_4q, fore_4q = construct_fcast_and_hist_dfs(m, :none, [:obs_gdp, :obs_cpi, :obs_nominalrate], use_4q = true)

if save_output
    fn = VERSION >= v"1.5" ? "$path/../reference/df_to_table_out_v1p5.jld2" : "$path/../reference/df_to_table_out.jld2"
    jldopen(fn, true, true, true, IOStream) do file
        file["hist"] = hist
        file["hist_noT"] = hist_noT
        file["hist_4q"] = hist_4q
        file["fore"] = fore
        file["fore_noT"] = fore_noT
        file["fore_4q"] = fore_4q
    end
end

fn = VERSION >= v"1.5" ? "$path/../reference/df_to_table_out_v1p5.jld2" : "$path/../reference/df_to_table_out.jld2"
saved_hist = load(fn, "hist")
saved_hist_noT = load(fn, "hist_noT")
saved_hist_4q = load(fn, "hist_4q")
saved_fore = load(fn, "fore")
saved_fore_noT = load(fn, "fore_noT")
saved_fore_4q = load(fn, "fore_4q")

@testset "Test df_to_table" begin
    @test Matrix(hist) == Matrix(saved_hist)
    @test Matrix(hist_noT) == Matrix(saved_hist_noT)
    @test Matrix(hist_4q) == Matrix(saved_hist_4q)
    @test Matrix(fore) == Matrix(saved_fore)
    @test Matrix(fore_noT) == Matrix(saved_fore_noT)
    @test Matrix(fore_4q) == Matrix(saved_fore_4q)
    @test_throws AssertionError construct_fcast_and_hist_dfs(m, :none, [:obs_gdp], save_to_table = true)
    # Test that saving to tex table functionality runs
    construct_fcast_and_hist_dfs(m, :none, [:obs_gdp], save_to_table = true, table_caption = "Test Caption",
                                 filename = "test.tex", savedir = "$path")
end

@testset "Test auxiliary methods" begin
    header_mappings = DSGE.create_table_header_mappings(m, collect(keys(m.observables)))
    @test header_mappings == Dict(:obs_gdp => Symbol("Real GDP Growth"), :obs_cpi => Symbol("CPI Inflation"), :obs_nominalrate => Symbol("Nominal FFR"))
    #df = load_data(m)
    @test DSGE.unit_mappings(m, df, header_mappings) == OrderedDict(Symbol("Real GDP Growth") =>
                                                             "(Q/Q) \\% Annualized",
                                                             Symbol("CPI Inflation") =>
                                                             "(Q/Q) \\% Annualized")
end

#These really belong in analysis/io.jl MeansBands tests but put them here to avoid duplicate meansbands computations
@testset "Test read_mb" begin
    # just make sure they run
    read_mb(m, :full, :none, :forecastobs)
    read_mb(m, :full, :none, :forecastobs, bdd_and_unbdd = true)
    DSGE.get_mb_metadata(m, :full, :none, :forecastobs)
    DSGE.get_mb_metadata(m, :full, :none, :irfobs)
end

@testset "Test creating q4q4 mb" begin
    mb_4q = read_mb(m, :full, :none, :forecast4qobs)
    @test typeof(create_q4q4_mb(mb_4q)) == MeansBands
end

@testset "Test write_meansbands_tables functions" begin
    @test_throws AssertionError write_meansbands_tables_timeseries(m, :mode, :none, :irf)
    write_meansbands_tables_timeseries(m, :full, :none, :histobs)
    write_meansbands_tables_timeseries(m, :full, :none, :histforecastobs)

    write_means_tables_shockdec(m, :full, :none, :obs)

    write_meansbands_tables_all(m, :full, :none, [:histobs, :shockdecobs])
    @test_throws ErrorException write_meansbands_tables_all(m, :full, :none, [:irfobs])
end

@testset "Test load_posterior_moments works" begin
    m <= Setting(:sampling_method, :MH)
    load_posterior_moments(m)
    moment_tables(m)
    moment_tables(m, use_mode = true)

    m <= Setting(:sampling_method, :SMC)
    m <= Setting(:n_parts, 20)
    m <= Setting(:n_Φ, 10)
    m <= Setting(:adaptive_tempering_target_smc, false)
    data = df_to_matrix(m, load_data(m))
    DSGE.smc2(m, data, verbose = :none, run_csminwel = false)
    load_posterior_moments(m)

end

@testset "Test meansbands_to_matrix works" begin
    meansbands_to_matrix(m, :full, :none, [:histobs])
end

fp = dirname(@__FILE__)
if isfile(joinpath(fp, "test.tex_forecast.tex"))
    rm(joinpath(fp, "test.tex_forecast.tex"))
end
if isfile(joinpath(fp, "test.tex_history.tex"))
    rm(joinpath(fp, "test.tex_history.tex"))
end


Base.convert(::Type{Expression}, ex::Expr) = Expression(ex)
function Expression(ex;mod=Main)
    ex.head === :if && (ex = Expr(:call, ifelse, ex.args...))
    ex.head === :call || throw(ArgumentError("internal representation does not support non-call Expr"))

    op = getproperty(mod,Symbol(ex.args[1]))
    args = convert.(Expression, ex.args[2:end])

    return Operation(op, args)
end
Base.convert(::Type{Expression}, x::Expression) = x
Base.convert(::Type{Expression}, x::Number) = Constant(x)
Base.convert(::Type{Expression}, x::Bool) = Constant(x)
Base.convert(::Type{Expression}, x::Variable) = convert(Operation,x)
Base.convert(::Type{Expression}, x::Operation) = x
Base.convert(::Type{Expression}, x::Symbol) = Operation(Variable(x),[])
Expression(x::Bool) = Constant(x)

function build_expr(head::Symbol, args)
    ex = Expr(head)
    append!(ex.args, args)
    ex
end

# used in parsing
isblock(x) = length(x) == 1 && x[1] isa Expr && x[1].head == :block
function flatten_expr!(x)
    isb = isblock(x)
    if isb
        x = MacroTools.striplines(x[1])
        filter!(z->z isa Symbol || z.head != :line, x.args)
        x = (x.args...,)
    end
    x
end

function detime_dvs(op::Operation)
  if op.op isa Variable
    Operation(Variable{vartype(op.op)}(op.op.name),Expression[])
  else
    Operation(op.op,detime_dvs.(op.args))
  end
end
detime_dvs(op::Constant) = op

function retime_dvs(op::Operation,dvs,iv)
  if op.op isa Variable && op.op ∈ dvs
    Operation(Variable{vartype(op.op)}(op.op.name),Expression[iv])
  else
    Operation(op.op,retime_dvs.(op.args,(dvs,),iv))
  end
end
retime_dvs(op::Constant,dvs,iv) = op

is_constant(::Constant) = true
is_constant(::Any) = false

is_operation(::Operation) = true
is_operation(::Any) = false

is_derivative(O::Operation) = isa(O.op, Differential)
is_derivative(::Any) = false

Base.occursin(t::Expression) = Base.Fix1(occursin, t)
Base.occursin(t::Expression, x::Operation ) = isequal(x, t) || any(occursin(t), x.args)
Base.occursin(t::Expression, x::Expression) = isequal(x, t)

vars(exprs) = foldl(vars!, exprs; init = Set{Variable}())
function vars!(vars, O)
    isa(O, Operation) || return vars
    O.op isa Variable && push!(vars, O.op)
    for arg ∈ O.args
        if isa(arg, Operation)
            isa(arg.op, Variable) && push!(vars, arg.op)
            vars!(vars, arg)
        end
    end

    return vars
end

# variable extraction
is_singleton(e) = false
is_singleton(e::Operation) = e.op isa Variable

"""
get_variables(O::Operation)

Returns the variables in the Operation
"""
get_variables!(vars, e::Constant, varlist=nothing) = vars
get_variables(e::Constant, varlist=nothing) = get_variables!(Operation[], e, varlist)

function get_variables!(vars, e::Operation, varlist=nothing)
    if is_singleton(e)
      (isnothing(varlist) ? true : (e.op in varlist)) && push!(vars, e)
    else
        foreach(x -> get_variables!(vars, x, varlist), e.args)
    end
    return unique(vars)
end
get_variables(e::Operation, varlist=nothing) = get_variables!(Operation[], e, varlist)

function get_variables!(vars, e::Equation, varlist=nothing)
  get_variables!(vars, e.rhs, varlist)
end
get_variables(e::Equation, varlist=nothing) = get_variables!(Operation[],e,varlist)

modified_states!(mstates, e::Equation, statelist=nothing) = get_variables!(mstates, e.lhs, statelist)

# variable substitution
"""
substitute(expr::Operation, s::Pair)
substitute(expr::Operation, s::Dict)
substitute(expr::Operation, s::Vector)

Performs the substitution `Operation => val` on the `expr` Operation.
"""
substitute(expr::Constant, s) = expr
substitute(expr::Operation, s::Pair) = _substitute(expr, [s[1]], [s[2]])
substitute(expr::Operation, dict::Dict) = _substitute(expr, keys(dict), values(dict))
substitute(expr::Operation, s::Vector) = _substitute(expr, first.(s), last.(s))

function _substitute(expr, ks, vs)
    _substitute(expr, Dict(map(Pair, map(to_symbolic, ks), map(to_symbolic, vs))))
end

function _substitute(expr, dict::Dict)
    simplify(SymbolicUtils.substitute(expr, dict))
end

@deprecate substitute_expr!(expr,s) substitute(expr,s)


using JSServe
using WGLMakie
using GeometryBasics
using FileIO
using WGLMakie: px
using JSServe: @js_str, onjs, App, Slider
using JSServe.DOM

JSServe.browser_display()

set_theme!(resolution=(1200, 800))

app = App() do
    return hbox(
        vbox(
            scatter(1:4, color=1:4),
            scatter(1:4, color=rand(RGBAf, 4)),
            scatter(1:4, color=rand(RGBf, 4)),
            scatter(1:4, color=:red),
            scatter(1:4)
        ),
        vbox(
            scatter(1:4, marker='☼'),
            scatter(1:4, marker=['☼', '◒', '◑', '◐']),
            scatter(1:4, marker="☼◒◑◐"),
            # scatter(1:4, marker=rand(RGBf, 10, 10), markersize=20px),
            scatter(1:4, markersize=20px),
            scatter(1:4, markersize=20, markerspace=Pixel),
            scatter(1:4, markersize=LinRange(20, 60, 4), markerspace=Pixel),
            scatter(1:4, marker='▲', markersize=0.3, rotations=LinRange(0, pi, 4)),
        )
    )
end

display(app)

app = App() do
    return DOM.div(
        meshscatter(1:4, color=1:4),
        meshscatter(1:4, color=rand(RGBAf, 4)),
        meshscatter(1:4, color=rand(RGBf, 4)),
        meshscatter(1:4, color=:red),
        meshscatter(rand(Point3f, 10), color=rand(RGBf, 10)),
        meshscatter(rand(Point3f, 10), marker=Pyramid(Point3f(0), 1f0, 1f0)),
    )
end

display(app)

app = App() do
    x = Point2f[(1, 1), (2, 2), (3, 2), (4, 4)]
    points = connect(x, LineFace{Int}[(1, 2), (2, 3), (3, 4)])
    return DOM.div(
        linesegments(1:4),
        linesegments(1:4, linestyle=:dot),
        linesegments(1:4, linestyle=[0.0, 1.0, 2.0, 3.0, 4.0]),
        linesegments(1:4, color=1:4),
        linesegments(1:4, color=rand(RGBf, 4), linewidth=10),
        linesegments(points)
    )
end

display(app)

app = App() do
    x = Point2f[(1, 1), (2, 2), (3, 2), (4, 4)]
    points = connect(x, LineFace{Int}[(1, 2), (2, 3), (3, 4)])
    return DOM.div(
        lines(1:4),
        lines(1:4, linestyle=:dot),
        lines(1:4, linestyle=[0.0, 1.0, 2.0, 3.0, 4.0]),
        lines(1:4, color=1:4),
        lines(1:4, color=rand(RGBf, 4), linewidth=10),
        lines(points)
    )
end

display(app)

app = App() do
    data = AbstractPlotting.peaks()
    return hbox(vbox(
        surface(-10..10, -10..10, data, show_axis=false),
        surface(-10..10, -10..10, data, color=rand(size(data)...))),
        vbox(surface(-10..10, -10..10, data, color=rand(RGBf, size(data)...)),
        surface(-10..10, -10..10, data, colormap=:magma, colorrange=(0.0, 2.0)),
    ))
end

display(app)

app = App() do
    return vbox(
        image(rand(10, 10)),
        heatmap(rand(10, 10)),
        image(rand(RGBAf, 10, 10)),
        heatmap(rand(RGBAf, 10, 10)),
        image(rand(RGBf, 10, 10)),
        heatmap(rand(RGBf, 10, 10)),
    )
end

display(app)

app = App() do
    return hbox(vbox(
        volume(rand(4, 4, 4), isovalue=0.5, isorange=0.01, algorithm=:iso),
        volume(rand(4, 4, 4), algorithm=:mip),
        volume(1..2, -1..1, -3..(-2), rand(4, 4, 4), algorithm=:absorption)),
        vbox(
        volume(rand(4, 4, 4), algorithm=Int32(5)),
        volume(rand(RGBAf, 4, 4, 4), algorithm=:absorptionrgba),
        contour(rand(4, 4, 4)),
    ))
end

display(app)

app = App() do
    cat = FileIO.load(MakieGallery.assetpath("cat.obj"))
    tex = FileIO.load(MakieGallery.assetpath("diffusemap.png"))
    return hbox(vbox(
        AbstractPlotting.mesh(Circle(Point2f(0), 1f0)),
        AbstractPlotting.poly(decompose(Point2f, Circle(Point2f(0), 1f0)))), vbox(
        AbstractPlotting.mesh(cat, color=tex),
        AbstractPlotting.mesh([(0.0, 0.0), (0.5, 1.0), (1.0, 0.0)]; color=[:red, :green, :blue], shading=false)
    ))
end

display(app)

function n_times(f, n=10, interval=0.5)
    obs = Observable(f(1))
    @async for i in 2:n
        try
            obs[] = f(i)
            sleep(interval)
        catch e
            @warn "Error!" exception=CapturedException(e, Base.catch_backtrace())
        end
    end
    return obs
end

function n_times(f, n=10, interval=1)
    obs = Observable(f(1))
    @async for i in 2:n
        try
            obs[] = f(i)
            sleep(interval)
        catch e
            @warn "Error!" exception=CapturedException(e, Base.catch_backtrace())
        end
    end
    return obs
end

app = App() do
    s1 = annotations(n_times(i-> map(j-> ("$j", Point2f(j*30, 0)), 1:i)), textsize=20,
                      limits=Rect2f(30, 0, 320, 50))
    s2 = scatter(n_times(i-> Point2f.((1:i).*30, 0)), markersize=20px,
                  limits=Rect2f(30, 0, 320, 50))
    s3 = linesegments(n_times(i-> Point2f.((2:2:2i).*30, 0)), limits=Rect2f(30, 0, 620, 50))
    s4 = lines(n_times(i-> Point2f.((2:2:2i).*30, 0)), limits=Rect2f(30, 0, 620, 50))
    return hbox(s1, s2, s3, s4)
end

display(app)

app = App() do
    outer_padding = 30
    scene, layout = layoutscene(
        outer_padding, resolution=(1200, 700),
        backgroundcolor=RGBf(0.98, 0.98, 0.98)
    )
    ax1 = layout[1, 1] = LAxis(scene, title="Sine")
    xx = 0:0.2:4pi
    line1 = lines!(ax1, sin.(xx), xx, color=:red)
    scat1 = scatter!(ax1, sin.(xx) .+ 0.2 .* randn.(), xx,
        color=(:red, 0.5), markersize=15px, marker='■')
    return scene
end

display(app)

app = App() do
    scene = Scene(resolution=(1000, 1000));
    campixel!(scene);

    maingl = GridLayout(scene, alignmode=Outside(30))

    las = Array{LAxis,2}(undef, 4, 4)

    for i in 1:4, j in 1:4
        las[i, j] = maingl[i, j] = LAxis(scene)
    end

    las[4, 1].attributes.aspect = AxisAspect(1)
    las[4, 2].attributes.aspect = AxisAspect(2)
    las[4, 3].attributes.aspect = AxisAspect(0.5)
    las[4, 4].attributes.aspect = nothing
    las[1, 1].attributes.maxsize = (Inf, Inf)
    las[1, 2].attributes.aspect = nothing
    las[1, 3].attributes.aspect = nothing

    subgl = gridnest!(maingl, 1, 1)
    cb1 = subgl[:, 2] = LColorbar(scene, width=30, height=Relative(0.66))

    subgl2 = gridnest!(maingl, 1:2, 1:2)
    cb2 = subgl2[:, 3] = LColorbar(scene, width=30, height=Relative(0.66))

    subgl3 = gridnest!(maingl, 1:3, 1:3)
    cb3 = subgl3[:, 4] = LColorbar(scene, width=30, height=Relative(0.66))

    subgl4 = gridnest!(maingl, 1:4, 1:4)
    cb4 = subgl4[:, 5] = LColorbar(scene, width=30, height=Relative(0.66))
    return scene

end

display(app)

app = App() do
    sl = JSServe.Slider(1:10)
    rect = Rect2f(0, -5, 1025, 10)
    chars = [collect('a':'z'); 0:9;]
    char2 = [collect('A':'Z'); 0:9;]
    tex1 = map(x->map(j-> ("$(chars[rand(1:length(chars))])", Point2f(j*30, 0)), 1:36), sl)
    tex2 = map(x->map(j-> ("$(char2[rand(1:length(char2))])", Point2f(j*30, 1)), 1:36), sl)
    scene = annotations(tex1, textsize=20, limits=rect, show_axis=false)
    annotations!(scene, tex2, textsize=20, limits=rect, show_axis=false)
    return DOM.div(sl, scene)
end

display(app)


facts("Async") do

    context("async_map") do
        x = Signal(1; name="x")
        t, y = async_map(-, 0, x)
        z = map(yv->2yv, y; name="z")

        @fact value(t) --> nothing
        @fact value(y) --> 0
        @fact value(z) --> 0

        push!(x, 2)
        step()
        step()

        @fact value(y) --> -2
        @fact value(z) --> -4
    end
end


#=
Script to generate QASM for Quantum Fourier Transform circuits
=#

using PyCall
using ArgParse
using PicoQuant

"""
    parse_commandline()

Parse command line options and return argument dictionary
"""
function parse_commandline(ARGS)
    s = ArgParseSettings()
        @add_arg_table! s begin
            "--number-qubits", "-n"
                help = "Size of circuit"
                arg_type = Int
                required = true
            "--output", "-o"
                help = "Output file to write tensor network graph to"
                arg_type = String
                default = ""
            "--latex-circuit"
                help = "Generate latex for circuit diagram"
                action = :store_true
            "--text-circuit"
                help = "Generate ASCII for circuit diagram"
                action = :store_true
        end
        return parse_args(ARGS, s)
end

function main(ARGS)
    parsed_args = parse_commandline(ARGS)

    qubits = parsed_args["number-qubits"]

    circuit = create_qft_circuit(qubits)

    if parsed_args["output"] == ""
        filename = "qft_$(qubits).qasm"
    else
        filename = parsed_args["output"]
    end

    open(filename, "w") do io
        write(io, circuit.qasm())
    end

    if parsed_args["latex-circuit"]
        latex_filename = "$(splitext(filename)[1]).tex"
        circuit.draw(output="latex_source", filename=latex_filename)
    end

    if parsed_args["text-circuit"]
        text_filename = "$(splitext(filename)[1]).txt"
        circuit.draw(output="text", filename=text_filename)
    end

end

if abspath(PROGRAM_FILE) == @__FILE__
    main(ARGS)
end


using Test
using Printf
using FileIO

include("../../src/image.jl")

# the tested code
include("../../src/synthPoints.jl")

include("tinyImage.jl")

@testset "SortedOffsets" begin

    testImage = createTestMaskedImage()

    testSynthPoints = generateOrderedSynthPoints(testImage)

    println(testSynthPoints)
    @debug "Synth points" testSynthPoints

    for point in testSynthPoints
        println(point)
    end

end