include("GeneralMethods.jl")
include("SuperOperator.jl")
include("Ising.jl")

###########################
###Core of the algorithm###
###########################
"""
    computeDerivative!(vectdRho, vectRho, Hinside, leftApprox, rightApprox, tempMatrix, Traces)

Compute the derivative of the densityMatrix
vectDRho stock the derivative
vectRho is the current vector of DM
Hinside is the Hamiltonian acting inside the DM
leftApprox correspond to the [TrL; TrR Tr L sz]^{-1}  [Tr ([sz_0 sz_1, .]); 0; 0]
rightApprox correspond to the [TrR; TrL Tr R sz]^{-1}  [Tr ([sl_0 sz_{l+1}}, .]); 0; 0]
tempMatrix is a simple temporary storage for some extremity matrix. Could be improved
Traces are the matrix of Trace operators
"""
function computeDerivative!(vectDRho, vectRho, Hinside, leftApprox, rightApprox, tempMatrix, Traces)
    insideEvolution!(vectDRho, vectRho, Hinside)
    outsideEvolution!(vectDRho, vectRho, leftApprox, rightApprox, tempMatrix, Traces)
end

"""
    insideEvolution!(vectDRho, vectRho, Hinside)

Compute the derivative of the densityMatrix according to Hinside
vectDRho stock the derivative
vectRho is the current vector of DM
Hinside is the Hamiltonian acting inside the DM
"""
function insideEvolution!(vectDRho, vectRho, Hinside)
    for j=1:length(vectRho)
        A_mul_B!(vectDRho[j], Hinside, vectRho[j])
    end
end

"""
    outsideEvolution!(vectdRho, vectRho, leftApprox, rightApprox, tempMatrix, Traces)

Increment the derivative of the densityMatrix by the link contribution
vectDRho stock the derivative
vectRho is the current vector of DM
leftApprox correspond to the [TrL; TrR Tr L sz]^{-1}  [Tr ([sz_0 sz_1, .]); 0; 0]
rightApprox correspond to the [TrR; TrL Tr R sz]^{-1}  [Tr ([sl_0 sz_{l+1}}, .]); 0; 0]
tempMatrix is a simple temporary storage for some extremity matrix. Could be improved
Traces are the matrix of Trace operators
"""
function outsideEvolution!(vectDRho, vectRho, leftApprox, rightApprox, tempMatrix, Traces)
    ###Auxiliary matrix to stock the boundaries  toDO=>removeThatStep
    leftFusionIdentity!(tempMatrix[1], reduceDensityMatrixRight(vectRho[1], Traces));
    rightFusionIdentity!(tempMatrix[2], reduceDensityMatrixLeft(vectRho[end], Traces));
    ###
    for j=1:length(vectDRho)-1
        vectDRho[j].=vectDRho[j]+rightApprox*vectRho[j+1]
    end
    vectDRho[end].=vectDRho[end]+rightApprox*tempMatrix[2]
    for j=2:length(vectDRho)
        vectDRho[j].=vectDRho[j]+leftApprox*vectRho[j-1]
    end
    vectDRho[1].=vectDRho[1]+leftApprox*tempMatrix[1]
end

"""
    extendZone!(vectDRho, vectRho, Traces)

Check if one needs to extend the considered range
vectDRho stock the derivative
vectRho is the current vector of DM
Traces are the matrix of Trace operators
"""
function extendZone!(vectDRho, vectRho, Traces)
    len=getLength(length(vectRho[1])); expanded=false
    ##Check if left is entangled
    mi=mutualInformation(vectRho[1], 0, 1, len, Traces)
    if mi>1e-7
        unshift!(vectRho, leftFusionIdentity(reduceDensityMatrixRight(vectRho[1], Traces)))
        unshift!(vectDRho, zeros(Complex128, length(vectDRho[1])))
        expanded=true
    end
    ##Same for right extremity
    mi=mutualInformation(vectRho[end], 0, len-1, len, Traces)
    if mi>1e-7
        push!(vectRho, rightFusionIdentity(reduceDensityMatrixLeft(vectRho[end], Traces)))
        push!(vectDRho, zeros(Complex128, length(vectDRho[1])))
    end
    if expanded
        return 1
    else
        return 0
    end
end

"""
    extendZone!(vectDRho, vectDRhoAux, vectRho, Traces)

Check if one needs to extend the considered range
vectDRho and vectDRhoAux stock the derivative
vectRho is the current vector of DM
Traces are the matrix of Trace operators
"""
function extendZone!(vectDRho, vectDRhoAux, vectRho, Traces)
    len=getLength(length(vectRho[1])); expanded=false
    ##Check if left is entangled
    mi=mutualInformation(vectRho[1], 0, 1, len, Traces)
    if mi>1e-7
        unshift!(vectRho, leftFusionIdentity(reduceDensityMatrixRight(vectRho[1], Traces)))
        unshift!(vectDRho, zeros(Complex128, length(vectDRho[1])))
        unshift!(vectDRhoAux, zeros(Complex128, length(vectDRhoAux[1])))
        expanded=true
    end
    ##Same for right extremity
    mi=mutualInformation(vectRho[end], 0, len-1, len, Traces)
    if mi>1e-7
        push!(vectRho, rightFusionIdentity(reduceDensityMatrixLeft(vectRho[end], Traces)))
        push!(vectDRho, zeros(Complex128, length(vectDRho[1])))
        push!(vectDRhoAux, zeros(Complex128, length(vectDRhoAux[1])))
    end
    if expanded
        return 1
    else
        return 0
    end
end

"""
    buildSpecialLinkProjector(maxLen, Hlink, Hcommuting)

Build the matrix operator corresponding to [TrL; TrR Tr L Hcommuting]^{-1}  [Tr ([sz_0 sz_1, .]); 0; 0]
and [TrR; TrL Tr R Hcommuting]^{-1}  [Tr ([sl_0 sz_{l+1}}, .]); 0; 0].
"""
function buildSpecialLinkProjector(maxLen, Hlink, Hcommuting)
    TrRight=buildTraceMatrixRight(maxLen-1);
    TrLeft=buildTraceMatrixLeft(maxLen-1);
    ###For the left link
    leftCommuter=buildSuperOperatorLeft(kron(speye(2^(maxLen-1)), Hcommuting))
    MatJoin=sparse(vcat(TrRight, TrLeft, TrLeft*leftCommuter));
    invMatJoin=sparse(triming.(pinv(full(MatJoin))));
    leftEvolutionOperator=invMatJoin[:, 1:2^(2*maxLen-2)]*TrLeft*Hlink[1]
    ###For the right link
    rightCommuter=buildSuperOperatorLeft(kron(Hcommuting, speye(2^(maxLen-1))))
    MatJoin=sparse(vcat(TrLeft, TrRight, TrRight*rightCommuter));
    invMatJoin=sparse(triming.(pinv(full(MatJoin))));
    rightEvolutionOperator=invMatJoin[:, 1:2^(2*maxLen-2)]*TrRight*Hlink[2]
    return leftEvolutionOperator, rightEvolutionOperator
end

"""
    buildSpecialLinkProjector_largeSize(maxLen, Hlink, Hcommuting)

Build the matrix operator corresponding to [TrL; TrR Tr L Hcommuting]^{-1}  [Tr ([sz_0 sz_1, .]); 0; 0]
and [TrR; TrL Tr R Hcommuting]^{-1}  [Tr ([sl_0 sz_{l+1}}, .]); 0; 0].
Useful for large matrices when the pseudoinverse cannot be taken
"""
function buildSpecialLinkProjector_largeSize(maxLen, Hlink, Hcommuting)
    TrRight=buildTraceMatrixRight(maxLen-1);
    TrLeft=buildTraceMatrixLeft(maxLen-1);
    ###For the left link
    leftCommuter=buildSuperOperatorLeft(kron(speye(2^(maxLen-1)), Hcommuting))
    MatJoin=sparse(vcat(TrRight, TrLeft, TrLeft*leftCommuter));
    tempLeftEvolutionOperator=speye(size(MatJoin)[1])[:, 1:2^(2*maxLen-2)]
    leftEvolutionOperator=spzeros(size(MatJoin)[2], 2^(2*maxLen-2))
    for j=1:2^(2*maxLen-2)
        leftEvolutionOperator[:, j]=triming.(MatJoin\full(tempLeftEvolutionOperator[:, j]))
    end
    leftEvolutionOperator=leftEvolutionOperator*TrLeft*Hlink[1]
    ###For the right link
    rightCommuter=buildSuperOperatorLeft(kron(Hcommuting, speye(2^(maxLen-1))))
    MatJoin=sparse(vcat(TrLeft, TrRight, TrRight*rightCommuter));
    tempRightEvolutionOperator=speye(size(MatJoin)[1])[:, 1:2^(2*maxLen-2)]
    rightEvolutionOperator=spzeros(size(MatJoin)[2], 2^(2*maxLen-2))
    for j=1:2^(2*maxLen-2)
        rightEvolutionOperator[:, j]=triming.(MatJoin\full(tempLeftEvolutionOperator[:, j]))
    end
    rightEvolutionOperator=rightEvolutionOperator*TrRight*Hlink[2]
    return leftEvolutionOperator, rightEvolutionOperator
end

###################
###Main function###
###################

###toDo look static arrays ?
function main(hx::Float64, hz::Float64, T::Float64; maxLen=3, dt=0.01, order=2, ini=1.0)
    #Define Pauli Matrices
    sx=[0 1; 1 0]; sz=-[1  0 ; 0 -1]; sy=[0 -1im; 1im 0]
    tim=0:dt:T
    #Define Hamiltonian
    Hinside=buildSuperCommutator(-1im*dt*buildInsideHamiltonian(hx, hz, maxLen));  #### Always multiplication on the right => slightly faster. Will have to recheck
    Hlink=buildSuperCommutator.(-1im*dt*buildLinkHamiltonian(hx, hz, maxLen));
    ###Setting up the global trace operators: note, the original one site reduced DM is marginally faster
    ###Choosing global matrixes seem to slow down code by ~1% but is much easier to read. More benchmark to be done later ?
    Traces=initializeTraceMatrices(maxLen);
    if maxLen<7
        leftApproximation, rightApproximation=buildSpecialLinkProjector(maxLen, Hlink, sparse(sx))
    else
        leftApproximation, rightApproximation=buildSpecialLinkProjector_largeSize(maxLen, Hlink, sparse(sx))
    end
    ###Initialization of the density operators.
    vectRho=map(x->reshape(x, length(x)), generateInitialState_z(maxLen, ini=ini));  minMatRho=-maxLen+1
    tempMatrix=[zeros(Complex128, 2^(2*maxLen)), zeros(Complex128, 2^(2*maxLen))]
    if order==1
        vectDRho=similar(vectRho)
        for j=1:length(vectDRho)
            vectDRho[j]=zeros(Complex128, 2^(2*maxLen))
        end
    elseif order==2
        vectDRho=similar(vectRho);  vectDRhoAux=similar(vectRho)
        for j=1:length(vectDRho)
            vectDRho[j]=zeros(Complex128, 2^(2*maxLen))
            vectDRhoAux[j]=zeros(Complex128, 2^(2*maxLen))
        end
    elseif order==4
        vectDRho=similar(vectRho);  vectDRhoAux=similar(vectRho)
        for j=1:length(vectDRho)
            vectDRho[j]=zeros(Complex128, 2^(2*maxLen))
            vectDRhoAux[j]=zeros(Complex128, 2^(2*maxLen))
        end
    else
        println("This order has not been implemented")
        return nothing
    end
    ###Observables
    tim2=0:10*dt:T;
    szObservable=zeros(length(tim2));
    sxObservable=zeros(length(tim2));
    test=zeros(length(tim));
    ###Last preparations
    tic()
    cnt=0; totaltime=0; cnt2=1;
    #szObservable[1]=real(trace(sz*reshape(buildReducedDensityMatrix_small(vectRho[1-minMatRho], 0), (2, 2))))
    #sxObservable[1]=real(trace(sx*reshape(buildReducedDensityMatrix_small(vectRho[1-minMatRho], 0), (2, 2))))
    ###Start of the time evolution
    for j in 1:length(tim)
        ####Printing update
        if mod(j, (length(tim)+99)÷100)==0
            cnt+=1
            ctime=toq()
            println(string(cnt)*"% of simulation done. Time for the last %: "*string(round(ctime, 2))*"sec.")
            totaltime+=ctime
            println("Total elapsed time: "*string(round(totaltime, 2))*"sec.")
            println((minMatRho, minMatRho+length(vectRho)+maxLen-2))
            tic()
        end
        ###Measure of observables
        if tim[j]==tim2[cnt2]
            szObservable[cnt2]=real(trace(sz*reshape(buildReducedDensityMatrix(vectRho[1-minMatRho], 0, Traces), (2, 2))))
            sxObservable[cnt2]=real(trace(sx*reshape(buildReducedDensityMatrix(vectRho[1-minMatRho], 0, Traces), (2, 2))))
            cnt2+=1
        end
        if order==1
            computeDerivative!(vectDRho, vectRho, Hinside, leftApproximation, rightApproximation, tempMatrix, Traces)
            for l=1:length(vectDRho)
                @. vectRho[l]+= vectDRho[l]
            end
            minMatRho-=extendZone!(vectDRho, vectRho, Traces)
        elseif order==2
            computeDerivative!(vectDRho, vectRho, Hinside, leftApproximation, rightApproximation, tempMatrix, Traces)
                for l=1:length(vectDRho)
                    @. vectRho[l]+= 0.5*vectDRho[l]
                end
                for l=1:length(vectDRho)
                    @. vectDRhoAux[l].=vectRho[l]
                    @. vectDRhoAux[l]+=0.5*vectDRho[l]
                end
                computeDerivative!(vectDRho, vectDRhoAux, Hinside, leftApproximation, rightApproximation, tempMatrix, Traces)
                for l=1:length(vectRho)
                    @. vectRho[l]+= 0.5*vectDRho[l]
                end
                minMatRho-=extendZone!(vectDRho, vectDRhoAux, vectRho, Traces)
            test[j]=minimum(minimum.(real.(eigvals.(map(x->reshape(x, (2^maxLen, 2^maxLen)), vectRho)))))
         elseif order==4
            println("Not yet implemented")
        else
            println("Bug in order")
        end
    end
    toc()
    return tim2, szObservable, sxObservable, test
end

#
# tim, sz, sx, test=main(0.25, -0.525, 100., maxLen=4, dt=0.0001, order=2, ini=0.6)
# tim2, sz2, sx2, test2=main(0.25, -0.525, 100., maxLen=5, dt=0.0001, order=2, ini=0.6)
# tim3, sz3, sx3, test3=main(0.25, -0.525, 100., maxLen=6, dt=0.0005, order=2, ini=0.6)
# tim4, sz4, sx4, test4=main(0.25, -0.525, 100., maxLen=7, dt=0.001, order=2, ini=0.6)
# timt, szt, sxt, testt=main(0.25, -0.525, 100., maxLen=4, dt=0.0001, order=2)
# timt2, szt2, sxt2, testt2=main(0.25, -0.525, 100., maxLen=5, dt=0.0001, order=2)
#
#
# figure()
# plot(log.(abs.(tim)), log.(abs.(sz/2*5)))
# plot(log.(abs.(tim2)), log.(abs.(sz2/2*5)))
# plot(log.(abs.(tim3)), log.(abs.(sz3/2*5)))
# plot(log.(abs.(tim4)), log.(abs.(sz4/2*5)))
# plot(log.(abs.(timt)), log.(abs.(szt/2)), ":")
# plot(log.(abs.(timt2)), log.(abs.(szt2/2)), ":")
# legend(["l=4", "l=5", "l=6", "l=7", "l=4 ori", "l=5 ori"])
#
# figure()
# plot(test)
# plot(test2)
# plot(test3)
# plot(test4)
# plot(testt)
# plot(testt2)
# legend(["l=4", "l=5", "l=6", "l=7", "l=4 ori", "l=5 ori"])


###############################################################################
#
# Containers for Simulation results
#
###############################################################################
"""
    mutable struct SPACContainer1L{FT}

Struct that contains 1-layer gas exchange information.

# Fields
$(TYPEDFIELDS)
"""
Base.@kwdef mutable struct SPACContainer1L{FT<:AbstractFloat}
    "Mean gross photosynthetic rate `[μmol m⁻² s⁻¹]`"
    ag::FT = FT(0)
    "Mean net photosynthetic rate `[μmol m⁻² s⁻¹]`"
    an::FT = FT(0)
    "Leaf internal CO₂ partial pressure `[Pa]`"
    c ::FT = FT(0)
    "Flow rate per basal area `[mol s⁻¹ m⁻²]`"
    e ::FT = FT(0)
    "Leaf diffusive conductance to H₂O `[mol m⁻² s⁻¹]`"
    gh::FT = FT(0)
    "Xylem end pressure `[MPa]`"
    p ::FT = FT(0)
    "Leaf temperature `[K]`"
    t ::FT = FT(0)
end

"""
    mutable struct SPACContainer2L{FT}

Struct that contains 2-layer gas exchange information.

# Fields
$(TYPEDFIELDS)
"""
Base.@kwdef mutable struct SPACContainer2L{FT<:AbstractFloat}
    # gas exchange information
    cont_sh::SPACContainer1L{FT} = SPACContainer1L{FT}();
    cont_sl::SPACContainer1L{FT} = SPACContainer1L{FT}();

    # partition information
    "Shaded layer fraction"
    frac_sh::FT = FT(0)
    "Sunlit layer fraction"
    frac_sl::FT = FT(0)
    "Shaded layer leaf area `[m²]`"
    la_sh  ::FT = FT(0)
    "Sunlit layer leaf area `[m²]`"
    la_sl  ::FT = FT(0)
    "Shaded layer LAI"
    lai_sh ::FT = FT(0)
    "Sunlit layer LAI"
    lai_sl ::FT = FT(0)
    "Shaded layer PAR `[μmol m⁻² s⁻¹]`"
    par_sh ::FT = FT(0)
    "Sunlit layer PAR `[μmol m⁻² s⁻¹]`"
    par_sl ::FT = FT(0)
    "Shaded layer absorbed energy `[W m⁻²]`"
    rad_sh ::FT = FT(0)
    "Sunlit layer absorbed energy `[W m⁻²]`"
    rad_sl ::FT = FT(0)
end


"""
`module  JAC.ManyElectronAZ`
    ... a submodel of JAC that contains methods that support tasks related to spectroscopic computation.
"""
module ManyElectronAZ

    using Printf, ..Basics, ..Defaults, ..ManyElectron, ..Radial

    export provideSubshellStates

    """
    `ManyElectron.provideSubshellStates(sh::Subshell, occ::Int64)`
        ... to provide all antisymmetric SubshellStates within the relativistic seniority scheme for the given
            shell and occupation; an Array{SubshellStateR,1} is returned.
    """
    function ManyElectron.provideSubshellStates(sh::Subshell, occ::Int64)

        # Use the j-value = wa[4] of the subshell to select the allowed SubshellStateR's
        wb = Basics.SubshellStateR[]

        if       Basics.subshell_2j(sh) == 1
            if       occ == 0  || occ == 2      push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) );       return(wb)
            elseif   occ == 1                   push!( wb, Basics.SubshellStateR( sh, occ, 1, 1) );       return(wb)
            end
        elseif   Basics.subshell_2j(sh) == 3
            if       occ == 0  || occ == 4      push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) );       return(wb)
            elseif   occ == 1  || occ == 3      push!( wb, Basics.SubshellStateR( sh, occ, 1, 3) );       return(wb)
            elseif   occ == 2                   push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2, 4) );       return(wb)
            end
        elseif   Basics.subshell_2j(sh) == 5
            if       occ == 0  || occ == 6      push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) );       return(wb)
            elseif   occ == 1  || occ == 5      push!( wb, Basics.SubshellStateR( sh, occ, 1, 5) );       return(wb)
            elseif   occ == 2  || occ == 4      push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2, 4) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2, 8) );       return(wb)
            elseif   occ == 3                   push!( wb, Basics.SubshellStateR( sh, occ, 1, 5) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 3, 3) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 3, 9) );       return(wb)
            end
        elseif   Basics.subshell_2j(sh) == 7
            if       occ == 0  || occ == 8      push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) );       return(wb)
            elseif   occ == 1  || occ == 7      push!( wb, Basics.SubshellStateR( sh, occ, 1, 7) );       return(wb)
            elseif   occ == 2  || occ == 6      push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2, 4) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2, 8) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2,12) );       return(wb)
            elseif   occ == 3  || occ == 5      push!( wb, Basics.SubshellStateR( sh, occ, 1, 7) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 3, 3) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 3, 5) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 3, 9) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 3,11) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 3,15) );       return(wb)
            elseif   occ == 4                   push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2, 4) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2, 8) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 2,12) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 4, 4) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 4, 8) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 4,10) )
                                                push!( wb, Basics.SubshellStateR( sh, occ, 4,16) );       return(wb)
            end
        elseif   Basics.subshell_2j(sh) in [ 9, 11, 13, 15]
            if       occ == 0                   push!( wb, Basics.SubshellStateR( sh, occ, 0, 0) );                 return(wb)
            elseif   occ == 1                   push!( wb, Basics.SubshellStateR( sh, occ, 1, subshell_2j(sh)) );   return(wb)
            else     error("stop b")
            end

        # For an extension to other subshell-occupations, see the Racah-manual; Table ...
        else  error("stop a")
        end
    end

end # module


function TwoBodiesBaryFrame(arg0::CelestialBody, arg1::CelestialBody)
    return TwoBodiesBaryFrame((CelestialBody, CelestialBody), arg0, arg1)
end


export
    initialize_XML,
    get_lane_length,
    handle_junctions,
    convert_roadway!,
    Connection,
    Junction,
    connect_two_lane_general!,
    connect_two_seg!,
    connect_two_seg_general!,
    add_connection!,
    add_junction!,
    gen_connected_lanes,
    gen_intersection,
    gen_loop_roadway,
    add_line!,
    in_lanes,
    get_max_curvature

include("roadway_generation.jl")
include("roadway_perception.jl")
include("roadway_intersection.jl")
include("convert2xodr.jl")


@testset "Vector" begin
    x = randn(2)
    syms = (:a, :b)

    # Constructors
    test_rrule(LArray{syms}, x)
    constructor = @SLArray (2, ) syms
    test_rrule(constructor, x)

    # `getproperty`
    lx = LArray{syms}(x)
    test_rrule(getproperty, lx, first(syms))
    test_rrule(getproperty, lx, last(syms))

    slx = constructor(x)
    test_rrule(getproperty, slx, first(syms))
    test_rrule(getproperty, slx, last(syms))
end


#load and process funds as desired
function prepfunds(returns::DataFrame,
    ms::Union{Nothing, MeasureSpec}=nothing,
    ;refreshfunds::Bool = PARAM[:refreshfunds],
    refreshcomom::Bool = PARAM[:refreshcomom],
    workingpath::String = PARAM[:workingpath],
    loadpath::NString = nothing,
    hfrname::String = PARAM[:hfrname],
    mfname::String = PARAM[:mfname],
    returntype::String = PARAM[:crspdatalabel],
    plname::String = PARAM[:plname],
    weeklycrsptype::String = string(PARAM[:crspweekly]),
    dtformat = PARAM[:wrdsdateformat])

  local hf::DataFrame
  local mf::DataFrame
  local comom::DataFrame

  if refreshfunds

    hf = prephf(returns::DataFrame, ms::MeasureSpec)
    hf |> CSV.write("$workingpath\\$(returntype)_$(hfrname).csv")

    mf = prepmf(returns::DataFrame, ms::MeasureSpec)
    mf |> CSV.write("$workingpath\\$(returntype)_$(mfname).csv")
  else
    hf = CSV.File("$workingpath\\$(returntype)_$(hfrname).csv") |> DataFrame
    mf = CSV.File("$workingpath\\$(returntype)_$(mfname).csv") |> DataFrame
  end
  if refreshcomom
    if PARAM[:crspfrequencysuffix] ≡ "m"
      comom = generatemonthlycomomentum()
      #comom = generatecomomentumbb()
    elseif PARAM[:crspfrequencysuffix] ≡ "w"
      comom = generateweeklycomomentum()
    else
      @assert false
    end
    comom |> CSV.write("$workingpath\\$(returntype)_$(plname).csv")
  else
    if isfile("$workingpath\\$(returntype)_$(plname).csv")
      comom = CSV.File("$workingpath\\$(returntype)_$(plname).csv") |> DataFrame
    else
      comom = DataFrame()
    end
  end

  return (;hf, mf, comom)
end

#=function betaweightsslice(df, extrabetaweights::Vector{Symbol}=PARAM[:fundextrabetaweights],
  Fbetacontrols=PARAM[:fundbetacontrols],
  betacorridorradius=PARAM[:fundbetacorridorradius])

  throw("THIS NEEDS TO BE COMPLETED")
  #this will hold the weights by fundid
  #the below version must work with missing data
  weights = DataFrame(fundid=unique(df.fundid))
  function betaw(t,Frhs)
    completeindices = sum()
    X = reduce(hcat, (F->t[F]).(Frhs)

    (svd(Xclean))\(t.netret))[1]
  end
  local Fweights = Vector{Symbol}()
  #now execute the regressions
  for Fbetaweight ∈ Fbetaweights
    Fweight = Symbol(:w_, Fbetaweight)
    Fabsweight = Symbol(:absw_, Fbetaweight)
    Fabscorweight = Symbol(:abscorw_, Fbetaweight)
    newweightfields = [Fweight,Fabsweight,Fabscorweight]

    #form a view with no missing values
    focalfields = [:date; :fundid; :some1s; Fbetaweight; :netret; Fbetacontrols]
    Frhs = [Fbetaweight; Fbetacontrols; :some1s;]
    cleandf = view(df, completecases(df, [Fbetaweight; Fbetacontrols; :netret]), focalfields)

    #run the regression for each fundid
    #we are primarily interested in the absolute vlaue of the weights
    #the corrodor version creates a buffer to guard against measurement error in the controls
    weight = combine(groupby(cleandf, :fundid), (:) |> AsTable =>
      ((t)->betaw(t,Frhs)) => Fweight)
    weight[!, Fabsweight] = abs.(weight[!, Fweight])
    weight[!, Fabscorweight] = (x->ifelse(x<betacorridorradius,0.0, x)).(weight[!, Fabsweight])
    weights = innerjoin(weights, weight[:, [:fundid; newweightfields]], on=[:fundid])
    append!(Fweights, newweightfields)
  end

end=#

function betaweights(df, returns, ms::MeasureSpec;
  extrabetaportfolios::Vector{Vector{Symbol}}=PARAM[:fundextrabetaportfolios],
  Fbetacontrols=PARAM[:fundbetacontrols],
  betacorridorradius=PARAM[:fundbetacorridorradius],
  datefrequency::Symbol = throw("datefrequency is required"),
  testlabel=throw("testlabel is required"),
  Frfr::Symbol = PARAM[:fundrfr],
  )

  @assert issorted(df, [:fundid, :date])

  #just get the parts we want to merge from the returns dataframe
      #crashed here in the returns file
  #throwing the error here from funds-mf: Merge key(s) in df1 are not unique
  df = mergereturns(df, returns,
    Fgroup = :fundid,
    Frets = [ms.Fξs; extrabetaportfolios...; Fbetacontrols;Frfr],
    validateexactfrequency=true;
    datefrequency, testlabel)

  #=WARNING- the below code uses all available data, but will use different fund-date
  #universes for different beta controls. I leave it disabled because its also very slow
  #also it shouldn't mkae a big difference
  for Fret ∈ [Fbetaweights;Fbetacontrols]
    if :indexdate ∈ propertynames(df)
      select!(df, Not([:indexdate]))
    end
    df = mergereturns(df, returns,
      Fgroup = :fundid,
      Frets = [Fret;],
      validateexactfrequency=true;
      datefrequency, testlabel)
  end=#

  #this is the intercept
  df.some1s = ones(nrow(df))
  df.netret = df.ret .- df[!, Frfr]

  #this will hold the weights by fundid
  weights = DataFrame(fundid=unique(df.fundid))

  #runs the regression
  #form the LHS matrix then
  #betaw(t,Fbetaweight) = (svd([t[Fbetaweight] t.some1s])\(t.ret))[1]
  #extracts the relevant coefficeints of interest for the betas
  function betaw(t,Frhs,Fportfolioweights)
    β = (svd(reduce(hcat, (F->t[F]).(Frhs)))\(t.netret))
    return Dict(Fw => β[i] for (i,Fw) ∈ enumerate(Fportfolioweights)) |> NamedTuple
  end

  local Fweights = Vector{Symbol}()
  Fbetaportfolios::Vector{Vector{Symbol}} =   [ms.Fξs]
  append!(Fbetaportfolios, extrabetaportfolios)
  #now execute the regressions
  for Fbetaportfolio ∈ Fbetaportfolios
    Fportfolioweights = Fbetaportfolio .|> s->Symbol(:w_, s)
    Fportfolioabsweights = Fbetaportfolio .|> s->Symbol(:absw_, s)
    Fportfolioabscorweights = Fbetaportfolio .|> s->Symbol(:abscorw_, s)
    newweightfields = [Fportfolioweights; Fportfolioabsweights; Fportfolioabscorweights]

    #form a view with no missing values
    focalfields = [:date; :fundid; :some1s; Fbetaportfolio; :netret; Fbetacontrols]
    Frhs = [Fbetaportfolio; Fbetacontrols; :some1s;]
    cleandf = view(df, completecases(df, [Fbetaportfolio; Fbetacontrols; :netret]), focalfields)

    #run the regression for each fundid
    #we are primarily interested in the absolute vlaue of the weights
    #the corrodor version creates a buffer to guard against measurement error in the controls
    weight = combine(groupby(cleandf, :fundid), (:) |> AsTable =>
      ((t)->betaw(t,Frhs,Fportfolioweights)) => AsTable)

    transform!(weight,
      Fportfolioweights .=>
        (v-> abs.(v)) .=> Fportfolioabsweights,
      Fportfolioweights .=>
        (v->(x->ifelse(x<betacorridorradius,0.0, x)).(abs.(v))) .=> Fportfolioabscorweights,
      )

    weights = innerjoin(weights, weight[:, [:fundid; newweightfields]], on=[:fundid])
    append!(Fweights, newweightfields)
  end

  #merge in the new weight fields
  df = leftjoin(df, weights, on=:fundid, validate=(false, true))
  if !issorted(df, [:fundid, :date])
    sort!(df, [:fundid, :date])
  end
  select!(df, Not([:some1s]))

  for Fweight ∈ Fweights
    #=winsorizewithin!(df, Fsource=Fweight, prop=PARAM[:fundwinsorizebeta], Fgroup=:date,
      twosided=true, Fnew=Symbol(:temp, Fweight))=#
    df[!, Symbol(:temp, Fweight)] = winsorizequantile(df[!,Fweight],
      PARAM[:fundwinsorizebeta]; twosided=true)
  end

  if PARAM[:testfundbeta]
    testmonth = PARAM[:fundbetatestmonth]
    testview::SubDataFrame = view(df, (month.(df.date) .== testmonth.month) .&
      (year.(df.date) .== testmonth.year), :)
    testview[1:(min(20_000, nrow(testview))),:] |> CSV.write(
      "$(PARAM[:testpath])\\$(PARAM[:crspdatalabel])_$(Tuple(testmonth))_$testlabel.csv")

    testfundid = PARAM[:fundbetatestfundid][testlabel |> Symbol]
    testview = view(df, df.fundid .== testfundid, :)
    testview[1:(min(20_000, nrow(testview))),:] |> CSV.write(
      "$(PARAM[:testpath])\\$(PARAM[:crspdatalabel])_$(testfundid)_$testlabel.csv")
  end

  #@info propertynames(df)
  for Fweight ∈ Fweights
    select!(df, Not([Fweight]))

    rename!(df, Dict(Symbol(:temp, Fweight)=>Fweight))
  end

  return Fweights, df
end

hfrbetaweights(hfr, returns, ms::MeasureSpec) = betaweights(
    hfr, returns, ms, testlabel="hfr", datefrequency=:month)

function prephf(returns::AbstractDataFrame, ms::MeasureSpec;
  hfrnamealive::String = PARAM[:hfrnamealive],
  hfrnamedead::String = PARAM[:hfrnamedead],
  hfrpath::String = PARAM[:hfrpath],
  minfundmonths::Int = PARAM[:fundminfundmonths],
  dropfinalmonths::Int = PARAM[:hfrdropfinalmonths],
  lpfilterperiods::Vector{<:Union{Day, Nothing}} = PARAM[:analysislpfilterperiods],
  incsvstream::Function = IN_CSV_STREAM,
  csvextension::String = CSV_EXTENSION)::DataFrame

  hfralive::DataFrame = incsvstream("$hfrpath\\$hfrnamealive.$csvextension") |> CSV.File |> DataFrame
  hfralive.alive = trues(size(hfralive,1))
  hfrdead::DataFrame = incsvstream("$hfrpath\\$hfrnamedead.$csvextension") |> CSV.File |> DataFrame
  hfrdead.alive = falses(size(hfrdead,1))

  hfr = vcat(hfralive, hfrdead)

  normalizename(s::String) = replace(s, "_"=>"") |> lowercase
  rename!(hfr, normalizename.(names(hfr)))

  #need to parse the dates
  for datefield ∈ [:date, :inception, :fundassetsasof, :firmassetsasof]
    hfr[!, datefield] = (d->parsehfr(Date, d)).(hfr[!, datefield])
  end

  sort!(hfr, [:fundid, :date,])
  hfr.firstdomdate = hfr.date .|> firstdayofmonth
  @assert sum(nonunique(hfr, [:fundid, :firstdomdate]))==0

  #classify the strategies- first read in a spreadsheet of the classifications
  stratcodes = CSV.File("$hfrpath\\classifications.csv") |>  DataFrame
  rename!(stratcodes, normalizename.(names(stratcodes)))
  stratcodes=stratcodes[:, Not([:mainstrategy, :substrategy])]

  #conform index classifications
  hfr.hfri = hfr.inhfri .=== "Yes"
  hfr.hfrx = hfr.inhfrx .=== "Yes"

  #then join the classifications with the funds
  hfr = innerjoin(hfr, stratcodes,on=[:strategycode, :substrategycode])
  if !issorted(hfr, [:fundid, :date])
    sort!(hfr, [:fundid, :date,])
  end
  @assert size(hfr,1) == size(hfrdead,1) + size(hfralive,1) #sanity check

  #keep only the relevant dates- is this the right approach? should I take the max range?
  #keep only dates within the return range
  #also drop the final month(s) due to incomplete data
  mindate=minimum(returns.date)|>firstdayofmonth
  maxdate = min(maximum(returns.date) |> lastdayofmonth,
    maximum(hfr.date) - Month(dropfinalmonths) |> lastdayofmonth)
  hfr = hfr[(mindate .< hfr.date) .& (hfr.date .< maxdate), :]
  hfr.ret = hfr.performance ./ 100
  select!(hfr, Not(:performance))

  #now compute AUMs by strategy bunches
  hfr = hfr[hfr.fundassetsdenomin .== "USD",:]
  hfr.assets ./= 1_000
  rename!(hfr, :assets=>:aum) #what about nav?

  #acquire the index of strategy returns
  hfr = hfr[completecases(hfr, [:aum, :ret]), :]
  transform!(groupby(hfr, :fundid), nrow => :fundmonths)
  hfr = hfr[hfr.fundmonths .≥ minfundmonths, :]

  println("Attempting to weight hfr fund aum by returns...")
  Fhfweights, hfr = hfrbetaweights(hfr, returns, ms)
  println("weighting successful.")

  #sums values in an array if the conditional is true and nothing is missing
  dotifs(v1,v2 = 1.0, v3 = 1.0) = sum((v1 .* v2 .* v3) .|> (x)->coalesce(x, 0.0))

  #now group the assets
  shfs = groupby(hfr, :date)
  hf = combine(shfs ,
    :aum => dotifs => :aum,
    [:aum, :equityonly] => dotifs => :A_aumequityonly,
    [:aum, :equity] => dotifs => :A_aumequity,
    [:aum, :quant] => dotifs => :A_aumquant,
    [:aum, :hfri] => dotifs => :A_aumhfri,
    [:aum, :hfrx] => dotifs => :A_aumhfrx,
  )
  sort!(hf, :date)

  #group the regression weights
  for Fhfweight ∈ Fhfweights
    hfweightdf = combine(shfs,
      [:aum, Fhfweight] => dotifs => Symbol(:A_,Fhfweight),
      [:aum, Fhfweight, :equity] => dotifs => Symbol(:A_aumequityX,Fhfweight),
      [:aum, Fhfweight, :equityonly] => dotifs => Symbol(:A_aumequityonlyX,Fhfweight),
    )
    hf = innerjoin(hf, hfweightdf, on=:date, validate=(true,true))
  end
  sort!(hf, :date)

  #create the log growwth columns
  logmaybe(::Missing) = missing
  logmaybe(x) = x ≤ 0 ? missing : log(x)
  for acol ∈ findcols(hf, :asset)
    @assert occursin(r"^A", string(acol))
    hf[!, Symbol(:lG_,acol)] = [missing; logmaybe.(hf[2:end, acol] ./ hf[1:(end-1), acol])]
    #hf[!, Symbol(:G_,acol)] = [missing; hf[2:end, acol] ./ hf[1:(end-1), acol] .- 1]
  end
  #average all cols via the LP filter
  hf = lpfilter(hf, focalcols=setdiff(propertynames(hf), [:date]); lpfilterperiods)

  #create normalized versions NOTE- I don't think this is the right approach- disabled
  #create normalized versions
  #=focalcols = setdiff(propertynames(hf), [:date, :aum])
  for f ∈ focalcols
    transform!(hf, [f, :aum] => ((x,y)->x ./ y) =>Symbol(:N, f))
  end

  for per ∈ lpfilterperiods
    (per === nothing) && continue

    #the suffix is how we will identify the lp columns during analysis
    hf = lpfilter(hf, lpfilterperiods=[per], focalcols=[focalcols; :aum])
    lpcols = focalcols .|> (f)->Symbol(f,lpsuffix(per))
    aumcol = Symbol(:aum, lpsuffix(per))
    lpcols = (s->Symbol(s, lpsuffix(per))).(focalcols)
    for lpcol ∈ lpcols
      transform!(hf, [lpcol, aumcol] => ((x,y)->x ./ y) =>Symbol(:N, lpcol),)
    end
  end=#

  return hf
end

parsehfr(::Type{Date}, s::String, hfrdateformat::DateFormat=dateformat"yyyymmdd"
  ) = Dates.Date(s, hfrdateformat)
parsehfr(::Type{<:Any}, ::Missing) = missing
parsehfr(::Type{Date}, i::Int, hfrdateformat::DateFormat=dateformat"yyyymmdd") = parsehfr(
  Date, "$i", hfrdateformat)

############################MF

mfbetaweights(mf, returns, ms::MeasureSpec;
  datefrequency=:month) = betaweights(
    mf, returns, ms;  testlabel="mf", datefrequency)

function prepmf(returns::AbstractDataFrame, ms::MeasureSpec;
  mfdatefrequency::Symbol=PARAM[:mfdatefrequency],
  mfname::String = PARAM[:mfname],
  mfnamedesc::String = PARAM[:mfnamedesc],
  mfpath::String = PARAM[:mfpath],
  minfundmonths::Int = PARAM[:fundminfundmonths],
  minfunds::Int = PARAM[:fundminfunds],
  lpfilterperiods::Vector{<:Union{Day, Nothing}} = PARAM[:analysislpfilterperiods],
  incsvstream::Function = IN_CSV_STREAM,
  csvextension::String = CSV_EXTENSION,
  dtformat::DateFormat = PARAM[:wrdsdateformat])

  #scrubs the names and dates
  function initialclean(df::DataFrame; #=startdate::Date = PARAM[:mfstartdate]=#)
    rename!(df, (cleanname).(names(df)))
    df.date = (i::Int->Date("$i", dtformat)).(df.caldt)
    #df.year = (d->year(d)).(df.date)
    transform!(df, :crspfundno => ByRow((s)->Symbol(:mf, s)) => :fundid)
    select!(df, Not([:crspfundno, :caldt]))

    return df
  end

  #start by working with the description file
  mfdesc::DataFrame = CSV.File("$mfpath\\$mfnamedesc.csv") |> DataFrame
  mfdesc = initialclean(mfdesc)
  println(describe(mfdesc))
  rename!(mfdesc, :crspobjcd =>:obj)

  #tehse are the classificaiton codes for selection
  mfdesc = mfdesc[mfdesc.obj .!== missing, :]
  mfdesc = mfdesc[length.(mfdesc.obj) .≥ 1,:]
  (!issorted(mfdesc, [:fundid, :date])) && sort!(mfdesc, [:fundid, :date])
  select!(mfdesc, Not([:date]))
  mfdesc = combine(last, groupby(mfdesc, :fundid), renamecols=false)

  transform!(mfdesc,
    :obj => ByRow(obj->obj[1] == 'E' || obj[1]=='M' || obj[1]=='O') => :equity,
    :obj => ByRow(obj->obj[1] == 'E') => :equityonly,
    :obj => ByRow((obj->(length(obj) > 1) && (obj[1] == 'E') && (obj[2] == 'D'))) => :domesticeo,
    )

  mfdesc.lipperclassname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperclassname)
  mfdesc.lipperobjname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperobjname)
  #=mfdesc.equity = (mfdesc.obj[1] .== 'E') .| (mfdesc.obj[1] .== 'M') .| (mfdesc.obj[1] .== 'O')
  mfdesc.equityonly = mfdesc.obj[1] .== 'E'
  mfdesc.domesticeo = .(mfdesc.obj)=#

  #mfdesc = combine(groupby(mfdesc, [:fundid, :year]), propertynames(mfdesc) .=>
  #  (v)->v[end], renamecols=false)
  #dups = mfdesc[nonunique(mfdesc[:, [:fundid, :year]]),:]
  #dups.id = 1:nrow(dups)
  ##println(innerjoin(mfdesc, dups[:, [:fundid, :year, :id]], on=[:fundid, :year]))
  #@assert sum(nonunique(mfdesc[:, [:fundid, :year]])) == 0

  #=mfdesc.lipperclassname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperclassname)
  mfdesc.lipperobjname .= (s::MString->
    ismissing(s) ? missing : lowercase(s)).(mfdesc.lipperobjname)

  #note- the classifications are by hand- the below is ONLY for support of the by-hand classifications
  unique(mfdesc[:, [:lipperclassname, :lipperobjname]]) |> CSV.write("mfclassificationsraw.csv")

  #only interested in the most recent, non-missing row for each fund
  mfdesc = mfdesc[(mfdesc.lipperclassname .!== missing) .| (mfdesc.lipperobjname .!== missing), :]
  mfdesc = combine(last, groupby(mfdesc, :fundid), renamecols=false)
  @assert sum(nonunique(mfdesc[:, [:fundid]])) == 0

  #get the classifications from a seperate file
  stratcodes = CSV.File("$mfpath\\mfclassifications.csv") |>  DataFrame
  select!(stratcodes, [:lipperclassname, :lipperobjname, :equity,:equityonly, :domesticeo])
  oldnrows = nrow(mfdesc)
  mfdesc = innerjoin(mfdesc, stratcodes, on=[:lipperclassname, :lipperobjname], matchmissing=:equal)=#

  #@assert nrow(mfdesc) == oldnrows
  @assert sum(nonunique(mfdesc[:, [:fundid]])) == 0

  #clean the data
  mf::DataFrame = incsvstream("$mfpath\\$mfname.$csvextension") |> CSV.File |> DataFrame
  mf = initialclean(mf)

  sort!(mf, [:fundid, :date])
  @assert sum(nonunique(mf[!, [:fundid, :date]])) == 0

  #parse relevant fields to a number- letter codes generally indicate missing
  for (Fold, Fnew) ∈ [:mret=>:ret, :mtna=>:aum, :mnav=>:nav]
    mf[!, Fnew] = (s->parsemf(Float64, s)).(mf[!, Fold])
  end

  #invalid values also generally represent missing
  mf.ret .= mf.ret .|> (x)-> (x ≡ missing ) || (x < -1.0) ? missing : x
  mf.aum .= mf.aum .|> (x)-> (x ≡ missing ) || (x < 0.0) ? missing : x
  mf.aum ./= 1000

  mf = dropmissing!(mf, :ret)

  #implmeent the reversal check of PST2015
  lagwithin2!(mf, [:aum], :fundid, date=:date)
  leadwithin2!(mf, [:aum], :fundid, date=:date)
  mf.Gaum = (mf.aum .- mf.Laum) ./ mf.Laum
  mf.rev = (mf.Naum .- mf.aum) ./ (mf.aum .- mf.Laum)
  transform!(groupby(mf, :fundid), [:Gaum, :rev, :Laum, :aum] => ByRow(
    (Gaum, rev, Laum, aum)->ifelse(
      (Gaum !== missing) && (rev !== missing) &&
      (abs(Gaum) > 0.5) && (-1.25 ≤ rev ≤ -0.75) && (Laum > 0.01), missing, aum))
    => :adjaum)

  @info "Dropped $(sum(mf.aum .!== mf.adjaum)) month-fund rows due to rapid  aum reversals"
  select!(mf, Not([:Gaum, :rev, :Laum, :aum, :Naum]))
  rename!(mf, :adjaum => :aum)

  mf = patchmfmonthlyreturngaps(mf)

  #no point in keeping a row if the assets field is missing
  mf = dropmissing!(mf, :aum)
  transform!(groupby(mf, :fundid), nrow => :fundmonths)
  mf = mf[mf.fundmonths .≥ minfundmonths, :]

  #trim down to the desired range with a buffer
  startdate = minimum(returns.date) - Year(1)
  enddate =  maximum(returns.date) + Year(1)
  mf = mf[(mf.date .≥ startdate) .&  (mf.date .≤ enddate), :]

  #if we want to patch in daily or weekly return data, this is where we do that
  select!(mf, [:fundid, :date, :aum, :ret])
  mf = patchmfreturns(mf, mfdatefrequency)

  #patch in the description info
  mf = leftjoin(mf, mfdesc[:, [:fundid, :equity, :equityonly, :domesticeo,
    :obj, :lipperclassname, :lipperobjname]], on=[:fundid], validate=(false, true))
  if !issorted(mf, [:fundid, :date])
    sort!(mf, [:fundid, :date])
  end

  transform!(groupby(mf, :date), nrow => :numfunds)
  mf = mf[mf.numfunds .≥  minfunds,:]

  #merge in the beta-based fund weights
  println("Attempting to weight mf fund aum by returns...")
  Fmfweights, mf = mfbetaweights(mf, returns, ms; datefrequency=mfdatefrequency)
  println("...weighting successful.")

  #sums values in an array if the conditional is true and nothing is missing
  zeromissings(x) =  coalesce(x, 0.0)
  dotifs(v1,v2 = 1.0, v3 = 1.0) = sum((v1 .* v2 .* v3) .|> zeromissings)
  desccount(aum,beta = 1.0, cond = 1.0) = sum((aum .* cond .* (beta .!==missing)) .|> zeromissings .|> (x)->x>0.0)
  descmin(beta,cond=true) = minimum((beta .* cond) |> skipmissing)
  descmax(beta,cond=true) = maximum((beta .* cond) |> skipmissing)
  descstd(beta,cond=true) = std((beta .* cond) |> skipmissing)
  descmedianabs(beta,cond=true) = median((beta .* cond) .|> abs |> skipmissing)
  descmeanabs(beta,cond=true) = mean((beta .* cond) .|> abs |> skipmissing)

  combine(groupby(view(mf,
    mf.date .== maximum(mf.date[(mf.date .≥ Date(2019,12,1)) .& (mf.date .≤ Date(2019,12,31))]),:),
      [:lipperclassname, :lipperobjname]),
      Fmfweights .=> ((v)->mean(skipmissing(abs.(v)))) .=> (s->Symbol(:meanabs_,s)).(Fmfweights),
        :aum => ((v)->sum(skipmissing(v))) => :aum,
      zip([:aum for i ∈ 1:length(Fmfweights)], Fmfweights) .|> collect .=> dotifs .=> (
        s->Symbol(:A_,s)).(Fmfweights),
      :equity => last => :equity,
      :equityonly => last => :equityonly,
      :domesticeo => last => :domesticeo) |> CSV.write(
        "$(PARAM[:testpath])\\mf_asset aggregates 20191227.csv")

  #now group the assets
  smfs = groupby(mf, :date)
  funds = combine(smfs,
    :aum => dotifs => :aum,
    [:aum, :equity] => dotifs => :aumequity,
    [:aum, :equityonly] => dotifs => :aumequityonly,
    [:aum, :domesticeo] => dotifs => :aumdomesticeo,
  )
  if !issorted(funds, :date)
    sort!(funds, :date)
  end

  #group the regression weights
  for Fmfweight ∈ Fmfweights
    oldnames = propertynames(funds)
    mfweightdf = combine(smfs,
      [:aum, Fmfweight] => dotifs => Symbol(:A_, Fmfweight),
      [:aum, Fmfweight, :equity] => dotifs => Symbol(:A_aumequityX,Fmfweight),
      [:aum, Fmfweight, :equityonly] => dotifs => Symbol(:A_aumequityonlyX,Fmfweight),
      [:aum, Fmfweight, :domesticeo] => dotifs => Symbol(:A_aumdomesticeoX,Fmfweight),

      [:aum, Fmfweight] => desccount => Symbol(:descn_, Fmfweight),
      [Fmfweight] => descmedianabs => Symbol(:descmedabs_, Fmfweight),
      [Fmfweight] => descmeanabs => Symbol(:descmeanabs_, Fmfweight),
      [Fmfweight] => descstd => Symbol(:descstd_, Fmfweight),
      [Fmfweight] => descmin => Symbol(:descmin_, Fmfweight),
      [Fmfweight] => descmax => Symbol(:descmax_, Fmfweight),

      [:aum, Fmfweight, :equity] => desccount => Symbol(:descn_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmedianabs => Symbol(:descmedabs_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmeanabs => Symbol(:descmeanabs_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descstd => Symbol(:descstd_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmin => Symbol(:descmin_aumequityX,Fmfweight),
      [Fmfweight, :equity] => descmax => Symbol(:descmax_aumequityX,Fmfweight),

      [:aum, Fmfweight, :equityonly] => desccount => Symbol(:descn_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmedianabs => Symbol(:descmedabs_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmeanabs => Symbol(:descmeanabs_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descstd => Symbol(:descstd_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmin => Symbol(:descmin_aumequityonlyX,Fmfweight),
      [Fmfweight, :equityonly] => descmax => Symbol(:descmax_aumequityonlyX,Fmfweight),

      [:aum, Fmfweight, :domesticeo] => desccount => Symbol(:descn_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmedianabs => Symbol(:descmedabs_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmeanabs => Symbol(:descmeanabs_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descstd => Symbol(:descstd_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmin => Symbol(:descmin_aumdomesticeoX,Fmfweight),
      [Fmfweight, :domesticeo] => descmax => Symbol(:descmax_aumdomesticeoX,Fmfweight),

    )
    funds = innerjoin(funds, mfweightdf, on=:date, validate=(true,true))
  end
  sort!(funds, :date)

  logmaybe(::Missing) = missing
  logmaybe(x) = x ≤ 0 ? missing : log(x)
  lpfiltercols = findcols(funds, :asset)
  for acol ∈ findcols(funds, :asset)
    @assert occursin(r"^A", string(acol))

    push!(lpfiltercols, Symbol(:lG_,acol))
    funds[!, lpfiltercols[end]] = [missing; logmaybe.(funds[2:end, acol] ./ funds[1:(end-1), acol])]
    #funds[!, Symbol(:G_,acol)] = [missing; funds[2:end, acol] ./ funds[1:(end-1), acol] .- 1]
  end

  funds = lpfilter(funds, focalcols=lpfiltercols; lpfilterperiods)

  return funds
end

function patchmfmonthlyreturngaps(mf::AbstractDataFrame,
    maxpatchlength=PARAM[:mftimetostale])

  #sanity checks to make sure everything is as expected
  @assert issorted(mf, [:fundid, :date])
  @assert sum(completecases(mf, [:ret])) == nrow(mf)

  #temporary helper fields
  mf.ret1 = mf.ret .+ 1
  mf.maxvaliddate = (mf.date .+ maxpatchlength) .|> lastdayofmonth

  function patchwithret(dates, maxdate, aum, ret)

    sum(aum .≡ missing) == length(aum) && return aum
    T = searchsortedfirst(dates, maximum(dates[aum .!== missing]))

    maxvaliddate = Date(1900,1,1)
    for t ∈ 1:T

      #if the aum is not missing, then record the max valid date
      if aum[t] !== missing
        maxvaliddate = maxdate[t]
        continue
      end

      if dates[t] ≤ maxvaliddate
        aum[t] = (1+ret[t]) * aum[t-1]
      end
    end

    return aum
  end

  #correct the dates
  mf.aumcorrected = mf.aum |> deepcopy
  transform!(groupby(mf, :fundid),
    [:date, :maxvaliddate, :aumcorrected, :ret] => patchwithret => :aumcorrected)
  mf.corrected = (mf.aum .≡ missing ) .& (mf.aumcorrected .!== missing)

  #check
  uncorrectedcomplete = view(mf, completecases(mf, :aum), :)
  @assert (uncorrectedcomplete.aum .== uncorrectedcomplete.aumcorrected) |> all

  testfunds = mf[mf.corrected, :fundid] |> unique |> (v)->v[1:3]
  mf[
    (mf.fundid .== testfunds[1] ) .| (mf.fundid .== testfunds[2] ) .| (mf.fundid .== testfunds[3])
    , :] |> CSV.write("$(PARAM[:testpath])\\mf_monthlyaumpatch.csv")

  mf.aum = mf.aumcorrected |> deepcopy
  select!(mf, Not([:aumcorrected, :ret1, :maxvaliddate]))
  return mf
end

function patchmfreturns(mf::DataFrame, mfdatefrequency::Symbol;
    mfdailyname::String = PARAM[:mfdailyname],
    mfpath::String = PARAM[:mfpath],
    incsvstream::Function = IN_CSV_STREAM,
    csvextension::String = CSV_EXTENSION,
    dtformat = PARAM[:wrdsdateformat],
    interpolate::Bool = PARAM[:mfdailyinterpolate],
  )

  if mfdatefrequency ≡ :month
    dropmissing!(mf, [:ret])
    winsorizewithin!(mf, Fsource=:ret, prop=PARAM[:mfqwinsorize], Fgroup=:date,
      twosided=true, Fnew=:WXret)
    select!(mf, Not([:ret]))
    rename!(mf, :WXret=>:ret)
    return mf #nothing to patch in this case
  end

  #we may be able to drop this assertion if it is checked later
  @assert mfdatefrequency ∈ [:day, :week]
  oldmonthlyaum = mf[!, [:date, :fundid, :aum]]
  rename!(oldmonthlyaum, :aum=>:aumoldmonthly)

  #load and clean daily data
  mfd::DataFrame = incsvstream(
    "$mfpath\\$mfdailyname.$csvextension") |> CSV.File |> DataFrame
  rename!(mfd, (cleanname).(names(mfd)))
  mfd.date = (i::Int->Date("$i", dtformat)).(mfd.caldt)
  mfd.yearmonth = mfd.date .|> (d->year(d) + month(d)/100)
  #df.year = (d->year(d)).(df.date)
  transform!(mfd, :crspfundno => ByRow((s)->Symbol(:mf, s)) => :fundid)
  mfd.ret = mfd.dret .|> (r-> parsemf(Float64, r))
  dropmissing!(mfd, [:fundid, :date, :ret])

  #quick and easy performance enhancement based on the time period covered
  mfd = mfd[
    (mfd.date .≥ firstdayofmonth(minimum(mf.date))) .&
    (mfd.date .≤ lastdayofmonth(maximum(mf.date))),:]

  #prep mf
  @assert issorted(mf, [:fundid, :date])

  #we want the beginning of period AUM, not the end of period
  mfN = deepcopy(mf)
  mfN.Nyearmonth = mf.date .+ Month(1) .|> (d->year(d) + month(d)/100)
  mfN = select(mfN, [:fundid, :Nyearmonth, :aum])

  #we want to use the eom aum onthe last day of the month,
  #and then use returns to interpolate at other parts of the month
  mf.yearmonth = mf.date .|> (d->year(d) + month(d)/100)
  mf = select!(mf, [:fundid, :yearmonth, :aum])
  rename!(mf, :aum=>:aumeom)
  mfN = leftjoin(mfN, mf, on=[:fundid, :Nyearmonth => :yearmonth])

  #NOTE: the below restriction ensures that only the AUM is shifted as desired
  #WARNING: read above before adding columns to below

  select!(mfd, [:fundid, :date, :yearmonth, :ret])
  mf = innerjoin(mfN, mfd, on=[:fundid, :Nyearmonth => :yearmonth], validate=(true, false))

  #validate df and make sure everything is as expected
  @assert nonunique(mf, [:fundid, :date]) |> !all

  mf = sort(mf, [:fundid, :date])
  @assert issorted(mf, [:fundid, :Nyearmonth])
  dropmissing!(mf, [:ret, :aum])

  #winsorze for data quality
  winsorizewithin!(mf, Fsource=:ret, prop=PARAM[:mfqwinsorize], Fgroup=:Nyearmonth,
    twosided=true, Fnew=:WXret)
  select!(mf, Not([:ret]))
  rename!(mf, :WXret=>:ret)
  println(describe(mf))

  mf.aumbom = mf.aum |> deepcopy
  mfbyfund = groupby(mf, [:fundid, :Nyearmonth])

  #the interpolation routine backs out net deposits, then discounts
  #them back to the beginning of period. The discounted deposits are then added
  #pro-rata to the base beginning of period amount and discoutned back up to the
  #focal day. In equations,
  # Let CUMR(t) be the cumulative return for the month on day t. Then
  # totaldepbom = (AUM(T)-AUM(0)*(1+CUMR(T))/(1+CUMR(T))
  # AUM(t)=(AUM(0)+totaldepbom*t/T)*CUMR(t)
  function interpolatereturns(
      ret::AbstractVector,  aum::AbstractVector, aumeom::AbstractVector) where T

    cumret = cumprod(ret .+ 1.0) .- 1.0
    netgains = aumeom[end] - (1.0 + cumret[end]) * aum[1]
    netgainsbom = coalesce(netgains / (1.0+cumret[end]), 0.0)
    N::Int = length(ret)

    interpolated = [(t/N*netgainsbom+aum[1])*(1+cumr) for (t,cumr) ∈ enumerate(cumret)]

    #maybe disable the assertion if it slows things down
    @assert (interpolated[end] + 1.0 ≈ coalesce(aumeom[end], aum[1]*(1+cumret[end])) + 1.0) "
      aumeom[end]=$(aumeom[end])
      but interpolated[end]=$(interpolated[end])"
    return interpolated
  end

  @assert issorted(mf, [:fundid, :date])
  if interpolate

    #this will be correct for all but the last day of the month
    transform!(mfbyfund, [:ret, :aum, :aumeom] => interpolatereturns => :aum)
  else
    transform!(mfbyfund,
      [:ret, :aum] => ((ret,aum)->cumprod(ret .+ 1.0) .* aum) => :aum)
    #now patch in the eom aum at the end of the month
    transform!(mfbyfund,
      [:aum, :aumeom] => ((aum, aumeom)->aumeom[end] ≡ missing ?
        aum : [aum[1:(end-1)]; aumeom[end]]) => :aum)
  end

  if mfdatefrequency ≡ :week
    mf.weekid = assignweekid(mf.date)
    mfw = combine(groupby(mf, [:fundid, :weekid], sort=true),
      :ret => (r -> prod(1.0 .+ r) - 1.0) => :ret,
      :date => last => :date,
      :aum => last => :aum,
      )

    #we need to conform the weekly dates
    conformeddates = combine(groupby(mfw[!, [:weekid, :date]], :weekid),
      :date => maximum => :date)
    select!(mfw, Not([:date]))
    oldrows = nrow(mfw)
    mfw = innerjoin(mfw, conformeddates, on=:weekid, validate=(false,true))
    @assert oldrows == nrow(mfw)
    if !issorted(mfw, [:fundid, :weekid])
      sort!(mfw, [:fundid, :weekid])
    end

    #writes the output to a CSV for inspection
    if PARAM[:testfunds]
      testfundid = mfw[findmax(mfw.aum)[2], :fundid]
      testmfw = mfw[mfw.fundid .== testfundid,:]
      rename!(testmfw, Dict(:ret=>:retw, :aum=>:aumw, :date=>:datew))
      testmf = innerjoin(testmfw, mf, on=[:fundid, :weekid])

      testmf.ym = testmf.date .|> (d->year(d) + month(d)/100)
      oldmonthlyaum.ym = oldmonthlyaum.date .|> (d->year(d) + month(d)/100)
      rename!(oldmonthlyaum, :date=>:dateoldaum)
      testmf = leftjoin(testmf, oldmonthlyaum, on=[:fundid, :ym])
      sort!(testmf, [:fundid, :date])
      testmf |> CSV.write(
        "$(PARAM[:testpath])\\$(PARAM[:crsptypesuffix])dailyforweeklymf_$(mfdatefrequency).csv")

    end
    mf = mfw
  else
    @assert mfdatefrequency ≡ :day
  end

  #throw("got through it- check now!")

  return mf

end

parsemf(::Type{Date}, s::String, mfdateformat::DateFormat=dateformat"yyyymmdd"
  ) = Dates.Date(s, mfdateformat)
parsemf(::Type{<:Any}, ::Missing) = missing
parsemf(::Type{Date}, i::Int, mfdateformat::DateFormat=dateformat"yyyymmdd") = parsemf(
  Date, "$i", mfdateformat)

#parse to a number if possible
parsemf(T::Type{<:Real}, s::String) = something(tryparse(T, s), missing)
parsemf(::Type{T}, x::T) where T = x


module Enzyme

export autodiff, autodiff_deferred
export Const, Active, Duplicated, DuplicatedNoNeed

"""
    abstract type Annotation{T}

Abstract type for [`autodiff`](@ref) function argument wrappers like
[`Const`](@ref), [`Active`](@ref) and [`Duplicated`](@ref).
"""
abstract type Annotation{T} end

"""
    struct Const{T} <: Annotation{T}

Constructor: `Const(x)`

Mark a function argument `x` of [`autodiff`](@ref) as constant,
Enzyme will not auto-differentiate in respect `Const` arguments.
"""
struct Const{T} <: Annotation{T}
    val::T
end

# To deal with Const(Int) and prevent it to go to `Const{DataType}(T)`
Const(::Type{T}) where T = Const{Type{T}}(T)

"""
    struct Active{T} <: Annotation{T}

Constructor: `Active(x)`

Mark a function argument `x` of [`autodiff`](@ref) as active,
Enzyme will auto-differentiate in respect `Active` arguments.

!!! note

    Enzyme gradients with respect to integer values are zero.
    [`Active`](@ref) will automatically convert plain integers to floating
    point values, but cannot do so for integer values in tuples and structs.
"""
struct Active{T} <: Annotation{T}
    val::T
end

Active(i::Integer) = Active(float(i))

"""
    struct Duplicated{T} <: Annotation{T}

Constructor: `Duplicated(x, ∂f_∂x)`

Mark a function argument `x` of [`autodiff`](@ref) as duplicated, Enzyme will
auto-differentiate in respect to such arguments, with `dx` acting as an
accumulator for gradients (so ``\\partial f / \\partial x`` will be *added to*)
`∂f_∂x`.
"""
struct Duplicated{T} <: Annotation{T}
    val::T
    dval::T
end

struct DuplicatedNoNeed{T} <: Annotation{T}
    val::T
    dval::T
end

Base.eltype(::Type{<:Annotation{T}}) where T = T

function guess_activity(T)
    if T <: AbstractFloat || T <: Complex{<:AbstractFloat}
        return Active{T}
    elseif T <: AbstractArray
        return Duplicated{T}
    else
        return Const{T}
    end
end

import LLVM

include("api.jl")
include("logic.jl")
include("typeanalysis.jl")
include("typetree.jl")
include("utils.jl")
include("compiler.jl")

# @inline annotate() = ()
# @inline annotate(arg::A, args::Vararg{Any, N}) where {A<:Annotation, N} = (arg, annotate(args...)...)
# @inline annotate(arg, args::Vararg{Any, N}) where N = (Const(arg), annotate(args...)...)

@inline function annotate(args::Vararg{Any, N}) where N
    ntuple(Val(N)) do i
        Base.@_inline_meta
        arg = @inbounds args[i]
        if arg isa Annotation
            return arg
        else
            return Const(arg)
        end
    end
end

prepare_cc() = ()
prepare_cc(arg::Duplicated, args...) = (arg.val, arg.dval, prepare_cc(args...)...)
prepare_cc(arg::DuplicatedNoNeed, args...) = (arg.val, arg.dval, prepare_cc(args...)...)
prepare_cc(arg::Annotation, args...) = (arg.val, prepare_cc(args...)...)

"""
    autodiff(f, Activity, args...)

Auto-differentiate function `f` at arguments `args`.

Limitations:

* `f` may only return a `Real` (of a built-in/primitive type) or `nothing`,
  not an array, struct, `BigFloat`, etc. To handle vector-valued return
  types, use a mutating `f!` that returns `nothing` and stores it's return
  value in one of the arguments, which must be wrapped in a
  [`Duplicated`](@ref).

* `f` may not allocate memory, this restriction is likely to be removed in
  future versions. Technically it can currently allocate memory directly, but not in a function called by `f`.

`args` may be numbers, arrays, structs of numbers, structs of arrays and so
on. Enzyme will only differentiate in respect to arguments that are wrapped
in an [`Active`](@ref) (for immutable arguments like primitive types and
structs thereof) or [`Duplicated`](@ref) (for mutable arguments like arrays,
`Ref`s and structs thereof). Non-annotated arguments will automatically be
treated as [`Const`](@ref).

`Activity` is the Activity of the return value, it may be `Const` or `Active`.

Example:

```jldoctest
using Enzyme

a = 4.2
b = [2.2, 3.3]; ∂f_∂b = zero(b)
c = 55; d = 9

f(a, b, c, d) = a * √(b[1]^2 + b[2]^2) + c^2 * d^2
∂f_∂a, ∂f_∂d = autodiff(f, Active, Active(a), Duplicated(b, ∂f_∂b), c, Active(d))

# output

(3.966106403010388, 54450.0)
```

here, `autodiff` returns a tuple
``(\\partial f/\\partial a, \\partial f/\\partial d)``,
while ``\\partial f/\\partial b`` will be *added to* `∂f_∂b` (but not returned).
`c` will be treated as `Const(c)`.

!!! note

    Enzyme gradients with respect to integer values are zero.
    [`Active`](@ref) will automatically convert plain integers to floating
    point values, but cannot do so for integer values in tuples and structs.
"""
@inline function autodiff(f::F, ::Type{A}, args...) where {F, A<:Annotation}
    args′  = annotate(args...)
    tt′    = Tuple{map(Core.Typeof, args′)...}
    if A <: Active
        tt    = Tuple{map(T->eltype(Core.Typeof(T)), args′)...}
        rt = Core.Compiler.return_type(f, tt)
        if !allocatedinline(rt)
            forward, adjoint = Enzyme.Compiler.thunk(f, #=df=#nothing, Duplicated{rt}, tt′, #=Split=# Val(true))
            res = forward(args′...)
            tape = res[1]
            if res[3] isa Base.RefValue
                res[3][] += one(eltype(typeof(res[3])))
            else
                res[3] += one(eltype(typeof(res[3])))
            end
            return adjoint(args′..., tape)
        end
    elseif A <: Duplicated
        throw(ErrorException("Duplicated Returns not yet handled"))
    end
    thunk = Enzyme.Compiler.thunk(f, #=df=#nothing, A, tt′, #=Split=# Val(false))
    rt = eltype(Compiler.return_type(thunk))
    if A <: Active
        args′ = (args′..., one(rt))
    end
    thunk(args′...)
end

@inline function autodiff(dupf::Duplicated{F}, ::Type{A}, args...) where {F, A<:Annotation}
    args′  = annotate(args...)
    tt′    = Tuple{map(Core.Typeof, args′)...}
    thunk = Enzyme.Compiler.thunk(#=f=#dupf.val, #=df=#dupf.dval, A, tt′, #=Split=# Val(false))
    if A <: Active
        rt = eltype(Compiler.return_type(thunk))
        args′ = (args′..., one(rt))
    end
    thunk(args′...)
end

"""
    autodiff(f, args...)

Like [`autodiff`](@ref) but will try to guess the activity of the return value.
"""
@inline function autodiff(f::F, args...) where {F}
    args′ = annotate(args...)
    tt′   = Tuple{map(Core.Typeof, args′)...}
    tt    = Tuple{map(T->eltype(Core.Typeof(T)), args′)...}
    rt    = Core.Compiler.return_type(f, tt)
    A     = guess_activity(rt)
    autodiff(f, A, args′...)
end

@inline function autodiff(dupf::Duplicated{F}, args...) where {F}
    args′ = annotate(args...)
    tt′   = Tuple{map(Core.Typeof, args′)...}
    tt    = Tuple{map(T->eltype(Core.Typeof(T)), args′)...}
    rt    = Core.Compiler.return_type(dupf.val, tt)
    A     = guess_activity(rt)
    autodiff(dupf, A, args′...)
end

"""
    autodiff_deferred(f, Activity, args...)

Same as [`autodiff`](@ref) but uses deferred compilation to support usage in GPU
code, as well as high-order differentiation.
"""
@inline function autodiff_deferred(f::F, ::Type{A}, args...) where {F, A<:Annotation}
    args′ = annotate(args...)
    tt′   = Tuple{map(Core.Typeof, args′)...}
    if A isa UnionAll
        tt = Tuple{map(T->eltype(Core.Typeof(T)), args′)...}
        rt = Core.Compiler.return_type(f, tt)
        rt = A{rt}
    else
        @assert A isa DataType
        rt = A
    end

    if eltype(rt) == Union{}
        error("Return type inferred to be Union{}. Giving up.")
    end

    ptr   = Compiler.deferred_codegen(Val(f), Val(tt′), Val(rt))
    thunk = Compiler.CombinedAdjointThunk{F, rt, tt′, Nothing}(f, ptr, #=df=#nothing)
    if rt <: Active
        args′ = (args′..., one(eltype(rt)))
    elseif A <: Duplicated
        throw(ErrorException("Duplicated Returns not yet handled"))
    end
    thunk(args′...)
end

"""
    autodiff_deferred(f, args...)

Like [`autodiff_deferred`](@ref) but will try to guess the activity of the return value.
"""
@inline function autodiff_deferred(f::F, args...) where {F}
    args′ = annotate(args...)
    tt′   = Tuple{map(Core.Typeof, args′)...}
    tt    = Tuple{map(T->eltype(Core.Typeof(T)), args′)...}
    rt    = Core.Compiler.return_type(f, tt)
    rt    = guess_activity(rt)
    autodiff_deferred(f, rt, args′...)
end

using Adapt
Adapt.adapt_structure(to, x::Duplicated) = Duplicated(adapt(to, x.val), adapt(to, x.dval))
Adapt.adapt_structure(to, x::DuplicatedNoNeed) = DuplicatedNoNeed(adapt(to, x.val), adapt(to, x.dval))
Adapt.adapt_structure(to, x::Const) = Const(adapt(to, x.val))
Adapt.adapt_structure(to, x::Active) = Active(adapt(to, x.val))

end # module


function extend!(
    crr::CorefResolver,
    params::ExtendParams,
    )
    source_pair, target_pair = crr |> innerspaces
    rel_pair = crr.crs.relps[params.rel_pair_ind]

    for (source_space, target_space, joiner) in zip(
            [source_pair.erspace1, source_pair.erspace2],
            [target_pair.erspace1, target_pair.erspace2],
            [rel_pair.sj1, rel_pair.sj2]
            )
        extend!(source_space, target_space, joiner, params.root_side)
        target_space.entity_types = source_space.entity_types
        m = target_space.width
        target_space.entity_types[m] = rel_pair.colmap[other(params.root_side)]
    end
end

function extend!(
    source_space::EntityRefSpace,
    target_space::EntityRefSpace,
    joiner::SortedJoiner,
    side::Int,
    )
    opp = other(side)
    source_arr = source_space.space
    target_arr = target_space.space

    n_source = source_space.height
    m_source = source_space.width

    i_target = 0
    m_target = m_source + 1
    # @inbounds
    for i_source in 1:n_source
        for j in searchsorted(joiner.sorted_views[side], source_arr[i_source, m_source])
            i_target += 1
            # inbounds problem - planning needed
            for k in 1:m_source target_arr[i_target, k] = source_arr[i_source, k] end
            target_arr[i_target, m_source + 1] = joiner.arr[joiner.sorted_inds[side][j], opp]
        end
    end
    target_space.width = m_target
    target_space.height = i_target
end


# run tests in parallel with 2 processes

push!(LOAD_PATH, abspath(joinpath(pwd(), "..")))

using PDESolver
#using Base.Test
using Base.Test
using ODLCommonTools
using PdePumiInterface  # common mesh interface - pumi
using SummationByParts  # SBP operators
using Utils
using EulerEquationMod
using LinearSolvers
using NonlinearSolvers   # non-linear solvers
using OptimizationInterface
using ArrayViews
import ArrayViews.view
import MPI
using Input

#------------------------------------------------------------------------------
# define test list
using TestSystem
include("../tags.jl")

global const EulerTests = TestList()
# define global const tags here

# NOTE: For adding parallel tests to TICON, Write them in a separate file and
# include them below `global const EulerTests = TestList()`.

include("test_parallel_derivatives.jl")

"""
  Run the parallel tests and compare against serial results run as part of
  the serial tests
"""
function test_parallel2()
  @testset "----- Testing Parallel -----" begin

    start_dir = pwd()

    # test rk4
    cd("./rk4/parallel")
    fname = "input_vals_parallel.jl"
    mesh, sbp, eqn, opts = solvePDE(fname)

    datas = readdlm("../serial/error_calc.dat")
    datap = readdlm("error_calc.dat")

    @test isapprox( datas[1], datap[1]) atol=1e-13
    @test isapprox( datas[2], datap[2]) atol=1e-13

    # test staggered_parallel
    cd("../staggered_parallel")
    fname = "input_vals_parallel.jl"
    mesh, sbp, eqn, opts = solvePDE(fname)

    datas = readdlm("../staggered_serial/error_calc.dat")
    datap = readdlm("error_calc.dat")

    @test isapprox( datas[1], datap[1]) atol=1e-13
    @test isapprox( datas[2], datap[2]) atol=1e-13

    cd("../../")

    cd("./lserk/parallel")
    fname = "input_vals_parallel.jl"
    mesh, sbp, eqn, opts = solvePDE(fname)

    datas = readdlm("../serial/error_calc.dat")
    datap = readdlm("error_calc.dat")

    @test isapprox( datas[1], datap[1]) atol=1e-13
    @test isapprox( datas[2], datap[2]) atol=1e-13

    cd("../../")

    # test newton
    cd("./newton/parallel")
    fname = "input_vals_parallel.jl"
    mesh, sbp, eqn, opts = solvePDE(fname)

    datas = readdlm("../serial/error_calc.dat")
    datap = readdlm("./error_calc.dat")
    @test isapprox( datas[1], datap[1]) atol=1e-13

    fill!(eqn.q, 1.0)
    fill!(eqn.res, 0.0)

    for i=1:mesh.npeers
      fill!(eqn.shared_data[i].q_send, 1.0)
      fill!(eqn.shared_data[i].q_recv, 1.0)
    end
    evalHomotopy(mesh, sbp, eqn, opts, eqn.res)

    @test isapprox( vecnorm(eqn.res), 0.0) atol=1e-13

    cd(start_dir)

  end

  return nothing
end

function test_parallel_nopre()

  @testset "----- Testing parallel nopre -----" begin
    start_dir = pwd()

    # test rk4
    cd("./rk4/parallel")
    fname = "input_vals_parallel.jl"

    mesh, sbp, eqn, opts = solvePDE(fname)

    EulerEquationMod.evalResidual(mesh, sbp, eqn, opts)
    res_orig = copy(eqn.res)

    opts["precompute_face_flux"] = false
    fill!(eqn.res, 0.0)

    EulerEquationMod.evalResidual(mesh, sbp, eqn, opts)

    @test isapprox( norm(vec(eqn.res - res_orig)), 0.0) atol=1e-13

    cd(start_dir)

    # test_newton
    cd("./newton/parallel")
    fname = "input_vals_parallel.jl"
    mesh, sbp, eqn, opts = solvePDE(fname)

    fill!(eqn.res, 0.0)
    EulerEquationMod.evalResidual(mesh, sbp, eqn, opts)
    res_orig = copy(eqn.res)

    opts["precompute_face_flux"] = false
    fill!(eqn.res, 0.0)

    EulerEquationMod.evalResidual(mesh, sbp, eqn, opts)

    @test isapprox( norm(vec(eqn.res - res_orig)), 0.0) atol=1e-13

    cd(start_dir)

  end  # end facts block

  return nothing
end

function test_restart()
  @testset "----- Testing restart -----" begin
    start_dir = pwd()

    # test rk4
    cd("./rk4/parallel")
    fname = "input_vals_restart"
    mesh, sbp, eqn, opts = solvePDE(fname)

    datas = readdlm("../serial/error_calc.dat")
    datap = readdlm("error_calc.dat")

    @test isapprox( datas[1], datap[1]) atol=1e-13
    @test isapprox( datas[2], datap[2]) atol=1e-13

    cd("../../lserk/parallel")
    fname = "input_vals_restart"
    mesh, sbp, eqn, opts = solvePDE(fname)

    datas = readdlm("../serial/error_calc.dat")
    datap = readdlm("error_calc.dat")

    @test isapprox( datas[1], datap[1]) atol=1e-13
    @test isapprox( datas[2], datap[2]) atol=1e-13

    cd(start_dir)
  end

  return nothing
end

#test_parallel2()
add_func1!(EulerTests, test_parallel2, [TAG_SHORTTEST])
add_func1!(EulerTests, test_parallel_nopre, [TAG_SHORTTEST])
add_func1!(EulerTests, test_restart, [TAG_SHORTTEST])

#------------------------------------------------------------------------------
# run tests
@testset "----- Running Euler 2 process tests -----" begin
  runTestSystem(EulerTests, solvePDE, ARGS)
end


using Automa
using Automa.RegExp: @re_str
const re = Automa.RegExp

ab = re"ab*"
c = re"c"
pattern = re.cat(ab, c)

ab.actions[:enter] = [:enter_ab]
ab.actions[:exit]  = [:exit_ab]
ab.actions[:all]   = [:all_ab]
ab.actions[:final] = [:final_ab]
c.actions[:enter]  = [:enter_c]
c.actions[:exit]   = [:exit_c]
c.actions[:final]  = [:final_c]

write("actions.dot", Automa.machine2dot(Automa.compile(pattern)))
run(`dot -Tpng -o src/figure/actions.png actions.dot`)


""" Add from private repo folder once "copyright" situatoin is resolved"""
function maximal_RPI_set() end


const CONFIG = MOIT.Config(Int)

const OPTIMIZER = CPLEXCP.Optimizer()
MOI.set(OPTIMIZER, MOI.Silent(), true)
const BRIDGED_OPTIMIZER = MOI.Bridges.full_bridge_optimizer(OPTIMIZER, Float64)

COIT.runtests(
    BRIDGED_OPTIMIZER,
    CONFIG
)


using LITS
using PowerSystems
using Test
using NLsolve
using DiffEqBase
using Sundials

const PSY = PowerSystems

tests = readdir(dirname( @__FILE__))
tests = filter(f -> startswith(f, "test_") &&
                               endswith(f, ".jl") &&
                               f != basename( @__FILE__),
                               tests)

@testset "BasicTests" begin

for test in tests
    print(splitext(test)[1], ": ")
    include(test)
    println()
end

end


# module for defining world coordinate system

using Plots
pyplot()

mutable struct World
    x_min
    x_max
    y_min
    y_max

    # init
    function World(x_min::Float64, x_max::Float64,
                   y_min::Float64, y_max::Float64)
        self = new()
        self.x_min = x_min
        self.x_max = x_max
        self.y_min = y_min
        self.y_max = y_max
        return self
    end
end

function draw(self::World)
    plot([], [], aspect_ratio=true,
         xlabel="X", ylabel="Y",
         xlims=(self.x_min, self.x_max),
         ylims=(self.y_min, self.y_max),
         legend=false)
end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    Data

A domain together with a table of values for each element of the domain.
The i-th row of the table is a vector of features associated with the
i-th element of the domain. If the domain is a mesh, then the table
stores all the properties of the elements.
"""
abstract type Data end

"""
    domain(data)

Return underlying domain of the `data`.
"""
function domain end

"""
    values(data)

Return the values of `data` as a table.
"""
values(data::Data)

"""
    constructor(D)

Return the constructor of the data type `D` as a function.
The function takes a `domain` and `table` as inputs and
combines them into an instance of the data type.
"""
function constructor end

# ----------
# FALLBACKS
# ----------

function (D::Type{<:Data})(stable)
  # build domain from geometry column
  ctable = Tables.columns(stable)
  elms   = Tables.getcolumn(ctable, :geometry)
  domain = Collection(elms)

  # build table from remaining columns
  vars = setdiff(Tables.columnnames(ctable), (:geometry,))
  cols = map(vars) do var
    var => Tables.getcolumn(ctable, var)
  end
  table = (; cols...)

  # combine the two with constructor
  constructor(D)(domain, table)
end

==(data₁::Data, data₂::Data) =
  domain(data₁) == domain(data₂) && values(data₁) == values(data₂)

# implement Domain traits for convenience
embeddim(data::Data) = embeddim(domain(data))
coordtype(data::Data) = coordtype(domain(data))
nelements(data::Data) = nelements(domain(data))
centroid(data::Data, ind) = centroid(domain(data), ind)

# -----------------
# TABLES INTERFACE
# -----------------

Tables.istable(::Type{<:Data}) = true

Tables.rowaccess(::Type{<:Data}) = true

Tables.rows(data::Data) = DataRows(domain(data), Tables.rows(values(data)))

# wrapper type for rows of the data table
# so that we can easily inform the schema
struct DataRows{𝒟,𝒯}
  domain::𝒟
  rtable::𝒯
end

function Base.iterate(rows::DataRows, state=1)
  if state > nelements(rows.domain)
    nothing
  else
    row, _ = iterate(rows.rtable, state)
    elm, _ = iterate(rows.domain, state)
    (; NamedTuple(row)..., geometry=elm), state + 1
  end
end

Base.length(rows::DataRows) = nelements(rows.domain)

function Tables.schema(rows::DataRows)
  geomtype = eltype(rows.domain)
  schema = Tables.schema(rows.rtable)
  names, types = schema.names, schema.types
  Tables.Schema((names..., :geometry), (types..., geomtype))
end

# data table is compatible with the Queryverse
TableTraits.isiterabletable(data::Data) = true
IIE.getiterator(data::Data) = Tables.datavaluerows(Tables.rows(data))
IIE.isiterable(data::Data) = true

Tables.materializer(D::Type{<:Data}) = D

# -------------------
# VARIABLE INTERFACE
# -------------------

"""
    getindex(data, var)

Returns the data for the variable `var` in `data` as a column vector.
"""
Base.getindex(data::Data, var::Symbol) =
  Tables.getcolumn(values(data), var)

Base.getindex(data::Data, var::String) =
  getindex(data, Symbol(var))

function variables(data::Data)
  s = Tables.schema(values(data))
  @. Variable(s.names, nonmissingtype(s.types))
end

# ----------
# UTILITIES
# ----------

"""
    asarray(data, var)

Returns the data for the variable `var` in `data` as a Julia array
with size equal to the size of the underlying domain if the size is
defined, otherwise returns a vector.
"""
function asarray(data::Data, var::Symbol)
  D = domain(data)
  hassize = hasmethod(size, (typeof(D),))
  hassize ? reshape(data[var], size(D)) : data[var]
end

asarray(data::Data, var::String) =
  asarray(data, Symbol(var))

# -----------
# IO METHODS
# -----------

function Base.show(io::IO, data::Data)
  name = nameof(typeof(data))
  nelm = nelements(domain(data))
  Dim  = embeddim(domain(data))
  T    = coordtype(domain(data))
  print(io, "$nelm $name{$Dim,$T}")
end

function Base.show(io::IO, ::MIME"text/plain", data::Data)
  𝒟 = domain(data)
  𝒯 = values(data)
  s = Tables.schema(𝒯)
  vars = zip(s.names, s.types)
  println(io, data)
  println(io, "  variables")
  varlines = ["    └─$var ($V)" for (var,V) in vars]
  println(io, join(sort(varlines), "\n"))
  print(  io, "  domain: ", 𝒟)
end


module TestLearningCompositesCore

# using Revise
using Test
using MLJBase
using ..Models
using ..TestUtilities
using CategoricalArrays
import Random.seed!
seed!(1234)

N =100
X = (x1=rand(N), x2=rand(N), x3=rand(N));
y = 2X.x1  - X.x2 + 0.05*rand(N);

@testset "error messages for invalid learning networks" begin

    Xs = source(X)
    ys = source(y)
    mach1 = machine(Standardizer(), Xs)
    W = transform(mach1, Xs)
    fit!(W, verbosity=0)
    @test_logs((:error, r"Failed"), @test_throws Exception W(34))

    mach2 = machine(DecisionTreeClassifier(), W, ys)
    yhat = predict(mach2, W)
    @test_logs((:error, r"Problem fitting"),
               (:info, r"Running type checks"),
               (:warn, MLJBase.alert_generic_scitype_mismatch(
                   scitype((X, y)),
                   MLJBase.fit_data_scitype(mach2.model),
                   typeof(mach2.model)
               )),
               (:info, r"^It seems an upstream"),
               (:error, r"^Problem fitting"),
               @test_throws Exception fit!(yhat, verbosity=-1))
    Xs = source()
    mach1 = machine(Standardizer(), Xs)
    W = transform(mach1, Xs)
    @test_logs((:error, r"Problem fitting"),
               (:warn, r"^Some learning network source"),
               (:info, r"Running type checks"),
               (:warn, MLJBase.alert_generic_scitype_mismatch(
                   Tuple{Nothing},
                   MLJBase.fit_data_scitype(mach1.model),
                   typeof(mach1.model)
               )),
               (:info, r"^It seems an upstream"),
               (:error, r"^Problem fitting"),
               @test_throws Exception fit!(W, verbosity=-1))
end

@testset "network #1" begin

    knn_ = KNNRegressor(K=7)

    # split the rows:
    allrows = eachindex(y);
    train, valid, test = partition(allrows, 0.7, 0.15);

    Xtrain = selectrows(X, train);
    ytrain = y[train];

    Xs = source(Xtrain);
    ys = source(ytrain);

    knn1 = machine(knn_, Xs, ys)
    @test_mach_sequence fit_only!(knn1) [(:train, knn1),]
    knn_.K = 5
    @test_logs((:info, r"Training"),
               fit_only!(knn1, rows=train[1:end-10], verbosity=2))
    @test_logs (:info, r"Training") fit_only!(knn1, verbosity=2)
    yhat = predict(knn1, Xs);
    yhat(selectrows(X, test))
    @test rms(yhat(selectrows(X, test)), y[test]) < 0.3

    @test_mach_sequence fit!(yhat) [(:skip, knn1),]
    pred = yhat();
end

@testset "network #2" begin

    N =100
    X = (x1=rand(N),
         x2=rand(N),
         x3=categorical(rand("yn",N)),
         x4=categorical(rand("yn",N)));

    y = 2X.x1  - X.x2 + 0.05*rand(N);
    X = source(X)
    y = source(y)

    hot = OneHotEncoder()
    hotM = machine(hot, X)
    W = transform(hotM, X)
    @test scitype(W) == CallableReturning{Table}

    knn = KNNRegressor()
    knnM = machine(knn, W, y)
    yhat = predict(knnM, W)
    @test scitype(yhat) ==
        MLJBase.CallableReturning{AbstractVector{Continuous}}

    @test_mach_sequence fit!(yhat) [(:train, hotM), (:train, knnM)]
    @test_mach_sequence fit!(yhat) [(:skip, hotM), (:skip, knnM)]

    hot.drop_last = true
    @test_mach_sequence fit!(yhat) [(:update, hotM), (:train, knnM)]
    @test_mach_sequence fit!(yhat) [(:skip, hotM), (:skip, knnM)]

    knn.K = 17
    @test_mach_sequence fit!(yhat) [(:skip, hotM), (:update, knnM)]

    # change rows:
    @test_mach_sequence fit!(yhat, rows=1:100) [(:train, hotM), (:train, knnM)]

    # change rows again:
    @test_mach_sequence fit!(yhat) [(:train, hotM), (:train, knnM)]

    # force:
    @test_mach_sequence fit!(yhat, force=true) [(:train, hotM), (:train, knnM)]

end

@testset "with parallel regressors and pre-processors" begin
    N =100
    X = (x1=rand(N),
         x2=rand(N),
         x3=categorical(rand("yn",N)),
         x4=categorical(rand("yn",N)));

    y = abs.(2X.x1  - X.x2 + 0.05*rand(N));
    X = source(X)
    y = source(y)

    cox = UnivariateBoxCoxTransformer()
    coxM = machine(cox, y, cache=true)
    z = transform(coxM, y)

    hot = OneHotEncoder()
    hotM = machine(hot, X, cache=true)
    W = transform(hotM, X)

    knn = KNNRegressor()
    knnM = machine(knn, W, z, cache=true)
    zhat1 = predict(knnM, W)

    @test_mach_sequence fit!(W) [(:train, hotM), ]
    @test_mach_sequence fit!(z) [(:train, coxM), ]
    evaluate!(knnM, measure=l2, verbosity=0);
    fit!(knnM, verbosity=0)

    cox.shift=true
    @test_mach_sequence(fit!(zhat1),
                   [(:update, coxM), (:skip, hotM), (:train, knnM)],
                   [(:skip, hotM), (:update, coxM), (:train, knnM)])

    # no training:
    @test_mach_sequence(fit!(zhat1),
                   [(:skip, coxM), (:skip, hotM), (:skip, knnM)],
                   [(:skip, hotM), (:skip, coxM), (:skip, knnM)])

    tree = DecisionTreeRegressor()
    treeM = machine(tree, W, z, cache=true)
    zhat2 = predict(treeM, W)

    zhat = 0.5*zhat1 + 0.5*zhat2
    @test elscitype(zhat) == Unknown

    @test_mach_sequence(fit!(zhat),
                   [(:skip, coxM), (:skip, hotM),
                     (:skip, knnM), (:train, treeM)],
                   [(:skip, hotM), (:skip, coxM),
                    (:skip, knnM), (:train, treeM)],
                   [(:skip, coxM), (:skip, hotM),
                    (:train, treeM), (:skip, knnM)],
                   [(:skip, hotM), (:skip, coxM),
                    (:train, treeM), (:skip, knnM)])

    yhat = inverse_transform(coxM, zhat)

    @test_mach_sequence(fit!(yhat),
                   [(:skip, coxM), (:skip, hotM),
                     (:skip, knnM), (:skip, treeM)],
                   [(:skip, hotM), (:skip, coxM),
                    (:skip, knnM), (:skip, treeM)],
                   [(:skip, coxM), (:skip, hotM),
                    (:skip, treeM), (:skip, knnM)],
                   [(:skip, hotM), (:skip, coxM),
                    (:skip, treeM), (:skip, knnM)])

    # error handling:
    MLJBase.rebind!(X, "junk5")
    @test_logs((:error, r""),
               (:error, r""),
               (:error, r""),
               (:error, r""),
               @test_throws Exception fit!(yhat, verbosity=-1, force=true))

end

@testset "network #3" begin

    N =100
    X = (x1=rand(N), x2=rand(N), x3=rand(N))
    y = 2X.x1  - X.x2 + 0.05*rand(N)

    XX = source(X)
    yy = source(y)

    # construct a transformer to standardize the target:
    uscale_ = UnivariateStandardizer()
    uscale = machine(uscale_, yy)

    # get the transformed inputs, as if `uscale` were already fit:
    z = transform(uscale, yy)

    # construct a transformer to standardize the inputs:
    scale_ = Standardizer()
    scale = machine(scale_, XX) # no need to fit

    # get the transformed inputs, as if `scale` were already fit:
    Xt = transform(scale, XX)

    # do nothing to the DataFrame
    Xa = node(identity, Xt)

    # choose a learner and make it machine:
    knn_ = KNNRegressor(K=7) # just a container for hyperparameters
    knn = machine(knn_, Xa, z) # no need to fit

    # get the predictions, as if `knn` already fit:
    zhat = predict(knn, Xa)

    # inverse transform the target:
    yhat = inverse_transform(uscale, zhat)

    # fit-through training:
    @test_mach_sequence(fit!(yhat, rows=1:50),
                   [(:train, uscale), (:train, scale), (:train, knn)],
                   [(:train, scale), (:train, uscale), (:train, knn)])

    @test_mach_sequence(fit!(yhat, rows=1:50),
                   [(:skip, uscale), (:skip, scale), (:skip, knn)],
                   [(:skip, scale), (:skip, uscale), (:skip, knn)])

    # change rows:
    @test_mach_sequence(fit!(yhat),
                   [(:train, uscale), (:train, scale), (:train, knn)],
                   [(:train, scale), (:train, uscale), (:train, knn)])

    knn_.K =67
    @test_mach_sequence(fit!(yhat),
                   [(:skip, uscale), (:skip, scale), (:update, knn)],
                   [(:skip, scale), (:skip, uscale), (:update, knn)])
end

@testset "network with machines sharing one model" begin

    N =100
    X = (x1=rand(N), x2=rand(N), x3=rand(N))
    y = 2X.x1  - X.x2 + 0.05*rand(N)

    XX = source(X)
    yy = source(y)

    # construct a transformer to standardize the target:
    uscale_ = UnivariateStandardizer()
    uscale = machine(uscale_, yy)

    # get the transformed inputs, as if `uscale` were already fit:
    z = transform(uscale, yy)

    # construct a transformer to standardize the inputs:
    xscale_ = Standardizer()
    xscale = machine(xscale_, XX) # no need to fit

    # get the transformed inputs, as if `scale` were already fit:
    Xt = transform(xscale, XX)

    # choose a learner and make two machines from it:
    knn_ = KNNRegressor(K=7) # just a container for hyperparameters
    knn1 = machine(knn_, Xt, z) # no need to fit
    knn2 = machine(knn_, Xt, z) # no need to fit

    # get the predictions, as if `knn` already fit:
    zhat1 = predict(knn1, Xt)
    zhat2 = predict(knn2, Xt)
    zhat = zhat1 + zhat2

    # inverse transform the target:
    yhat = inverse_transform(uscale, zhat)

    fit!(yhat, verbosity=0)

    θ = fitted_params(yhat)

end

mutable struct PermuteArgs <: MLJBase.Static
    permutation::NTuple{N,Int} where N
end

MLJBase.transform(p::PermuteArgs, _, args...) =
    Tuple([args[i] for i in p.permutation])
MLJBase.inverse_transform(p::PermuteArgs, _, args...) =
    Tuple([args[i] for i in sortperm(p.permutation |> collect)])

@testset "nodes wrapping Static transformers can be called with rows" begin

    y1 = source(10*ones(Int, 3))
    y2 = source(20*ones(Int, 3))
    y3 = source(30*ones(Int, 3))

    permuter = PermuteArgs((2, 3, 1))
    mach = machine(permuter)
    y = transform(mach, y1, y2, y3)

    @test y(rows=1:2) == (20*ones(Int, 2), 30*ones(Int, 2), 10*ones(Int, 2))

end

@testset "overloading methods for AbstractNode" begin

    A  = rand(3,7)
    As = source(A)
    @test MLJBase.matrix(MLJBase.table(As))() == A

    X = (x1 = [1,2,3], x2=[10, 20, 30], x3=[100, 200, 300])
    Xs = source(X)
    @test selectrows(Xs, 1)() == selectrows(X, 1)
    @test selectrows(Xs, 2:3)() == selectrows(X, 2:3)
    @test selectcols(Xs, 1)() == selectcols(X, 1)
    @test selectcols(Xs, 2:3)() == selectcols(X, 2:3)
    @test selectcols(Xs, :x1)() == selectcols(X, :x1)
    @test selectcols(Xs, [:x1, :x3])() == selectcols(X, [:x1, :x3])
    @test nrows(Xs)() == 3

    y = rand(4)
    ys = source(y)
    @test vcat(ys, ys)() == vcat(y, y)
    @test hcat(ys, ys)() == hcat(y, y)
    @test log(ys)() == log.(y)
    @test exp(ys)() == exp.(y)

    Z = (rand(4), rand(4), rand(4))
    Zs = source(Z)
    @test mean(Zs)() == mean.(Z)
    @test mode(Zs)() == mode.(Z)
    @test median(Zs)() == median.(Z)

    a, b, λ = rand(4), rand(4), rand()
    as, bs = source(a), source(b)
    @test (as + bs)() == a + b
    @test (λ * bs)() == λ * b

    X = source(1)
    y = source(7)
    @test glb(X, y)() == (1, 7)
    glb_node = @tuple X y
    @test glb_node() == (1, 7)

    X = source(1:10)
    Y = node(X->selectrows(X, 3:4), X)
    @test nrows_at_source(Y) == 10
    @test nrows(Y)() == 2

end

@testset "reformat logic - with upstream transformer learning from data" begin
    X = (x1=rand(5), x2=rand(5))
    y = categorical(collect("abaaa"))

    Xs = source(X)
    ys = source(y)

    std = Standardizer()
    mach1 = machine(std, Xs)
    W = transform(mach1, Xs)

    # a classifier with reformat front-end:
    clf = ConstantClassifier(testing=true)
    mach2 = machine(clf, W, ys; cache=true)
    yhat = predict(mach2, W)
    @test_logs((:info, "reformatting X, y"),
               (:info, "resampling X, y"),
               fit!(yhat, verbosity=0, rows=1:3))

    # training network with new rows changes upstream state of
    # classifier and hence retriggers reformatting of data:
    @test_logs((:info, "reformatting X, y"),
               (:info, "resampling X, y"),
               fit!(yhat, verbosity=0, rows=1:2))

    # however just changing classifier hyper-parameter avoids
    # reformatting and resampling:
    clf.bogus = 123
    @test_logs fit!(yhat, verbosity=0, rows=1:2)
end

mutable struct Scale <: MLJBase.Static
    scaling::Float64
end

function MLJBase.transform(s::Scale, _, X)
    X isa AbstractVecOrMat && return X * s.scaling
    MLJBase.table(s.scaling * MLJBase.matrix(X), prototype=X)
end

function MLJBase.inverse_transform(s::Scale, _, X)
    X isa AbstractVecOrMat && return X / s.scaling
    MLJBase.table(MLJBase.matrix(X) / s.scaling, prototype=X)
end

@testset "reformat logic - with upstream Static transformer" begin
    X = (x1=ones(5), x2=ones(5))
    y = categorical(collect("abaaa"))

    Xs = source(X)
    ys = source(y)

    scaler = Scale(2.0)
    mach1 = machine(scaler)
    W = transform(mach1, Xs)

    # a classifier with reformat front-end:
    clf = ConstantClassifier(testing=true)
    mach2 = machine(clf, W, ys, cache=true)
    yhat = predict(mach2, W)
    @test_logs((:info, "reformatting X, y"),
               (:info, "resampling X, y"),
               fit!(yhat, verbosity=0, rows=1:3))

    # training network with new rows does not change upstream state of
    # classifier, because `scaler isa Static` and no reformatting of
    # data:
    @test_logs((:info, "resampling X, y"),
               fit!(yhat, verbosity=0, rows=1:2)
               )

    # however changing an upstream hyperparameter forces reforamatting:
    scaler.scaling = 3.0
    @test_logs((:info, "reformatting X, y"),
               (:info, "resampling X, y"),
               fit!(yhat, verbosity=0, rows=1:2))

    # however just changing classifier hyper-parameter avoids
    # reformatting and resampling:
    clf.bogus = 123
    @test_logs fit!(yhat, verbosity=0, rows=1:2)
end

@testset "@node" begin
    X1 = source(4)
    X2 = source(5)
    Xp = source(1:10)

    add(a, b, c) = a + b + c
    N = @node add(X1, 1, X2)
    @test N() == 10

    N = @node tuple(X1, 5, X1)
    @test N() == (4, 5, 4)

    Y = @node selectrows(Xp, 3:4)
    @test Y() == 3:4
    @test Y([:one, :two, :three, :four]) == [:three, :four]
end

end

true


# rotatez.jl

using SPECTrecon: rotl90!, rotr90!, rot180!, rot_f90!, rot_f90_adj!
using Test: @test, @testset, @test_throws

@testset "rotl90!" begin
    T = Float32
    N = 20
    A = rand(T, N, N)
    B = similar(A, N, N)
    rotl90!(B, A)
    @test isequal(B, rotl90(A))
end

@testset "rotr90!" begin
    T = Float32
    N = 20
    A = rand(T, N, N)
    B = rand(T, N, N)
    rotr90!(B, A)
    @test isequal(B, rotr90(A))
end

@testset "rot180!" begin
    T = Float32
    N = 20
    A = rand(T, N, N)
    B = rand(T, N, N)
    rot180!(B, A)
    @test isequal(B, rot180(A))
end

@testset "rot_f90!" begin
    T = Float32
    N = 20
    A = rand(T, N, N)
    B = rand(T, N, N)
    @test_throws String rot_f90!(A, B, -1)
    @test_throws String rot_f90_adj!(A, B, -1)
    @test_throws String rot_f90!(A, B, 4)
    @test_throws String rot_f90_adj!(A, B, 4)
end


mutable struct JaxArray{T,N} <: AbstractJaxArray{T,N}
  o::PyObject
  dims::NTuple{N,Int}
end

const JaxVector{T} = JaxArray{T,1}
const JaxMatrix{T} = JaxArray{T,2}

JaxArray(arr::AbstractArray) = JaxArray(collect(arr))
function JaxArray(arr::Array{T,N}) where {T,N}
  # When converting an array to Jax Array we transpose it.
  arr = PyReverseDims(arr)
  return numpy.array(arr)
end

JaxNumber(a::Number) = numpy.array(a)

function JaxArray(parr::PyObject)
  if !pyisinstance(parr, jax.interpreters.xla.DeviceArray)
    error("It is not jax.interpreters.xla.DeviceArray but $parr ")
  end
  T = np_to_jl_type(parr.dtype)
  N = length(parr.shape)
  return JaxArray{T,N}(parr, reverse(parr.shape))
end

JaxArray(d::Union{Integer,Dims}...) = JaxArray(Float32, d...)
JaxArray(T::Type{<:Number}, d...) = JaxArray(T, undef, d...)
JaxArray(T::Type{<:Number}, u::UndefInitializer, d::Integer...) =
  JaxArray(T, u, Dims(d))
function JaxArray(T::Type{<:Number}, u::UndefInitializer, d::Dims)
  return numpy.empty(reverse(d), dtype = jl_to_np_type(T))#JaxArray()
end

Base.size(a::JaxArray) = a.dims
Base.length(a::JaxArray) = prod(a.dims)
Base.eltype(a::JaxArray{T}) where {T} = T
Base.axes(a::JaxArray) = map(Base.OneTo, size(a))

PyCall.PyObject(a::JaxArray) = a.o
Base.convert(::Type{<:JaxArray}, o::PyObject) = JaxArray(o)

_transpose(x::Union{AbstractVector,AbstractMatrix}) = transpose(x)
_transpose(x::AbstractArray{T,N}) where {T,N} = permutedims(x, reverse(1:N))

_values(a::JaxArray) = PyArray_ReadOnly(a.o._value) |>_transpose
_values(a::JaxArray{T,0}) where T = PyArray_ReadOnly(a.o._value)
_values(a::JaxArray{T,1}) where T = PyArray_ReadOnly(a.o._value)
Base.collect(a::JaxArray) = copy(_values(a))
Base.convert(::Type{<:Array}, a::JaxArray) = copy(_values(a))

# show
_szstr(d::Tuple{}) = "$(0)-dimensional"
_szstr(d::Tuple{Int}) = "$(first(d))-element"
_szstr(dims::Dims) = prod(["$i×" for i in dims])[1:end-1]
function Base.show(io::IO, m::MIME"text/plain", a::JaxArray{T,N}) where {T,N}
  print(io, "$(_szstr(a.dims)) JaxArray{$T,$N}")
  isempty(a) && return
  print(io, ":")

  if !haskey(io, :compact) && length(axes(a, 2)) > 1
      io = IOContext(io, :compact => true)
  end
  if get(io, :limit, false) && eltype(a) === Method
      # override usual show method for Vector{Method}: don't abbreviate long lists
      io = IOContext(io, :limit => false)
  end

  if get(io, :limit, false) && displaysize(io)[1]-4 <= 0
      return print(io, " …")
  else
      println(io)
  end

  io = IOContext(io, :typeinfo => eltype(a))

  Base.print_array(io, _values(a))
end

function Base.show(io::IO, a::JaxArray{T,N}) where {T,N}
  print(io, "$(_szstr(a.dims)) JaxArray{$T,$N}:")
  vals = _values(a)
  if N == 0
    print(io, "[$(first(vals))]\n")
  else
    print(io, "[$(first(vals)) … $(last(vals))]\n")
  end
end

function Base.show(io::IO, m::MIME"text/plain", a::JaxArray{Bool,0})
  print(io, "JaxScalar{Bool}[$((first(_values(a)))==true)]")
end

function Base.show(io::IO, m::MIME"text/plain", a::JaxArray{T,0}) where T
  print(io, "JaxScalar{$T}[$((first(_values(a))))]")
end

# very slow
# Base.getindex(a::JaxArray, args...) = (getindex(_values(a), args...))

# iterators
Base.eachrow(a::JaxArray) = (get(a.o, i-1) for i in axes(a, 1))

block_until_ready(a::JaxArray) = a.o.block_until_ready()

##
#=
abstract type AbstractJaxNumber end

mutable struct JaxNumber{T} <: AbstractJaxNumber
  o::PyObject
end

function JaxNumber(n::Array{T,0}) where {T}
  # When converting an array to Jax Array we transpose it.
  return JaxNumber(numpy.array(n))
end
=#


include("roblib.jl")

function find_border!(r::Robot, mov::HorizonSide, bord::HorizonSide)
    while isborder(r, bord) == false
        if isborder(r, mov) == false
            move!(r, mov)
        else
            move!(r, bord)
            mov = inverse(mov)
        end
    end
    return mov
end

function fin!(r::Robot)
    num_y1 = moves!(r, Sud)
    num_x = moves!(r, West)
    num_y2 = moves!(r, Sud)
    side = find_border!(r, Nord, Ost)
    mark_perimetr!(r, side)
    moves!(r, West)
    moves!(r, Sud)
end

fin!(r)


using Stations
s = Stations.load_station_info("espc2.info")
Stations.load_station_data(s, "espc2_1.csv")

t = Stations.obs_times(s)
o = Stations.observations(s, "RH")
o1 = o[t[1]]

println(o1)


"""
	grdfilter(cmd0::String="", arg1=nothing, kwargs...)

Filter a grid file in the time domain using one of the selected convolution or non-convolution
isotropic or rectangular filters and compute distances using Cartesian or Spherical geometries.

Full option list at [`grdfilter`]($(GMTdoc)grdfilter.html)

Parameters
----------

- **F** | **filter** :: [Type => Str]

    Sets the filter type.
    ($(GMTdoc)grdfilter.html#f)
- **D** | **distflag** | **distance** :: [Type => Number]

    Distance flag tells how grid (x,y) relates to filter width.
    ($(GMTdoc)grdfilter.html#d)
- **G** | **outgrid** :: [Type => Str]

    Output grid file name. Note that this is optional and to be used only when saving
    the result directly on disk. Otherwise, just use the G = grdfilter(....) form.
    ($(GMTdoc)grdfilter.html#g)
- **I** | **inc** :: [Type => Str or Number]

    *x_inc* [and optionally *y_inc*] is the grid spacing.
    ($(GMTdoc)grdfilter.html#i)
- **N** | **nans** :: [Type => Str]

    Determine how NaN-values in the input grid affects the filtered output. Values are i|p|r
    ($(GMTdoc)grdfilter.html#n)
- $(GMT.opt_R)
- **T** | **toggle** :: [Type => Bool]

    Toggle the node registration for the output grid so as to become the opposite of the input grid
    ($(GMTdoc)grdfilter.html#t)
- $(GMT.opt_V)
- $(GMT.opt_f)
"""
function grdfilter(cmd0::String="", arg1=nothing; kwargs...)

	length(kwargs) == 0 && return monolitic("grdfilter", cmd0, arg1)

	d = init_module(false, kwargs...)[1]		# Also checks if the user wants ONLY the HELP mode
	cmd, = parse_common_opts(d, "", [:R :I :V_params :f])
	cmd  = parse_these_opts(cmd, d, [[:D :distflag :distance], [:F :filter], [:G :outgrid], [:N :nans], [:T :toggle]])

	common_grd(d, cmd0, cmd, "grdfilter ", arg1)		# Finish build cmd and run it
end

# ---------------------------------------------------------------------------------------------------
grdfilter(arg1, cmd0::String=""; kw...) = grdfilter(cmd0, arg1; kw...)


const file  = joinpath(dirname(@__FILE__), "..", "fortran", "FSM.so")

function exp_fortran(val)

    res = ccall((:exp, file), Float64, (Float64,), val)

    return res

end

val = -10.0000001

val_julia = exp(val)

val_fortran = exp_fortran(val)

val_fortran - val_julia


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# FixedPenalization penalty
# ------------------------------------------------------------
"""
    FixedPenalization

Functor object for using fixed penalty in a progressive-hedging algorithm. Create by supplying a [`Fixed`](@ref) object through `penalty` in the `ProgressiveHedgingSolver` factory function and then pass to a `StochasticPrograms.jl` model.

...
# Parameters
- `r::T = 1.00`: Fixed penalty
...
"""
mutable struct FixedPenalization{T <: AbstractFloat} <: AbstractPenalization
    r::T

    function FixedPenalization(r::AbstractFloat)
        T = typeof(r)
        return new{T}(r)
    end
end
function penalty(::AbstractProgressiveHedging, penalty::FixedPenalization)
    return penalty.r
end
function initialize_penalty!(::AbstractProgressiveHedging, ::FixedPenalization)
    nothing
end
function update_penalty!(::AbstractProgressiveHedging, ::FixedPenalization)
    nothing
end

# API
# ------------------------------------------------------------
"""
    Fixed

Factory object for [`FixedPenalization`](@ref). Pass to `penalty` in the `ProgressiveHedgingSolver` factory function. See ?FixedPenalization for parameter descriptions.

"""
struct Fixed{T <: AbstractFloat} <: AbstractPenalizer
    r::T

    function Fixed(; r::AbstractFloat = 1.0)
        T = typeof(r)
        return new{T}(r)
    end
end

function (fixed::Fixed)()
    return FixedPenalization(fixed.r)
end

function str(::Fixed)
    return "fixed penalty"
end