"""
new optimazation idea  - try to put all data in boolean arrays in shared memory  when getting means
next we would need only to read shared memory - yet first one need to check wheather there would be enough shmem on device

- we can also put the data abount slice wise count in shared memory - together with slicewise covariances

varianceZGlobal - should be reduced atomically in the end with all other variables - not slicewise like now
"""

module MeansMahalinobis
using ..CUDAGpuUtils, CUDA, ..IterationUtils, ..ReductionUtils, ..MemoryUtils
export meansMahalinobisKernel,prepareMahalinobisKernel

"""
prepares all equired arguments and gives back the  arguments and thread and block configuration
for calculation of Mahalanobis distance - the latter is based on the occupancy API
"""
function prepareMahalinobisKernel()

    numberToLooFor=1
    #we will fill it after we work with launch configuration
    loopXdim = UInt32(1);loopYdim = UInt32(1) ;loopZdim = UInt32(1) ;
    sizz = (2,2,2);maxX = UInt32(sizz[1]);maxY = UInt32(sizz[2]);maxZ = UInt32(sizz[3])
    #gold
    totalXGold= CuArray([0.0]);
    totalYGold= CuArray([0.0]);
    totalZGold= CuArray([0.0]);
    totalCountGold= CuArray([0]);
    #segm
    totalXSegm= CuArray([0.0]);
    totalYSegm= CuArray([0.0]);
    totalZSegm= CuArray([0.0]);

    varianceXGlobalGold,covarianceXYGlobalGold,covarianceXZGlobalGold,varianceYGlobalGold,covarianceYZGlobalGold,varianceZGlobalGold= CuArray([0.0]),CuArray([0.0]),CuArray([0.0]),CuArray([0.0]),CuArray([0.0]),CuArray([0.0]);
    varianceXGlobalSegm,covarianceXYGlobalSegm,covarianceXZGlobalSegm,varianceYGlobalSegm,covarianceYZGlobalSegm,varianceZGlobalSegm= CuArray([0.0]),CuArray([0.0]),CuArray([0.0]),CuArray([0.0]),CuArray([0.0]),CuArray([0.0]);

    totalCountSegm= CuArray([0]);
    totalCountGold= CuArray([0]);

    countPerZGold= CUDA.zeros(Float32,sizz[3]+1);
    countPerZSegm= CUDA.zeros(Float32,sizz[3]+1);

    # covariancesSliceWiseGold= CUDA.zeros(Float32,6,sizz[3]+1);
    # covariancesSliceWiseSegm= CUDA.zeros(Float32,6,sizz[3]+1);

    covarianceGlobal= CUDA.zeros(Float32,12,1);

    mahalanobisResGlobal= CUDA.zeros(1);

    args = (numberToLooFor
    ,loopYdim,loopXdim,loopZdim
    ,(maxX, maxY,maxZ)
    ,totalXGold,totalYGold,totalZGold,totalCountGold
    ,totalXSegm,totalYSegm,totalZSegm,totalCountSegm,
        countPerZGold, countPerZSegm,
        #,covariancesSliceWiseGold, covariancesSliceWiseSegm,
    varianceXGlobalGold,covarianceXYGlobalGold,covarianceXZGlobalGold,varianceYGlobalGold,covarianceYZGlobalGold,varianceZGlobalGold
        ,varianceXGlobalSegm,covarianceXYGlobalSegm,covarianceXZGlobalSegm,varianceYGlobalSegm,covarianceYZGlobalSegm,varianceZGlobalSegm
        ,mahalanobisResGlobal
        #, mahalanobisResSliceWise
    )

        get_shmem(threads) = (sizeof(UInt32)*3*4)

    threads ,blocks  = getThreadsAndBlocksNumbForKernel(get_shmem,meansMahalinobisKernel,(CUDA.zeros(2,2,2),CUDA.zeros(2,2,2),args...))

        loopXdim = UInt32(fld(maxX, threads[1]))
        loopYdim = UInt32(fld(maxY, threads[2]))
        loopZdim = UInt32(fld(maxZ,blocks ))

    args = (numberToLooFor
    ,loopYdim,loopXdim,loopZdim
    ,(maxX, maxY,maxZ)
    ,totalXGold,totalYGold,totalZGold,totalCountGold
    ,totalXSegm,totalYSegm,totalZSegm,totalCountSegm
        ,countPerZGold, countPerZSegm,
        #,covariancesSliceWiseGold, covariancesSliceWiseSegm,
    varianceXGlobalGold,covarianceXYGlobalGold,covarianceXZGlobalGold,varianceYGlobalGold,covarianceYZGlobalGold,varianceZGlobalGold
        ,varianceXGlobalSegm,covarianceXYGlobalSegm,covarianceXZGlobalSegm,varianceYGlobalSegm,covarianceYZGlobalSegm,varianceZGlobalSegm
        ,mahalanobisResGlobal
        #, mahalanobisResSliceWise

    )

  return(args,threads ,blocks)
end

"""
executed after prepareMahalinobisKernel - will execute on given arrays
    and return the Mahalinobis distance between gold standard mask and the result of segmentation
"""
function calculateMalahlinobisDistance(goldGPU,segmGPU,args,threads ,blocks,numberToLooFor)

# setting main arays

##### setting to 0 all entries that require it
for i in 6:28
    CUDA.fill!(args[i],0)
end

numbToLooFor,loopYdima,loopXdima,loopZdima,dims,totalXGold,totalYGold,totalZGold,totalCountGold,totalXSegm,totalYSegm,totalZSegm,totalCountSegm,countPerZGold, countPerZSegm,varianceXGlobalGold,covarianceXYGlobalGold,covarianceXZGlobalGold,varianceYGlobalGold,covarianceYZGlobalGold,varianceZGlobalGold,varianceXGlobalSegm,covarianceXYGlobalSegm,covarianceXZGlobalSegm,varianceYGlobalSegm,covarianceYZGlobalSegm,varianceZGlobalSegm,mahalanobisResGlobal=args

numberToLooFor
sizz = size(goldGPU)

maxX = UInt32(sizz[1]);maxY = UInt32(sizz[2]);maxZ = UInt32(sizz[3])
loopXdim = UInt32(fld(maxX, threads[1]))
loopYdim = UInt32(fld(maxY, threads[2]))
loopZdim = UInt32(fld(maxZ,blocks ))
countPerZGold= CUDA.zeros(Float32,sizz[3]+1);
countPerZSegm= CUDA.zeros(Float32,sizz[3]+1);
# println( "loopXdim $(loopXdim) loopYdim $(loopYdim)  loopZdim $(loopZdim) maxX $(maxX) maxY $(maxY) maxZ $(maxZ) \n")
args = (numberToLooFor
,loopYdim,loopXdim,loopZdim
,(maxX, maxY,maxZ)
,totalXGold,totalYGold,totalZGold,totalCountGold
,totalXSegm,totalYSegm,totalZSegm,totalCountSegm
    ,countPerZGold, countPerZSegm,
    #,covariancesSliceWiseGold, covariancesSliceWiseSegm,
varianceXGlobalGold,covarianceXYGlobalGold,covarianceXZGlobalGold,varianceYGlobalGold,covarianceYZGlobalGold,varianceZGlobalGold
    ,varianceXGlobalSegm,covarianceXYGlobalSegm,covarianceXZGlobalSegm,varianceYGlobalSegm,covarianceYZGlobalSegm,varianceZGlobalSegm
    ,mahalanobisResGlobal
    #, mahalanobisResSliceWise

)

@cuda cooperative=true threads=threads blocks=blocks meansMahalinobisKernel(goldGPU,segmGPU,args...)

return  args[28][1]
end

"""
part of the above kernel that will be executed in order to get means
first of the gold standard array and then for other
    arrAnalyzed- gold standard or other array
    countPerZ- global memory array holding per slice counts of trues

"""
macro iterateForMeans(countPerZ,arrAnalyzed)
 return esc(quote

    @iter3dAdditionalzActs(arrDims,loopXdim,loopYdim,loopZdim,
    #inner expression
    if(  @inbounds($arrAnalyzed[x,y,z])  ==numberToLooFor)
        #updating variables needed to calculate means
        sumX+=Float32(x) ;  sumY+=Float32(y)  ; sumZ+=Float32(z)   ; count+=Float32(1)
    end,
    #after z expression - we get slice wise true counts from it
    begin
        sync_threads()
        #reducing count only
        if(z<=arrDims[3])
            countTemp = count
            @redWitAct(offsetIter,shmemSum, count,+)
            #saving to global memory count of this slice
            @ifXY 1 1 begin
                 $countPerZ[z]=(shmemSum[1,1] - oldZVal[1] )
                oldZVal[1]=shmemSum[1,1]
            end
            #clear shared memory only first row was used and sync threads
            clearSharedMemWarpLong(shmemSum, UInt8(1), Float32(0.0))
            count=countTemp#to preserve proper value for total count
        end#if ar dims
    end )#if bool in arr

    #tell what variables are to be reduced and by what operation
    @redWitAct(offsetIter,shmemSum,  sumX,+,     sumY,+    ,sumZ,+   ,count,+)

end)#quote
end

"""
reset variables and arrays to be able to calculate the variances and Covariances
"""

macro resetForVarAndCov(totalX,totalY, totalZ, totalCount)
    return esc(quote
        #at this spot we should have all counts of x,y,z and counts - we can get means from it
        clearSharedMemWarpLong(shmemSum, UInt8(6), Float32(0.0))

        @ifY 4 @unroll for i in 1:5
            @ifX i @inbounds intermedieteRes[i]= Float32(0.0)
        end

        @ifXY 1 1  meanxyz[1]= $totalX[1]/$totalCount[1]
        @ifXY 1 2  meanxyz[2]= $totalY[1]/$totalCount[1]
        @ifXY 1 3  meanxyz[3]= $totalZ[1]/$totalCount[1]
        #from now one sumX is variance of x ,sumY is covariance xy ,sumZ is covariance xz
        sumX,sumY,sumZ ,count= Float32(0.0),Float32(0.0),Float32(0.0),Float32(0.0)

    end)#quote
end

"""
calculates variances and covariances needed for calculation of mahalanobis

"""
macro calculateVariancesAdCov(countPerZ,arrToAnalyze,varianceXGlobal,covarianceXYGlobal,covarianceXZGlobal,varianceYGlobal,covarianceYZGlobal,varianceZGlobal)
    return esc(quote
    @iter3dAdditionalxyzActsAndZcheck(arrDims,loopXdim,loopYdim,loopZdim
    #z check
    ,((z<= arrDims[3] && $countPerZ[z]>0.0)),
    #inner expression
    if(  @inbounds($arrToAnalyze[x,y,z])  ==numberToLooFor)
        #getting variance x and x covariances
        sumX+=(Float32(x)-meanxyz[1])*(Float32(x)-meanxyz[1])
        sumY+=((Float32(y)-meanxyz[2])*(Float32(x)-meanxyz[1]))
        sumZ+=((Float32(z)-meanxyz[3])*(Float32(x)-meanxyz[1]))
        count+=Float32(1.0)
         end,
    #x additional fun - currently nothing to implement here
    begin    end,
    #y additional fun - as we added all x variances and covariances we need now to update y variance and y covariance
    #we need to add this the same amount of time as we added the x variables so we use count for it
    # of course if count is 0 we can ignore this step
    begin
   #we put reduced values into share memory
    sync_threads()
  @redOnlyStepOne(offsetIter,shmemSum,  sumX,+,     sumY,+    ,sumZ,+   ,count,+);

   if(threadIdxX()==UInt32(1) && count>Float32(0.0))
       @inbounds shmemSum[threadIdxY(),1]+= sumX
       @inbounds shmemSum[threadIdxY(),2]+= sumY
       @inbounds shmemSum[threadIdxY(),3]+= sumZ
       #putting  variance y and covariance yz manually to shared memory multiply appropriate amount of time
       @inbounds shmemSum[threadIdxY(),4]+= count*(y-meanxyz[2])*(y-meanxyz[2])#variance y
       @inbounds shmemSum[threadIdxY(),5]+= count*((y-meanxyz[2])*(z-meanxyz[3]))#covariance yz
       @inbounds shmemSum[threadIdxY(),6]+= count

   end;
   #reset as values are already saved in shmemsum
   sumX,sumY,sumZ ,count= Float32(0.0),Float32(0.0),Float32(0.0),Float32(0.0)
   sync_warp()

    end,
    #z additional fun
    begin
        sync_threads()
        #no point in analyzing if it is empty
        if(z<= arrDims[3] && $countPerZ[z]>0.0)
            #we do the last step of reductions to get all of the values into first spots of shared memory
            @redOnlyStepThree(offsetIter,shmemSum, +,+,+  ,+,+,+)
            sync_threads()

            #we will use it later to get slicewise results and in the end we will send those to global memory
            @ifY 1 @unroll for i in 1:5
                @ifX i intermedieteRes[i]+=shmemSum[1,i]
            end

            @ifXY 1 7 @inbounds intermedieteRes[6]+=(((z- meanxyz[3])*(z- meanxyz[3])  )*shmemSum[1,6])

            @ifXY 1 7 @inbounds CUDA.@atomic $varianceZGlobal[]+=((z- meanxyz[3])*(z- meanxyz[3])  )*shmemSum[1,6]
            ###########remove

            # @ifY 2 @unroll for i in 1:5
            #     @ifX i @inbounds $covariancesSliceWise[i,z]+=shmemSum[1,i]

            #     if(shmemSum[1,i]>0)
            #         CUDA.@cuprint " i $(i) val $(shmemSum[1,i]) "
            #     end

            # end
            #     @ifXY 2 6 @inbounds $covariancesSliceWise[6,z]+= ((z- meanxyz[3])*(z- meanxyz[3]))*shmemSum[1,6]
            sync_threads()
            # #clear shared memory
            clearSharedMemWarpLong(shmemSum, UInt8(6), Float32(0.0))

        end#if covariance non empty
    end  )# iterations loop

    sync_threads()

    #at this point we should have all variances and covariances in intermedieteRes and we can send it to global results
    @ifXY 1 1  @inbounds CUDA.@atomic $varianceXGlobal[]+=intermedieteRes[1]
    @ifXY 1 2 @inbounds CUDA.@atomic $covarianceXYGlobal[]+=intermedieteRes[2]
    @ifXY 1 3  @inbounds CUDA.@atomic $covarianceXZGlobal[]+=intermedieteRes[3]
    @ifXY 1 4  @inbounds CUDA.@atomic $varianceYGlobal[]+=intermedieteRes[4]
    @ifXY 1 5 @inbounds CUDA.@atomic $covarianceYZGlobal[]+=intermedieteRes[5]
    #@ifXY 1 7  @inboundsCUDA.@atomic $varianceZGlobal[]+=intermedieteRes[6]

end)#quote
end#calculateVariancesAdCov

"""
this will enable final calculations of the Mahalanobis distance

#original formula developed on basis of https://stats.stackexchange.com/questions/147210/efficient-fast-mahalanobis-distance-computation/147222#147222?newreg=a68aa51b2f8c45daaece49163105845c
#unrolled 3 by 3 cholesky decomposition
# a = sqrt(varianceX)
# b = (covarianceXY)/a
# c = (covarianceXZ)/a
# e = sqrt(varianceY - b*b)
# d = (covarianceYZ -(c * b))/e
# #unrolled forward substitiution
# ya= x[1]/a
# yb = (x[2]-b*ya)/e
# yc= (x[3]-yb*d-ya* c)/sqrt(varianceZ - c*c -d*d )
#taking square euclidean distance
# returnya*ya+yb*yb+yc*yc

"""
macro getFinalResults()
    return esc(quote
    #first getting total counts to all threads
    sumY= totalCountGold[1]#total count gold
    sumZ= totalCountSegm[1]#total count segm

    #values needed for calculating means
    @ifXY 14 1 sumX = totalXGold[1]
    @ifXY 16 1 sumX = totalYGold[1]
    @ifXY 18 1 sumX = totalZGold[1]
    @ifXY 20 1 sumX = totalXSegm[1]
    @ifXY 22 1 sumX = totalYSegm[1]
    @ifXY 24 1 sumX = totalZSegm[1]
    sync_warp()
    @ifXY 14 1 shmemSum[7,1] = sumX/sumY #mean X gold
    @ifXY 16 1 shmemSum[8,1] = sumX/sumY #mean Y gold
    @ifXY 18 1 shmemSum[9,1] = sumX/sumY #mean Z gold
    @ifXY 20 1 shmemSum[10,1] = sumX/sumZ #mean X segm
    @ifXY 22 1 shmemSum[11,1] =  sumX/sumZ #mean Y segm
    @ifXY 24 1 shmemSum[12,1] = sumX/sumZ #mean Z segm
    sync_threads()# now all blocks should have available means

    if(blockIdxX()==1)# this one for global result should be executed only once
        @unroll for numb in 1:1
            #first we upload to registers variables needed to calculate Mahalinobis
            @ifXY 1 numb sumX = varianceXGlobalGold[1]
            @ifXY 2 numb shmemSum[13,1] = varianceXGlobalSegm[1]
            @ifXY 3 numb sumX = covarianceXYGlobalGold[1]
            @ifXY 4 numb shmemSum[14,1] = covarianceXYGlobalSegm[1]
            @ifXY 5 numb sumX = covarianceXZGlobalGold[1]
            @ifXY 6 numb shmemSum[15,1] = covarianceXZGlobalSegm[1]
            @ifXY 7 numb sumX = varianceYGlobalGold[1]
            @ifXY 8 numb shmemSum[16,1] = varianceYGlobalSegm[1]
            @ifXY 10 numb sumX = covarianceYZGlobalGold[1]
            @ifXY 11 numb shmemSum[17,1] = covarianceYZGlobalSegm[1]
            @ifXY 12 numb sumX = varianceZGlobalGold[1]
            @ifXY 13 numb shmemSum[18,1] = varianceZGlobalSegm[1]

            sync_warp()
            #sumX+=shfl_down_sync(active_mask(),sumX,UInt8(1))

            @ifXY 1 numb shmemSum[1,1]= (sumX+shmemSum[13,1])/(sumY+sumZ)  #common variance x
            @ifXY 3 numb shmemSum[2,1]= (sumX+shmemSum[14,1])/(sumY+sumZ) #common covariance xy
            @ifXY 5 numb shmemSum[3,1] = (sumX+shmemSum[15,1])/(sumY+sumZ)#common covariance xz
            @ifXY 7 numb shmemSum[4,1] = (sumX+shmemSum[16,1])/(sumY+sumZ) #common variance y
            @ifXY 10 numb shmemSum[5,1] = (sumX+shmemSum[17,1])/(sumY+sumZ) #common covariance yz
            @ifXY 12 numb shmemSum[6,1] = (sumX+shmemSum[18,1])/(sumY+sumZ) #common variance z
            sync_warp()
            @ifXY 1 1 begin
                # #unrolled 3 by 3 cholesky decomposition
                a = CUDA.sqrt(shmemSum[1,1])  #getting varianceX
                b = (shmemSum[2,1])/a #getting covarianceXY
                c = (shmemSum[3,1])/a  #getting covarianceXZ
                e = CUDA.sqrt(shmemSum[4,1] - b*b)  #getting varianceY
                d = (shmemSum[5,1] -(c * b))/e #getting covarianceYZ
              #unrolled forward substitiution - we reuse the variables to reduce register preassure (hopefully)
                sumX= (shmemSum[7,1] -shmemSum[10,1]  )   /a
                sumY = ( (shmemSum[8,1] -  shmemSum[11,1] )  -b*sumX)/e
                sumZ= ( (shmemSum[9,1] - shmemSum[12,1]  )  -sumY*d-sumX* c)/CUDA.sqrt(shmemSum[6,1] - c*c -d*d ) #getting  varianceZ
                #taking square euclidean distance
                mahalanobisResGlobal[1]= CUDA.sqrt(sumX*sumX+sumY*sumY+sumZ*sumZ)
            end # @ifXY  numb
          end#for numb
    end# if block
#### now in case we want the slice wise results we need to loop through the
#sync_threads()
#in order to maximize parallelization we will use 6 warps at a time (if needed )
# and each will use up diffrent row of shared memory so we will loop through slicewise
#and concurrently update the slicewise results list
# our data is stored in covariancesSliceWiseGold and  covariancesSliceWiseSegm
#1) variance x 2) covariance xy 3) covariance xz 4) variance y 5) covariance yz 6) variance z this applies to both

# @unroll for i in 0:cld(loopZdim,6)
#     @unroll for j in 1:6
#         #we need to stay in the existing slices
#         index = (i*6+j)*gridDim().x+blockIdxX()
#         if(index<arrDims[3])
#         #first we load data from gold
#         sumX,sumY,sumZ,count = Float32(0),Float32(0),Float32(0),Float32(0)
#         shmemSum[threadIdxX(),j]=Float32(0.0)
#             if(threadIdxX()<7 )
#                 sumx = covariancesSliceWiseGold[threadIdxX(),index]
#             #then from segm
#             elseif(threadIdxX()<13 )
#                 shmemSum[12+threadIdxX() ,1] = covariancesSliceWiseSegm[threadIdxX()-6,index]
#             end #else if
#             sync_warp()
#             #we set it on diffrent warps

#             @ifXY 1 j shmemSum[1,j] = (sumX+shmemSum[13,j])/(sumY+sumZ)  #common variance x
#             @ifXY 3 j shmemSum[2,j] = (sumX+shmemSum[14,j])/(sumY+sumZ) #common covariance xy
#             @ifXY 5 j shmemSum[3,j] = (sumX+shmemSum[15,j])/(sumY+sumZ)#common covariance xz
#             @ifXY 7 j shmemSum[4,j] = (sumX+shmemSum[16,j])/(sumY+sumZ) #common variance y
#             @ifXY 10 j shmemSum[5,j] = (sumX+shmemSum[17,j])/(sumY+sumZ) #common covariance yz
#             @ifXY 12 j shmemSum[6,j] = (sumX+shmemSum[18,j])/(sumY+sumZ) #common variance z

#            # CUDA.@cuprint " varx $(shmemSum[1,j]) cov xy  $(shmemSum[2,j])  cov xz $(shmemSum[3,j])  var y $(shmemSum[4,j]) covyz $(shmemSum[5,j]) var z  $(shmemSum[6,j]) \n"

#             sync_warp()

#             @ifXY 1 j begin
#                 # #unrolled 3 by 3 cholesky decomposition
#                 a = CUDA.sqrt(shmemSum[1,j])  #getting varianceX

#                 b = (shmemSum[2,j])/a #getting covarianceXY
#                 c = (shmemSum[3,j])/a  #getting covarianceXZ
#                 e = CUDA.sqrt(shmemSum[4,j] - b*b)  #getting varianceY
#                 d = (shmemSum[5,j] -(c * b))/e #getting covarianceYZ
#               #unrolled forward substitiution - we reuse the variables to reduce register preassure (hopefully)
#                 sumX= (shmemSum[7,1] -shmemSum[10,1]  )   /a
#                 sumY = ( (shmemSum[8,1] -  shmemSum[11,1] )  -b*sumX)/e
#                 sumZ= ( (shmemSum[9,1] - shmemSum[12,1]  )  -sumY*d-sumX* c)/CUDA.sqrt(shmemSum[6,j] - c*c -d*d ) #getting  varianceZ
#                 #taking square euclidean distance
#                 mahalanobisResSliceWise[index]= CUDA.sqrt(sumX*sumX+sumY*sumY+sumZ*sumZ)
#                 #CUDA.@cuprint "slice wise $(CUDA.sqrt(sumX*sumX+sumY*sumY+sumZ*sumZ))"
#             end # @ifXY  numb
#         end#if
#     end#for
#     sync_warp()
# end#for

    end)
end

"""
IMPORTANT x dim of threadblock needs to be always 32
goldArr,segmArr  - arrays we analyze
numberToLooFor - number we are intrested in in the array
loopYdim, loopXdim, loopZdim - number of times we nned to loop over those dimensions in order to cover all - important we start iteration from 0 hence we should use fld ...
maxX, maxY ,maxZ- maximum possible x and y - used for bound checking
totalX,totalY,totalZ - holding the results of summation of x,y and z's
totalCount - total number of non 0 entries
countPerZ - count of non empty entries per slice - particularly usefull in case it is 0
covariancesSliceWiseGold,covariancesSliceWiseSegm  - slice wise covariances - needed in case we want slice wise results - a matrix where each row entry is
        I)gold standard mask values
        1)variance x    2)cov xy     3)cov xz     4)var y    5)cov yz      6)var z

covarianceGlobal (just one column but entries exactly the same as above)
mahalanobisResGlobal - global result of Mahalinobis distance
mahalanobisResSliceWise - global result of Mahalinobis distance
"""
function meansMahalinobisKernel(goldArr,segmArr
    ,numberToLooFor
    ,loopYdim::UInt32
    ,loopXdim::UInt32
    ,loopZdim::UInt32
    ,arrDims::Tuple{UInt32,UInt32,UInt32}
    ,totalXGold,totalYGold,totalZGold,totalCountGold
    ,totalXSegm,totalYSegm,totalZSegm,totalCountSegm
    ,countPerZGold,countPerZSegm
    ,varianceXGlobalGold,covarianceXYGlobalGold,covarianceXZGlobalGold,varianceYGlobalGold,covarianceYZGlobalGold,varianceZGlobalGold
    ,varianceXGlobalSegm,covarianceXYGlobalSegm,covarianceXZGlobalSegm,varianceYGlobalSegm,covarianceYZGlobalSegm,varianceZGlobalSegm
    ,mahalanobisResGlobal   )

    grid_handle = this_grid()
    # keeping counter of old z value - in order to be able to get slicewise z counter
    oldZVal= @cuStaticSharedMem(Float32, (1))
    #summing coordinates of all voxels we are intrested in
    sumX,sumY,sumZ,count = Float32(0),Float32(0),Float32(0),Float32(0)
    #count how many voxels of intrest there are so we will get means
    #for storing results from warp reductions
    shmemSum= @cuStaticSharedMem(Float32, (32,6))
    clearSharedMemWarpLong(shmemSum, UInt8(6), Float32(0.0))
    @ifXY 1 1 oldZVal[1]=0
    #just needed for reductions
    offsetIter = UInt8(1)
    #### first we analyze gold standard array
    sync_threads()

    @iterateForMeans(countPerZGold,goldArr)

    #now we have needed values in  shmemSum[1,1] - sumX  shmemSum[1,2] - sumY shmemSum[1,3] - sumZ and in shmemSum[1,4] - offsetIter
    #important we need to send values to the atomics on the same warps as we did reduction
    #so we can avoid thread synchronization in such situation
    @addAtomic(shmemSum,totalXGold, totalYGold ,totalZGold,totalCountGold)

    sync_threads()
    ### now analyzing segmentation array
    #resetting variables
    sumX,sumY,sumZ = Float32(0),Float32(0),Float32(0)
    count= Float32(0)
    clearSharedMemWarpLong(shmemSum, UInt8(6), Float32(0.0))
    @ifXY 1 1 oldZVal[1]=0
    sync_threads()
    #iterations

    @iterateForMeans(countPerZSegm,segmArr)

    @addAtomic(shmemSum,totalXSegm, totalYSegm ,totalZSegm,totalCountSegm)

    sync_grid(grid_handle)
##################### getting covariances

##first gold mask
    #first prepare all needed memory
    # we are using shared memory to hold means of 1)x 2)y and 3)z
    meanxyz= @cuStaticSharedMem(Float32, (3))
    #for storing intermediate results in shared memory
    #1)variance x    2)cov xy     3)cov xz     4)var y    5)cov yz      6)var z
    intermedieteRes= @cuStaticSharedMem(Float32, (6))
    #reset required memory
    @resetForVarAndCov(totalXGold,totalYGold, totalZGold, totalCountGold)
    sync_threads()
    #calculate means and covariances
    @calculateVariancesAdCov(countPerZGold,goldArr,varianceXGlobalGold,covarianceXYGlobalGold,covarianceXZGlobalGold,varianceYGlobalGold,covarianceYZGlobalGold,varianceZGlobalGold)

    ######### other mask
    sync_threads()
    @resetForVarAndCov(totalXSegm,totalYSegm,totalZSegm,totalCountSegm)
    sync_threads()
    @calculateVariancesAdCov(countPerZSegm,segmArr,varianceXGlobalSegm,covarianceXYGlobalSegm,covarianceXZGlobalSegm,varianceYGlobalSegm,covarianceYZGlobalSegm,varianceZGlobalSegm)

###################### getting final results

sync_grid(grid_handle)
    #preparing space
    sumX,sumY,sumZ,count = Float32(0),Float32(0),Float32(0),Float32(0)
    clearSharedMemWarpLong(shmemSum, UInt8(6), Float32(0.0))
    sync_threads()
    #calculate final results
    @getFinalResults()

end#meansMahalinobisKernel

end#MeansMahalinobis

# """
# in order to avoid overfilling of local result list we need to from time to time push it into the global
# and clear it
# """
# function pushlocalResToGlobal(intermidiateResX, intermidiateResY, intermidiateResZ,intermediateResCounter, resList, resListCounter,currVal )
#     #opdate the counter for the global list use old as offset where we start to put our results
#     if(currVal>1)
#         #oldLocal = CUDA.atomic_xchg!(pointer(intermediateResCounter), UInt32(1))
#         intermediateResCounter[1]= UInt32(1)
#         oldCount::UInt32 = @inboundsCUDA.@atomic resListCounter[]+=UInt32(currVal-UInt32(1))

#             #pushing from local to global queue
#             @unroll for i in 1:currVal
#                 @inbounds  resList[oldCount+i,1] = intermidiateResX[i]
#                 @inbounds  resList[oldCount+i,2] = intermidiateResY[i]
#                 @inbounds  resList[oldCount+i,3] = intermidiateResZ[i]
#                 CUDA.@cuprint "intermidiateRes[i,1] $(intermidiateResX[i])  intermidiateRes[i,2] $(intermidiateResY[i]) intermidiateRes[i,3] $(intermidiateResZ[i])  \n "

#             end#for z dim
#         #reset local counter
#     end
# end

# function meansMahalinobisKernel(goldArr,segmArr
#     ,numberToLooFor
#     ,loopYdim::UInt32
#     ,loopXdim::UInt32
#     ,loopZdim::UInt32
#     ,arrDims::Tuple{UInt32,UInt32,UInt32}
#     ,totalXGold,totalYGold,totalZGold,totalCountGold
#     ,totalXSegm,totalYSegm,totalZSegm,totalCountSegm  )

#     #summing coordinates of all voxels we are intrested in
#     sumX,sumY,sumZ = UInt64(0),UInt64(0),UInt64(0)
#     #count how many voxels of intrest there are so we will get means
#     count::UInt16 = UInt16(0)
#     #for storing results from warp reductions
#     shmemSum= @cuStaticSharedMem(UInt32, (32,4))
#     clearSharedMemWarpLong(shmemSum, UInt8(4))
#     #just needed for reductions
#     offsetIter = UInt8(1)
#     #### first we analyze gold standard array
#     @iter3d arrDims loopXdim loopYdim  loopZdim if(  @inbounds(goldArr[x,y,z])  ==numberToLooFor)
#         #updating variables needed to calculate means
#         sumX+=UInt64(x) ;  sumY+=UInt64(y)  ; sumZ+=UInt64(z)   ; count+=UInt16(1)
#     end#if bool in arr

#     #tell what variables are to be reduced and by what operation
#     @redWitAct(offsetIter,shmemSum,  sumX,+,     sumY,+    ,sumZ,+   ,count,+)
#     #now we have needed values in  shmemSum[1,1] - sumX  shmemSum[1,2] - sumY shmemSum[1,3] - sumZ and in shmemSum[1,4] - offsetIter
#     #important we need to send values to the atomics on the same warps as we did reduction
#     #so we can avoid thread synchronization in such situation
#     @addAtomic(shmemSum,totalXGold, totalYGold ,totalZGold,totalCountGold)

#     ### now analyzing segmentation array
#     #resetting variables
#     sumX,sumY,sumZ = UInt64(0),UInt64(0),UInt64(0)
#     count= UInt16(0)
#     clearSharedMemWarpLong(shmemSum, UInt8(4))
#     offsetIter = UInt8(1)

#     @iter3d arrDims loopXdim loopYdim  loopZdim if(  @inbounds(segmArr[x,y,z])  ==numberToLooFor)
#         sumX+=UInt64(x) ;  sumY+=UInt64(y)  ; sumZ+=UInt64(z)   ; count+=UInt16(1)
#     end#if bool in arr
#     @redWitAct(offsetIter,shmemSum,  sumX,+,     sumY,+    ,sumZ,+   ,count,+)
#     @addAtomic(shmemSum,totalXSegm, totalYSegm ,totalZSegm,totalCountSegm)

#     return
#     end#meansMahalinobisKernel

# end#MeansMahalinobis


struct GTIData <: JAXTAMData
    mission        :: Mission
    instrument     :: Symbol
    obsid          :: String
    e_range        :: Tuple{Float64,Float64}
    bin_time       :: Real
    gti_index      :: Int
    gti_start_time :: Real
    counts         :: Array
    times          :: StepRangeLen
    group          :: Int
end

"""
    _lc_filter_gtis(binned_times::StepRangeLen, binned_counts::Array{Int,1}, gtis::DataFrames.DataFrame, mission::Symbol, instrument::Symbol, obsid::String; min_gti_sec=16)

Splits the lightcurve (count) data into GTIs

First, removes GTIs under `min_gti_sec`, then puts the lightcurve data
into a `Dict{Int,GTIData}`, with the key as the `index` of the GTI
"""
function _lc_filter_gtis(mission::Mission, instrument::Symbol, obsid::String, e_range::Tuple{Float64,Float64},
        binned_times::StepRangeLen, binned_counts::Array{Int,1}, gtis::DataFrames.DataFrame;
        min_gti_sec=16
    )
    gti_data = Dict{Int,GTIData}()

    gti_groups = gtis[:group]
    gti_times  = hcat(gtis[:start], gtis[:stop])

    bin_time = binned_times[2] - binned_times[1]

    gti_mask = (gtis[:stop] .- gtis[:start]) .>= min_gti_sec # Exclude GTIs under `min_gti_sec`
    gti_times[gti_mask.==false, :] .= -1 # Set excluded GTIs to -1, GTIs aren't just discarded so that their absolute index can be kept track of

    excluded_gti_count = count(gti_mask.==false)

    @info "               -> prelim. excluded $excluded_gti_count GTIs under $(min_gti_sec)s"

    if excluded_gti_count == length(gti_groups)
        # All GTIs have been excluded
        throw(JAXTAMError("All GTIs under $(min_gti_sec)s, none left", :gtis, BoundsError(Any[], (1,))))
    end

    # Do this as the first GTI being zero screws with the initial start index
    # in the upcoming `for gtis` loop
    if gti_times[1, 1] == 0
        gti_times[1, 1] = eps()
    end

    # Dodgy way to convert a matrix into an array of arrays
    # so each GTI is stored as an array of [start; finish]
    # and each of those GTI arrays is an array itself
    # makes life a bit easier for the following `for gti in gtis` loop
    gti_times = [gti_times[x, :] for x in 1:size(gti_times, 1)]

    @info "               -> sorting GTIs"

    # Offset if binned_times are a subarray of the lightcurve
    if typeof(binned_times) <: SubArray
        subarray_offset = binned_times.offset1
    else
        subarray_offset = 0
    end

    for (i, gti) in enumerate(gti_times) # For each GTI, store the selected times and count rate within that GTI
        if gti[1] == -1 # Short GTIs have their start time set to -1, skipped here
            continue
        end

        start = ceil(Int, gti[1]/bin_time)+1  - subarray_offset
        stop  = floor(Int, gti[2]/bin_time)-1 - subarray_offset

        # Subtract GTI start time from all times, so all start from t=0
        array_times = binned_times[start:stop].-gti[1]
        range_times = array_times[1]:bin_time:array_times[end]

        gti_data[Int(i)] = GTIData(mission, instrument, obsid, e_range, bin_time, i, start,
            binned_counts[start:stop], range_times, gti_groups[i])
    end

    total_counts = sum(binned_counts)[1]
    gti_counts   = sum([sum(gti.counts) for gti in values(gti_data)])
    count_delta  = gti_counts-total_counts
    delta_prcnt  = round(count_delta/total_counts*100, digits=2)

    @info "               -> original counts: $total_counts | remaining: $gti_counts | delta: $count_delta ($delta_prcnt %)"

    if delta_prcnt > 10
        @warn "Count delta > 10% of total counts"
    end

    return gti_data
end

"""
    _gtis(lc::BinnedData)

Calls `_lc_filter_gtis` using `BinnedData` input
"""
function _gtis(lc::BinnedData)
    gti_data = _lc_filter_gtis(lc.mission, lc.instrument, lc.obsid, lc.e_range, lc.times, lc.counts, lc.gtis)

    return gti_data
end

"""
    _gtis_save(gtis, gti_dir::String)

Splits up the GTI data into a `_meta.feather` file containing non-array variables for each GTI (index, start/stop times, etc...)
and multiple `_gti.feather` files for each GTI containing the counts and times
"""
function _gtis_save(mission::Mission, obs_row::DataFrames.DataFrameRow{DataFrames.DataFrame},
        instrument::Union{String,Symbol}, e_range::Tuple,
        gtis_dir::String, gtis::Dict{Int64,JAXTAM.GTIData}; log=true
    )
    @info "Saving $instrument gti data"
    mkpath(gtis_dir)
    log_entries = Dict{Union{Symbol,Int},String}()

    gti_indecies = [k for k in keys(gtis)]
    gti_starts   = [t.gti_start_time for t in values(gtis)]
    gti_example  = gtis[gti_indecies[1]]
    gti_groups   = [o.group for o in values(gtis)]

    gti_basename = string("$(gti_example.instrument)_lc_$(gti_example.bin_time)_gti")
    gtis_meta    = DataFrame(mission=string(_mission_name(gti_example.mission)), instrument=String(gti_example.instrument),
        obsid=gti_example.obsid, bin_time=gti_example.bin_time, indecies=gti_indecies, starts=gti_starts, groups=gti_groups)

    Feather.write(joinpath(gtis_dir, "$(gti_basename)_meta.feather"), gtis_meta)
    log_entries[:path_meta] = joinpath(gtis_dir, "$(gti_basename)_meta.feather")

    for index in gti_indecies
        gti_savepath = joinpath(gtis_dir, "$(gti_basename)_$(index).feather")
        Feather.write(gti_savepath, DataFrame(counts=gtis[index].counts, times=gtis[index].times))
        log_entries[index] = gti_savepath
    end

    if log
        _log_add(mission, obs_row,
            Dict("data" =>
                Dict(:gtis =>
                    Dict(e_range =>
                        Dict(gti_example.bin_time =>
                            Dict(instrument =>
                                log_entries
                            )
                        )
                    )
                )
            )
        )
    end
    @info "Saving $instrument gti data finished"
end

"""
    _gtis_load(gti_dir, instrument, bin_time)

Loads and parses the `_meta` and `_gti` files, puts into a `GTIData` constructor, returns `Dict{Int,GTIData}`
"""
function _gtis_load(gti_dir::String, instrument, e_range::Tuple{Float64,Float64}, bin_time)
    @info "Loading $instrument gti data"
    bin_time = float(bin_time)

    gti_basename  = string("$(instrument)_lc_$(bin_time)_gti")
    gti_meta_path = joinpath(gti_dir, "$(gti_basename)_meta.feather")

    gti_meta = Feather.read(gti_meta_path)

    gti_data = Dict{Int,GTIData}()

    for gti_row in DataFrames.eachrow(gti_meta)
        gti_mission = _mission_symbol_to_type(Symbol(gti_row[:mission]))
        gti_inst    = Symbol(gti_row[:instrument])
        gti_obsid   = gti_row[:obsid]
        gti_bin_t   = gti_row[:bin_time]
        gti_idx     = gti_row[:indecies]
        gti_starts  = gti_row[:starts]
        current_gti = Feather.read(joinpath(gti_dir, "$(gti_basename)_$(gti_idx).feather"))
        gti_counts  = current_gti[:counts]
        gti_times   = current_gti[:times]
        gti_times   = gti_times[1]:gti_bin_t:gti_times[end] # Convert Array to Step Range
        gti_group   = gti_row[:groups]

        gti_data[gti_idx] = GTIData(gti_mission, gti_inst, gti_obsid, e_range, gti_bin_t, gti_idx,
            gti_starts, gti_counts, gti_times, gti_group)
    end

    @info "Loading $instrument gti data finished"

    return gti_data
end

"""
    gtis(mission_name::Symbol, obs_row::DataFrames.DataFrame, bin_time::Number;  =false)

Handles file management, checks to see if GTI files exist already and loads them, if files do not
exist then the `_gits` function is ran, then the data is saved

TODO: FIX E_RANGE PROPAGATION WHEN LCURVE_DATA IS GIVEN
"""
function gtis(mission::Mission, obs_row::DataFrames.DataFrameRow{DataFrames.DataFrame}, bin_time::Number; overwrite=false,
        e_range::Tuple{Float64,Float64}=_mission_good_e_range(mission),
        lcurve_data::Dict{Symbol,BinnedData}=Dict{Symbol,BinnedData}()
    )
    obsid       = obs_row[:obsid]
    instruments = _mission_instruments(mission)
    gti_files   = _log_query(mission, obs_row, "data", :gtis, e_range, bin_time)
    gtis_path   = abspath(_obs_path_local(mission, obs_row; kind=:jaxtam), "JAXTAM", _log_query_path(; category=:data, kind=:gtis, e_range=e_range, bin_time=bin_time))

    gtis = Dict{Symbol,Dict{Int64,JAXTAM.GTIData}}()
    for instrument in instruments
        if ismissing(gti_files) || !haskey(gti_files, instrument) || overwrite
            @info "Missing gti files for $instrument"

            if !haskey(lcurve_data, instrument)
                lcurve_data = lcurve(mission, obs_row, bin_time, e_range=e_range)
            end

            @assert _recursive_first(lcurve_data).e_range == e_range "lcurve energy range does not match input: $(_recursive_first(gtis_data).e_range) != $e_range"

            @info "Selecting GTIs for $instrument"
            gtis_data = _gtis(lcurve_data[Symbol(instrument)])

            _gtis_save(mission, obs_row, instrument, e_range, gtis_path, gtis_data)

            gtis[instrument] = gtis_data
        else
            gtis[instrument] = _gtis_load(gtis_path, instrument, e_range, bin_time)
        end
    end

    if ismissing(_log_query(mission, obs_row, "meta", :countrates, e_range))
        countrates = Dict{Symbol,Float64}()
        for instrument in instruments
            countrates[instrument] = mean([sum(gti[2].counts)/gti[2].times[end] for gti in gtis[instrument]])
        end
        _log_add(mission, obs_row, Dict("meta" => Dict(:countrates => Dict(e_range => countrates))))
    end

    return gtis
end

function gtis(mission::Mission, obsid::String, bin_time::Number; overwrite=false)
    obs_row = master_query(mission, :obsid, obsid)

    return gtis(mission, obs_row, bin_time, overwrite=overwrite)
end


# This file was generated, do not modify it. # hide
best_ensemble = fitted_params(tuned_ensemble).best_model
@show best_ensemble.atom.K
@show best_ensemble.bagging_fraction


struct IterationOutput{M, R}
    solution::M
    value::R
    gradient::M
end

to_iterator(x::Any) = x
to_iterator(n::T) where T <: Number = Iterators.repeated(n)

include("optimizers/gradient_descent.jl")
include("optimizers/polyak.jl")
include("optimizers/nesterov.jl")
include("optimizers/barzilai_borwein.jl")
include("optimizers/adagrad.jl")
include("optimizers/rmsprop.jl")
include("optimizers/adadelta.jl")
include("optimizers/adam.jl")


# Create object for making move (Rock, Paper, Scissors)
# One perk of this structure is that we can add new moves with slight changes to this struct anytime!
# e.g. Rock, Paper, Scissors, Spock, Lizard and other derivatives of Rock Paper Scissors
struct Thing
    name::String
    wins_with::String
end

# Create possible move objects
scissors = Thing("scissors", "paper")
rock = Thing("rock", "scissors")
paper = Thing("paper", "rock")

# Define winner check function
check_winner(player1::Thing, player2::Thing) = (player1.name == player2.name) ? 0 : (player1.wins_with == player2.name ?  1 : 2)

# Function to change string to a move object
function to_thing(str)
    arr = [scissors, rock, paper]
    return arr[findfirst(x -> x.name == str, arr)]
end

# Function to play_again
function play_again()
    (ask_play_again("Play again? (y/n): ") == "y") ? run(`julia $PROGRAM_FILE`) : exit()
end

# Input part of play_again
function ask_play_again(prompt)
    print(prompt)
    txt = split(lowercase(readline()))
    ((length(txt) > 1) || !(txt[1] in ["y", "n"])) ? ask_play_again(prompt) : return txt[1]
end

# Get input from user, and scrub input
function get_input(prompt)
    print(prompt)
    txt = split(lowercase(readline()))
    ((length(txt) > 1) || !(txt[1] in ["rock", "scissors", "paper"])) ? get_input(prompt) : return txt[1]
end

# Create points for players
AI_points = 0
human_points = 0

while (AI_points + human_points) < 3
    global AI_points, human_points
    # Ask for input and get move by user
    player1_move = to_thing(get_input("Enter your move (Rock, Paper or Scissors): "))

    # Make AI make random move
    player2_move = rand([rock, scissors, paper])

    # Show user the move the AI made
    println("AI: ", player2_move.name)

    # Check winner
    result = check_winner(player1_move, player2_move)

    # Print result. If tie, then game continues, since game counter is sum of AI_points and human_points, the game goes on one more time
    (result == 1) ? (println("You win"); human_points += 1) : (result == 2) ? (println("AI wins"); AI_points += 1) : (println("Tie. No one gets a point. Play again.\n"); continue)

    # Print current standings
    println("Current points - You: $human_points, AI: $AI_points\n")

    # If any player gets 2 points out of 3, playing game again is pointless, so let the player with 2 points win.
    max(AI_points, human_points) == 2 ? break : ""
end

# Print game result
(AI_points > human_points) ? println("\nGAME! AI Wins.") : println("\nGAME! You Win.")

# Ask if user wants to play again
play_again()


isdefined(Base, :__precompile__) && __precompile__()

module CaseModule

using DataFrames

export
      setConfig,
      case2dfs

"""
setConfig(c)
--------------------------------------------------------------------------------------\n
Author: Huckleberry Febbo, Graduate Student, University of Michigan
Date Create: 2/25/2018, Last Modified: 2/25/2018 \n
--------------------------------------------------------------------------------------\n
"""
function setConfig(c, key_name; kwargs...)
  for (key,value) in kwargs
    if haskey(c[key_name],string(key))
      c[key_name][string(key)]=value
    else
      error(string(" \n Unknown key: ", key, " for ", c[key_name], " used in setMisc() \n "))
    end
  end
  return c
end

"""
case2dfs(c)
--------------------------------------------------------------------------------------\n
Author: Huckleberry Febbo, Graduate Student, University of Michigan
Date Create: 3/11/2017, Last Modified: 2/15/2018 \n
--------------------------------------------------------------------------------------\n
"""

function case2dfs(c)
    dfs = DataFrame()

    ##############
    # weights
    dfs[:wgoal] = c["weights"]["goal"]
    dfs[:wpsi] = c["weights"]["psi"] # 0
    dfs[:wtime] = c["weights"]["time"]
    dfs[:whaf] = c["weights"]["haf"] # 0
    dfs[:wce] = c["weights"]["ce"]
    dfs[:csa] = c["weights"]["sa"]
    dfs[:wsr] = c["weights"]["sr"]
    dfs[:wjx] = c["weights"]["jx"]
    dfs[:path] = c["weights"]["path"]
    dfs[:driver] = c["weights"]["driver"]
    # weights
    ##############

    ##############
    # tolerances
    # initial state tolerances
    dfs[:ixt] = c["tolerances"]["ix"]
    dfs[:iyt] = c["tolerances"]["iy"]
    dfs[:ivt] = c["tolerances"]["iv"]
    dfs[:irt] = c["tolerances"]["ir"]
    dfs[:ipsit] = c["tolerances"]["ipsi"]
    dfs[:isat] = c["tolerances"]["isa"]
    dfs[:iut] = c["tolerances"]["iu"]
    dfs[:iaxt] = c["tolerances"]["iax"]

    # final state tolerances
    dfs[:fxt] = c["tolerances"]["fx"]
    dfs[:fyt] = c["tolerances"]["fy"]
    dfs[:fvt] = c["tolerances"]["fv"]
    dfs[:frt] = c["tolerances"]["fr"]
    dfs[:fpsit] = c["tolerances"]["fpsi"]
    dfs[:fsat] = c["tolerances"]["fsa"]
    dfs[:fut] = c["tolerances"]["fu"]
    dfs[:faxt] = c["tolerances"]["fax"]
    # tolerances
    ##################

    #####################
    # solver settings
    dfs[:outlev] = c["solver"]["outlev"]
    dfs[:feastolAbs] = c["solver"]["feastol_abs"]
    dfs[:maxit] = c["solver"]["maxit"]
    dfs[:maxtimeCpu] = c["solver"]["maxtime_cpu"]
    dfs[:ftol] = c["solver"]["ftol"]
    dfs[:feastol] = c["solver"]["feastol"]
    dfs[:infeastol] = c["solver"]["infeastol"]
    dfs[:maxfevals] = c["solver"]["maxfevals"]
    dfs[:maxtimeReal] = c["solver"]["maxtime_real"]
    dfs[:opttol] = c["solver"]["opttol"]
    dfs[:opttolAbs] = c["solver"]["opttol_abs"]
    dfs[:xtol] = c["solver"]["xtol"]
    dfs[:acceptableObjChangeTol] = c["solver"]["acceptable_obj_change_tol"]
    dfs[:warmStartInitPoint] = c["solver"]["warm_start_init_point"]
    dfs[:dualInfTol] = c["solver"]["dual_inf_tol"]
    dfs[:acceptableTol] = c["solver"]["acceptable_tol"]
    dfs[:acceptableConstrViolTol] = c["solver"]["acceptable_constr_viol_tol"]
    dfs[:acceptableDualInfTol] = c["solver"]["acceptable_dual_inf_tol"]
    dfs[:acceptableComplInfTol] = c["solver"]["acceptable_compl_inf_tol"]
    dfs[:acceptableObjChangeTol] = c["solver"]["acceptable_obj_change_tol"]
    dfs[:divergingIteratesTol] = c["solver"]["diverging_iterates_tol"]
    # solver settings
    #####################

    ####################
    # obstacles
    dfs[:radius] = string(c["obstacle"]["radius"]')
    dfs[:sX] = string(c["obstacle"]["vx"]')
    dfs[:sy] = string(c["obstacle"]["vy"]')
    dfs[:Xi] = string(c["obstacle"]["x0"]')
    dfs[:Yi] = string(c["obstacle"]["y0"]')
    #dfs[:status] = string(c["obstacle"]["status"]')
    # obstacles
    ####################

    ###############
    # goal
    dfs[:xRef] = c["goal"]["x"]
    dfs[:yRef] = c["goal"]["yVal"]
    dfs[:psiRef] = c["goal"]["psi"]
    dfs[:goalTol] = c["goal"]["tol"]
    # goal
    ###############

    ###################
    # X0 parameters
    dfs[:xi] = c["X0"]["x"]
    dfs[:yi] = c["X0"]["yVal"]
    dfs[:vi] = c["X0"]["v"]
    dfs[:ri] = c["X0"]["r"]
    dfs[:psii] = c["X0"]["psi"]
    dfs[:sai] = c["X0"]["sa"]
    dfs[:ui] = c["X0"]["ux"]
    dfs[:axi] = c["X0"]["ax"]
    # X0 parameters
    ###################

    ###################
    # misc. parameters
    dfs[:model] = c["misc"]["model"]
    dfs[:Xmin] = c["misc"]["Xmin"]
    dfs[:Xmax] = c["misc"]["Xmax"]
    dfs[:Ymin] = c["misc"]["Ymin"]
    dfs[:Ymax] = c["misc"]["Ymax"]
    dfs[:tp] = c["misc"]["tp"]
    dfs[:tex] = c["misc"]["tex"]
    dfs[:sms] = c["misc"]["sm"]
    dfs[:smh] = c["misc"]["sm2"]
    dfs[:Lr] = c["misc"]["Lr"]
    dfs[:Lrd] = c["misc"]["L_rd"]

    if c["misc"]["integrationScheme"]==:lgrExplicit || c["misc"]["integrationScheme"]==:lgrImplicit
        dfs[:NI] = length(c["misc"]["Nck"])
        dfs[:colPts] = sum(c["misc"]["Nck"])
        dfs[:Nck] = string(c["misc"]["Nck"]')
    else
        dfs[:NI] = NaN
        dfs[:colPts] = c["misc"]["N"]
        dfs[:Nck] = NaN
    end
    dfs[:solver] = c["misc"]["solver"]
    dfs[:MPCmaxIter] = c["misc"]["mpc_max_iter"]
    dfs[:predictX0] = c["misc"]["PredictX0"]
    dfs[:fixedTp] = c["misc"]["FixedTp"]
    #dfs[:activeSafety] = c["misc"]["activeSafety"]
    #dfs[:followDriver] = c["misc"]["followDriver"]
    #dfs[:followPath] = c["misc"]["followPath"]
    #dfs[:NF] = c["misc"]["NF"]
    dfs[:integrationScheme] = c["misc"]["integrationScheme"]
    dfs[:tfMax] = c["misc"]["tfMax"]
    # misc. parameters
    ###################

    return dfs
end

end # module


using Dates
using TimeFrames
using TimeFrames: TimePeriodFrame, DatePeriodFrame
using TimeFrames: period_step, _period_step
using TimeFrames: CustomTimeFrame, tonext

using Test

@testset "low level (period frame)" begin
    @testset "TimePeriodFrame" begin
        tf = TimePeriodFrame{Dates.Hour}()
        @test tf.period.value == 1
        @test tf.boundary == Begin

        tf = TimePeriodFrame{Dates.Hour}(5)
        @test tf.period.value == 5
        @test tf.boundary == Begin

        tf2 = TimePeriodFrame{Dates.Hour}(5)
        @test tf == tf2

        tf3 = TimePeriodFrame{Dates.Hour}(3)
        @test tf != tf3
    end

    @testset "DatePeriodFrame" begin
        tf = DatePeriodFrame{Dates.Month}()
        @test tf.period.value == 1
        @test tf.boundary == Begin

        tf = DatePeriodFrame{Dates.Month}(5)
        @test tf.period.value == 5
        @test tf.boundary == Begin

        tf2 = DatePeriodFrame{Dates.Month}(5)
        @test tf == tf2

        tf3 = DatePeriodFrame{Dates.Month}(3)
        @test tf != tf3
    end

    @testset "NoTimeFrame" begin
        tf = TimeFrame()
        @test typeof(tf) == NoTimeFrame

        tf2 = TimeFrame()
        @test tf == tf2
    end

    @testset "_period_step" begin
        @test _period_step(DateTime) == period_step
        @test _period_step(Date) == Dates.Day(1)
    end
end

@testset "high level" begin
    @testset "YearEnd, Minute, ..." begin
        tf = YearEnd()
        @test tf.period.value == 1

        @test YearEnd() == YearEnd()
        @test YearEnd() != YearBegin()
        @test YearEnd() != YearEnd(5)

        tf = TimeFrames.Minute()
        @test tf.period.value == 1

        tf = TimeFrames.Minute(15)
        @test tf.period.value == 15

        tf1 = TimeFrames.Minute(15)
        tf2 = TimeFrames.Minute(15)
        @test tf1 == tf2

        tf1 = TimeFrames.Minute(15)
        tf2 = TimeFrames.Minute(30)
        @test tf1 != tf2
    end

    @testset "to string" begin
        tf = TimeFrames.Minute()
        @test String(tf) == "T"

        tf = TimeFrames.Minute(15)
        @test String(tf) == "15T"
    end

    @testset "parse" begin

        @testset "simple parse" begin
            tf = TimeFrame("15T")
            @test tf.period.value == 15
            @test typeof(tf) == TimeFrames.Minute

            tf = TimeFrame("T")
            @test tf.period.value == 1
            @test typeof(tf) == TimeFrames.Minute

            tf = TimeFrame("15Min")
            @test tf.period.value == 15
            @test String(tf) == "15T"
            @test typeof(tf) == TimeFrames.Minute

            tf = TimeFrame("5H")
            @test tf.period.value == 5
            @test typeof(tf) == TimeFrames.Hour
        end

        @testset "NoTimeFrame parse" begin
            tf = TimeFrame("")
            @test typeof(tf) == NoTimeFrame
            @test NoTimeFrame() == NoTimeFrame(1,2,3)
        end

        @testset "boundary" begin
            tf = TimeFrame("3A")
            @test tf == YearEnd(3)
            @test tf.period == Dates.Year(3)
            @test tf.boundary == End

            tf = TimeFrame("3AS")
            @test tf == YearBegin(3)
            @test tf.period == Dates.Year(3)
            #@test tf.boundary == Begin  # ToFix

            tf = TimeFrame("3M")
            @test tf == MonthEnd(3)
            @test tf.period == Dates.Month(3)
            @test tf.boundary == End

            tf = TimeFrame("3MS")
            @test tf == MonthBegin(3)
            @test tf.period == Dates.Month(3)
            #@test tf.boundary == Begin  # ToFix
        end

        @testset "*" begin
            @testset "TimePeriodFrame * n / n * TimePeriodFrame" begin
                tf = TimeFrame("2H")
                @test tf * 3 == TimeFrame("6H")

                tf = TimeFrame("2H")
                @test 3 * tf == TimeFrame("6H")
            end
            @testset "DatePeriodFrame * n / n * DatePeriodFrame" begin
                tf = TimeFrame("2M")
                @test tf * 3 == TimeFrame("6M")

                tf = TimeFrame("2M")
                @test 3 * tf == TimeFrame("6M")
            end
        end

        @testset "grouper/apply" begin
            @test apply(MonthEnd(), Date(2010, 2, 20)) == Date(2010, 2, 28)
            @test apply(MonthEnd(), DateTime(2010, 2, 20)) == DateTime(2010, 3, 1) - period_step

            d = Date(2016, 7, 20)
            dt = DateTime(2016, 7, 20, 13, 24, 35, 245)

            tf = TimeFrame(dt -> floor(dt, Dates.Minute(15)))  # custom TimeFrame with lambda function as DateTime grouper
            @test apply(tf, dt) == DateTime(2016, 7, 20, 13, 15, 0, 0)
            @test typeof(tf) == CustomTimeFrame

            tf = TimeFrame(Dates.Minute(15))  # TimePeriodFrame using TimePeriod
            @test apply(tf, dt) == DateTime(2016, 7, 20, 13, 15, 0, 0)

            tf = TimeFrame(Dates.Day(1))  # TimePeriodFrame using DatePeriod
            @test apply(tf, dt) == DateTime(2016, 7, 20, 0, 0, 0, 0)

            tf = YearBegin()
            #@test apply(tf, dt) == DateTime(2016, 1, 1, 0, 0, 0, 0)
            @test apply(tf, dt) == Date(2016, 1, 1)
            #@test typeof(f_group(dt)) == Date

            #tf = YearEnd(boundary=End)
            #@test apply(tf, dt) == DateTime(2016, 1, 1, 0, 0, 0, 0)
            #@test apply(tf, dt) == Date(2016, 1, 1)

            tf = MonthBegin()
            #@test apply(tf, dt) == DateTime(2016, 7, 1, 0, 0, 0, 0)
            @test apply(tf, dt) == Date(2016, 7, 1)

            tf = TimeFrames.Week()
            #@test apply(tf, dt) == DateTime(2016, 7, 18, 0, 0, 0, 0)
            @test apply(tf, dt) == Date(2016, 7, 18)

            tf = TimeFrames.Day()
            #@test apply(tf, dt) == DateTime(2016, 7, 20, 0, 0, 0, 0)
            @test apply(tf, dt) == Date(2016, 7, 20)

            tf = TimeFrames.Hour()
            @test apply(tf, dt) == DateTime(2016, 7, 20, 13, 0, 0, 0)

            tf = TimeFrames.Minute()
            @test apply(tf, dt) == DateTime(2016, 7, 20, 13, 24, 0, 0)

            tf = TimeFrames.Second()
            @test apply(tf, dt) == DateTime(2016, 7, 20, 13, 24, 35, 0)

            tf = TimeFrames.Millisecond()
            @test apply(tf, dt) == DateTime(2016, 7, 20, 13, 24, 35, 245)

            tf = YearBegin(10)
            @test apply(tf, dt) == DateTime(2010, 1, 1, 0, 0, 0, 0)

            tf = YearEnd(10)
            @test apply(tf, d) == DateTime(2019, 12, 31)
            @test apply(tf, dt) == DateTime(2020, 1, 1) - period_step

            tf = TimeFrames.Minute(15)
            @test apply(tf, dt) == DateTime(2016, 7, 20, 13, 15, 0, 0)
        end

        @testset "tonext" begin
            dt = DateTime(2010, 1, 1, 10, 30)
            @test tonext(TimeFrame("2H"), dt) == DateTime(2010, 1, 1, 12, 0)

            dt = DateTime(2010, 1, 1, 0, 0)  # ; same=false
            @test tonext(TimeFrame("2H"), dt) == DateTime(2010, 1, 1, 2, 0)

            dt = DateTime(2010, 1, 1, 0, 0)
            @test tonext(TimeFrame("2H"), dt; same=true) == DateTime(2010, 1, 1, 0, 0)

            dt = DateTime(2010, 1, 1, 10, 30)
            @test tonext(TimeFrame("1D"), dt) == DateTime(2010, 1, 2, 0, 0)

            dt = DateTime(2010, 1, 1, 10, 30)
            @test tonext(TimeFrame("1MS"), dt) == DateTime(2010, 2, 1, 0, 0)

            dt = DateTime(2010, 1, 1, 10, 30)
            @test tonext(TimeFrame("1M"), dt) == DateTime(2010, 2, 1) - period_step

            dt = DateTime(2010, 6, 5, 10, 30)
            @test tonext(TimeFrame("1A"), dt) == DateTime(2011, 1, 1) - period_step

            dt = DateTime(2010, 6, 5, 10, 30)
            @test tonext(TimeFrame("1AS"), dt) == DateTime(2011, 1, 1)

          end

        @testset "range" begin
            dt1 = DateTime(2010, 1, 1, 20)
            dt2 = DateTime(2010, 1, 14, 16)
            tf = Dates.Day(1)
            rng = range(dt1, TimeFrame(tf), dt2)
            @test rng[1] == DateTime(2010, 1, 1)
            @test rng[end] == DateTime(2010, 1, 14)

            rng = range(dt1, TimeFrame(tf), dt2, apply_tf=false)
            @test rng[1] == DateTime(2010, 1, 1, 20)
            @test rng[end] == DateTime(2010, 1, 13, 20)

            tf = Dates.Day(1)
            N = 5
            rng = range(dt1, TimeFrame(tf), N)
            @test length(rng) == N
            @test rng[1] == DateTime(2010, 1, 1, 20)
            @test rng[end] == DateTime(2010, 1, 5, 20)

            tf = Dates.Day(1)
            N = 5
            rng = range(TimeFrame(tf), dt2, N)
            @test length(rng) == N
            @test rng[end] == DateTime(2010, 1, 13, 16)
            @test rng[1] == DateTime(2010, 1, 9, 16)

            dt1 = DateTime(2010, 1, 1, 20)
            dt2 = DateTime(2010, 1, 14, 16)
            tf = NoTimeFrame()
            rng = range(dt1, tf, dt2)
            @test length(rng) == 1
            @test rng[1] == dt1
        end

        @testset "@tf" begin
            @test tf"1T" == TimeFrame("1T")
            @test tf"1M" == TimeFrame("M")
        end  # @testset "@tf
    end

    @testset "+" begin
        d = Date(2017, 12, 1)
        let tf = tf"1A", ans = Date(2018, 12, 1)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1AS", ans = Date(2018, 12, 1)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1M", ans = Date(2018, 1, 1)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1MS", ans = Date(2018, 1, 1)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1W", ans = Date(2017, 12, 8)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1D", ans = Date(2017, 12, 2)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1H", ans = DateTime(2017, 12, 1, 1, 0, 0, 0)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1T", ans = DateTime(2017, 12, 1, 0, 1, 0, 0)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1S", ans = DateTime(2017, 12, 1, 0, 0, 1, 0)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1L", ans = DateTime(2017, 12, 1, 0, 0, 0, 1)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        d = Dates.Time(0, 1, 2, 3)
        let tf = tf"1H", ans = Dates.Time(1, 1, 2, 3)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1T", ans = Dates.Time(0, 2, 2, 3)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1S", ans = Dates.Time(0, 1, 3, 3)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        let tf = tf"1L", ans = Dates.Time(0, 1, 2, 4)
            @test isdefined(tf, :period)
            @test d + tf == ans
            @test tf + d == ans
            @inferred d + tf
            @inferred tf + d
        end

        d = Dates.Time(0, 1, 2, 3)
        @test_throws InexactError d + tf"1A"
        @test_throws InexactError d + tf"1AS"
        @test_throws InexactError d + tf"1M"
        @test_throws InexactError d + tf"1MS"
        @test_throws InexactError d + tf"1W"
        @test_throws InexactError d + tf"1D"

        @test_throws InexactError tf"1A"  + d
        @test_throws InexactError tf"1AS" + d
        @test_throws InexactError tf"1M"  + d
        @test_throws InexactError tf"1MS" + d
        @test_throws InexactError tf"1W"  + d
        @test_throws InexactError tf"1D"  + d
    end  # @testset "+"

    @testset "-" begin
        d = Date(2017, 12, 1)
        let tf = tf"1A"
            @test isdefined(tf, :period)
            @test d - tf == Date(2016, 12, 1)
            @inferred d - tf
        end

        let tf = tf"1AS"
            @test isdefined(tf, :period)
            @test d - tf == Date(2016, 12, 1)
            @inferred d - tf
        end

        let tf = tf"1M"
            @test isdefined(tf, :period)
            @test d - tf == Date(2017, 11, 1)
            @inferred d - tf
        end

        let tf = tf"1MS"
            @test isdefined(tf, :period)
            @test d - tf == Date(2017, 11, 1)
            @inferred d - tf
        end

        let tf = tf"1W"
            @test isdefined(tf, :period)
            @test d - tf == Date(2017, 11, 24)
            @inferred d - tf
        end

        let tf = tf"1D"
            @test isdefined(tf, :period)
            @test d - tf == Date(2017, 11, 30)
            @inferred d - tf
        end

        let tf = tf"1H"
            @test isdefined(tf, :period)
            @test d - tf == DateTime(2017, 11, 30, 23, 0, 0, 0)
            @inferred d - tf
        end

        let tf = tf"1T"
            @test isdefined(tf, :period)
            @test d - tf == DateTime(2017, 11, 30, 23, 59, 0, 0)
            @inferred d - tf
        end

        let tf = tf"1S"
            @test isdefined(tf, :period)
            @test d - tf == DateTime(2017, 11, 30, 23, 59, 59, 0)
            @inferred d - tf
        end

        let tf = tf"1L"
            @test isdefined(tf, :period)
            @test d - tf == DateTime(2017, 11, 30, 23, 59, 59, 999)
            @inferred d - tf
        end

        d = Dates.Time(0, 1, 2, 3)
        let tf = tf"1H"
            @test isdefined(tf, :period)
            @test d - tf == Dates.Time(23, 1, 2, 3)
            @inferred d - tf
        end

        let tf = tf"1T"
            @test isdefined(tf, :period)
            @test d - tf == Dates.Time(0, 0, 2, 3)
            @inferred d - tf
        end

        let tf = tf"1S"
            @test isdefined(tf, :period)
            @test d - tf == Dates.Time(0, 1, 1, 3)
            @inferred d - tf
        end

        let tf = tf"1L"
            @test isdefined(tf, :period)
            @test d - tf == Dates.Time(0, 1, 2, 2)
            @inferred d - tf
        end

        d = Dates.Time(0, 1, 2, 3)
        @test_throws InexactError d - tf"1A"
        @test_throws InexactError d - tf"1AS"
        @test_throws InexactError d - tf"1M"
        @test_throws InexactError d - tf"1MS"
        @test_throws InexactError d - tf"1W"
        @test_throws InexactError d - tf"1D"
    end  # @testset "-"
end  # @testset "high level"


using LinearAlgebra

module JuliaPetra

# Internal Utilities
include("Utils.jl")

# Communication interface
include("Comm.jl")
include("LocalComm.jl")
include("Distributor.jl")
include("Directory.jl")

include("BlockMapData.jl")
include("BlockMap.jl")

include("BasicDirectory.jl")
include("DirectoryMethods.jl")

# Communication implementations
include("SerialComm.jl")

# Data interface
include("ImportExportData.jl")
include("Import.jl")
include("Export.jl")

include("DistObject.jl")

# Dense Data types
include("MultiVector.jl")
include("DenseMultiVector.jl")

# Sparse Data types
include("Operator.jl")
include("RowGraph.jl")
include("RowMatrix.jl")

include("SparseRowView.jl")
include("LocalCSRGraph.jl")
include("LocalCSRMatrix.jl")

include("CSRGraphConstructors.jl")
include("CSRGraphInternalMethods.jl")
include("CSRGraphExternalMethods.jl")

include("CSRMatrix.jl")

function __init__()
    #code dependant on MPI.jl can't be precompiled
    sourcedir = dirname(@__FILE__)
    include(joinpath(sourcedir, "MPIUtil.jl"))
    include(joinpath(sourcedir, "MPIComm.jl"))
    include(joinpath(sourcedir, "MPIDistributor.jl"))
end

end # module


module EMLreboiler_cost
	using DanaTypes
	using DotPlusInheritance
	using Reexport
	@reexport using ...stage_separators.EMLreboiler
	import EMLtypes.length
	include("reboiler_cost/reboiler_cost.jl")
end


using CuArrays, Test

include("util.jl")

using Random
Random.seed!(1)

using CUDAnative
using CUDAdrv

# GPUArrays has a testsuite that isn't part of the main package.
# Include it directly.
import GPUArrays
gpuarrays = pathof(GPUArrays)
gpuarrays_root = dirname(dirname(gpuarrays))
include(joinpath(gpuarrays_root, "test", "testsuite.jl"))

testf(f, xs...; kwargs...) = TestSuite.compare(f, CuArray, xs...; kwargs...)

import CuArrays: allowscalar, @allowscalar
allowscalar(false)

@testset "CuArrays" begin

@testset "GPUArrays test suite" begin
  TestSuite.test(CuArray)
end

include("base.jl")
include("blas.jl")
include("rand.jl")
include("fft.jl")
include("sparse.jl")
include("solver.jl")
include("sparse_solver.jl")
include("dnn.jl")
include("tensor.jl")
include("forwarddiff.jl")

CuArrays.memory_status()
CuArrays.pool_timings()
CuArrays.alloc_timings()
CuArrays.reset_timers!()

end


struct PTrie{N}
    mask::UInt
    slots::BitSet
end

function PTrie{N}() where {N}
    slots = BitSet()
    mask = sum(1 << i for i in 0:(N - 1))
    PTrie{N}(mask, slots)
end

function Base.push!(p::PTrie, collection)
    h = zero(UInt)
    @inbounds for element in collection
        h = hash(element, h)
        push!(p.slots, h & p.mask)
    end
end

function Base.in(collection, p::PTrie)
    h = zero(UInt)
    @inbounds for element in collection
        h = hash(element, h)
        if (h & p.mask) ∉ p.slots
            return false
        end
    end
    (h & p.mask) in p.slots
end

function Base.getindex(p::PTrie, collection)
    h = zero(UInt)
    for element in collection
        h = hash(element, h)
        if (h & p.mask) ∉ p.slots
            return nothing
        end
    end
    return h
end

function has_concatenation(p::PTrie, h::UInt, suffix)
    @inbounds for element in suffix
        h = hash(element, h)
        if (h & p.mask) ∉ p.slots
            return false
        end
    end
    (h & p.mask) in p.slots
end

function has_concatenation(p::PTrie, collections::Vararg{String, N}) where {N}
    h = zero(UInt)
    for collection in collections
        @inbounds for element in collection
            h = hash(element, h)
            # if !p.slots[(h & p.mask) + 1]
            if (h & p.mask) ∉ p.slots
                return false
            end
        end
    end
    (h & p.mask) in p.slots
end


procs = addprocs(1)

push!(LOAD_PATH,joinpath("..","src"))
push!(LOAD_PATH,joinpath("..","..","..","POMCP.jl","src"))
using Multilane
using POMCP
using POMDPToolbox
using GenerativeModels
using MCTS
using JLD

function POMCP.extract_belief(::POMDPToolbox.FastPreviousObservationUpdater{MLObs}, node::RootNode)
  rand(MersenneTwister(1),node.B)
end

POMCP.initialize_belief(u::FastPreviousObservationUpdater{MLObs}, o::Union{MLState,MLObs}) = o

POMCP.create_belief(u::FastPreviousObservationUpdater{MLObs}) = nothing

POMCP.extract_belief(::POMDPToolbox.FastPreviousObservationUpdater{MLObs}, node::BeliefNode) = node.label[2]

function write!(var_name::AbstractString, val::NoCrashStats, fname::AbstractString="results.jld")
  # XXX kinda hacky but AFAIK JLD has no append functionality
  # NOTE im not sure how to use this functionality AND do the addrequire stuff
  if !isfile(fname)
    d = Dict{ByteString,Any}()
  else
    d = load(fname)
  end
  if var_name in keys(d)
    warn("trying to overwrite value, adding spaghetti")
    var_name = string(var_name,"99999")
  end
  d[var_name] = val
  # TODO addrequire
  save(fname, d)
end

function test_solver(problems::Array{NoCrashMDP,1}, solver, solver_name::AbstractString, initial_states::Array{MLState,1})
  if typeof(solver) <: DPWSolver
    _ec = solver.exploration_constant
    best_score = -Inf
    for (i,ec) in enumerate(Float64[0.1,0.3,1.,3.,10.,30.,100.])
      solver.exploration_constant = ec
      _r = Multilane.evaluate_performance(problems, initial_states, solver)
      score = mean(_r)[3]
      if score > best_score
        r = _r
        best_score = score
      end
    end
    solver.exploration_constant = _ec
  else
    r = Multilane.evaluate_performance(problems, initial_states, solver)
  end
  # TODO hyperparameter optimization here, save best
  write!(solver_name, r)
  return r
end

function test_solver(problems::Array{NoCrashPOMDP,1}, solver, solver_name::AbstractString, initial_states::Array{MLState,1}, bu)
  r = Multilane.evaluate_performance(problems, initial_states, solver, bu)
  write!(solver_name, r)
  return r
end

function test_pomdp(problem::NoCrashPOMDP, solver, solver_name::AbstractString, initial_states, bu)
  r = NoCrashStats([test_run(problem, bu, s, solver) for s in initial_states], problem.rmodel)
  write!(solver_name, r)
  return r
end

nb_lanes = 4
pp = PhysicalParam(nb_lanes,lane_length=100.) #2.=>col_length=8\n",
_discount = 1.
nb_cars=10
dmodel = NoCrashIDMMOBILModel(nb_cars, pp)

N = 10

models = JLD.load("rmodels.jld")
rmodels = models["rmodels"]

# NOTE: random solver and heuristic solver are rmodel agnostic--won't get pareto curve
rmodel = rmodels[1]
mdp = NoCrashMDP(dmodel, rmodel, _discount);
isrng = MersenneTwister(123)
# Initial state is indifferent to rmodel
initial_states = MLState[initial_state(mdp, isrng) for i in 1:N]
problems = NoCrashMDP[mdp for i in 1:N]

#r_random = test_solver(problems, RandomSolver(), "random", initial_states)

#println("Random Average Reward: $(mean(r_random))")

#r_heur = test_solver(problems, SimpleSolver(), "simple_heuristic", initial_states)

#println("Heuristic Average Reward: $(mean(r_heur))")

# exploration constants for POMCP
ecs = [3., 5., 20., 50., 150., 300., 600., 1500., 3000., 5000.]

for (k,(rmodel,ec)) in enumerate(zip(rmodels,ecs))
  mdp = NoCrashMDP(dmodel, rmodel, _discount);

  problems = NoCrashMDP[mdp for i in 1:N]

  dpws = DPWSolver(depth=10,n_iterations=100) #k_action, alpha_action, k_state, alpha_state

  #r_dpw = test_solver(problems, dpws, string("dpw", k), initial_states)

  #println("DPW Average Reward: $(mean(r_dpw))")

  single_dpws = SingleBehaviorSolver(dpws, IDMMOBILBehavior("normal", 30.0, 10.0, 1))

  #r_single_dpw = test_solver(problems, single_dpws, string("single_dpw", k), initial_states)

  #println("Single Behavior DPW Average Reward: $(mean(r_single_dpw))")

  # POMDP stuff

  pomdp = NoCrashPOMDP(dmodel, rmodel, _discount)

  problems = NoCrashPOMDP[pomdp for i in 1:N]

  bu = ParticleUpdater(100, pomdp, MersenneTwister(555))

  pomcp = POMCPDPWSolver( k_observation=0.3,
                          alpha_observation=0.75,
                          k_action=0.5,
                          alpha_action=5.,
                          tree_queries=100,
                          c=ec,
                          eps=0.01)
                          #rollout_solver=SimpleSolver())

                          # Can't use simple solver! Issue is that somehow POMCPDPW is somehow generating an invalid action...

  #pomcp = POMCPSolver()
  r_pomcp = test_pomdp(pomdp, pomcp, string("pomcp", k), initial_states, bu)

  println("POMCP Avg Reward: $(mean(r_pomcp))")

  #r_despot = test_solver(problems, DespotSolver(pomdp,belief), string("dpw", k), initial_states)

  #println("DESPOT Avg Reward: $(mean(r_despot))")

end

rmprocs(procs)


"""
function addEventDeath_!(p::Program_,abm::Agent,space::SimulationSpace,platform::String)

Generate the functions for division events.
"""
function addEventDeath_!(p::Program_,abm::Agent,space::SimulationSpace,platform::String)

if "EventDeath" in keys(abm.declaredUpdates)

    condition = abm.declaredUpdates["EventDeath"].args[end]

    #Create function to check dead elements
    if platform == "cpu"
        up= :(removeList_Pos_ = Threads.atomic_add!(remV_,1) + 1)
    elseif platform == "gpu"
        up= :(removeList_Pos_ = CUDA.atomic_add!(pointer(remV_,1),Int(1)) + 1)
    end

    code = quote
        if $condition
            keepList_[ic1_] = 0
            $up
            removeList_[removeList_Pos_] = ic1_
        else
            keepList_[ic1_] = ic1_
        end
    end

    code = vectorize_(abm,code,p)

    f1 = simpleFirstLoopWrapInFunction_(platform,:checkDyingAgents_!,code)

    #Create function to put agents in their new position
    code = quote
        oldPos_ = keepList_[ic1_]
        newPos_ = removeList_[ic1_]
    end
    if !isempty(abm.declaredSymbols["Local"])
        push!(code.args,
            :(begin
                if oldPos_ > 0
                    for ic2_ in 1:$(length(abm.declaredSymbols["Local"]))
                        localV[newPos_, ic2_] = localV[oldPos_, ic2_]
                    end
                end
            end)
        )
    end
    if !isempty(abm.declaredSymbols["Identity"])
        push!(code.args,
            :(begin
                if oldPos_ > 0
                    for ic2_ in 1:$(length(abm.declaredSymbols["Identity"]))
                        identityV[newPos_, ic2_] = identityV[oldPos_, ic2_]
                    end
                end
            end)
        )
    end
    if !isempty(p.update["Local"])
        push!(code.args,
            :(begin
                if oldPos_ > 0
                    for ic2_ in 1:$(length(p.update["Local"]))
                        localVCopy[newPos_, ic2_] = localVCopy[oldPos_, ic2_]
                    end
                end
            end)
        )
    end
    if !isempty(p.update["Identity"])
        push!(code.args,
            :(begin
                if oldPos_ > 0
                    for ic2_ in 1:$(length(p.update["Identity"]))
                        identityVCopy[newPos_, ic2_] = identityVCopy[oldPos_, ic2_]
                    end
                end
            end)
        )
    end

    f2 = simpleFirstLoopWrapInFunction_(platform,:assignDeadAgentsSpaces_!,code)
    if platform == "cpu"
        f2 = postwalk(x->@capture(x,N) ? :(remV_[]) : x, f2)
    elseif platform == "gpu"
        f2 = postwalk(x->@capture(x,N) ? :(remV_[1]) : x, f2)
    end

    #Make wrap function of the algorithm
    if platform == "cpu"
        code = quote
            remV_[] = 0
            if N > 0
                @platformAdapt checkDyingAgents_!(ARGS_)
                sort!(keepList_,rev=true)
                @platformAdapt assignDeadAgentsSpaces_!(ARGS_)
            end
        end
    elseif platform == "gpu"
        code = quote
            remV_ .= 0
            if N > 0
                @platformAdapt checkDyingAgents_!(ARGS_)
                sort!(keepList_,rev=true)
                @platformAdapt assignDeadAgentsSpaces_!(ARGS_)
            end
        end
    end

    f3 = wrapInFunction_(:removeDeadAgents_!,code)

    # Add to program
    if platform == "cpu"
        push!(p.declareVar.args,
            :(begin
                remV_ = Threads.Atomic{Int}()
                keepList_ = zeros(Int,nMax)
                removeList_ = zeros(Int,nMax)
            end)
        )
    elseif platform == "gpu"
        push!(p.declareVar.args,
            :(begin
                remV_ = CUDA.zeros(Int,1)
                keepList_ = zeros(Int,nMax)
                removeList_ = zeros(Int,nMax)
            end)
        )
    end

    push!(p.args,:remV_,:keepList_,:removeList_)

    push!(p.declareF.args,f1,f2,f3)

    if platform == "cpu"
        push!(p.execInloop.args,:(removeDeadAgents_!(ARGS_); N -= remV_[]))
    else
        push!(p.execInloop.args,:(removeDeadAgents_!(ARGS_); N -= Core.Array(remV_)[1]))
    end
end

return nothing
end


export batchsize,lr,δ,momentum,λ,▽max,order
const batchsize=50
const lr=1e-2 #learning rate
const δ=0.  #decrease constant lr=lr/(1+δ*t)
const momentum=0.
const λ=0. #regularization coefficient
const ▽max=0.1 #largest gradient
global order
global sharedParams=[]


const FLAG_SIMULATION_BERNOULLI_VARY_RANK_SMALL                 = false
const FLAG_SIMULATION_BERNOULLI_VARY_RANK_MEDIUM                = true
const FLAG_SIMULATION_BERNOULLI_VARY_RANK_LARGE                 = false

const FLAG_SIMULATION_BERNOULLI_VARY_MISSING_PERCENTAGE_SMALL   = false
const FLAG_SIMULATION_BERNOULLI_VARY_MISSING_PERCENTAGE_MEDIUM  = false
const FLAG_SIMULATION_BERNOULLI_VARY_MISSING_PERCENTAGE_LARGE   = false

FLAG_SIMULATION_BERNOULLI_VARY_RANK_SMALL ?
  include("simulation_bernoulli_vary_rank_small.jl")  : nothing

FLAG_SIMULATION_BERNOULLI_VARY_RANK_MEDIUM ?
  include("simulation_bernoulli_vary_rank_medium.jl") : nothing

FLAG_SIMULATION_BERNOULLI_VARY_RANK_LARGE ?
  include("simulation_bernoulli_vary_rank_large.jl") : nothing

# @testset "$(format("ADMM Algorithm: Small Input Simulation [Bernoulli 400 x 400]"))" begin
#   if SIMULATION_STATUS_BERNOULLI_VARY_RANK_SMALL == false
#     let
#       Random.seed!(65536)
#       ROW = 400
#       COL = 400
#       for input_rank in 1:2:400
#         @printf("small case: rank = %d\n", input_rank)
#         dist = Bernoulli()
#         timer = TimerOutput()
#         Base.Filesystem.mkpath("./test_result/bernoulli/small(400x400)")
#         io = open("./test_result/bernoulli/small(400x400)/rank"*string(input_rank)*".log", "w")
#         truth_matrix        = rand([(FixedRankMatrix(Distributions.Bernoulli(0.5), rank = input_rank), ROW, COL)])
#         sample_model        = provide(Sampler{BernoulliModel}(), rate = 0.8)
#         input_matrix        = sample_model.draw(truth_matrix)
#         manual_type_matrix  = Array{Symbol}(undef, ROW, COL)
#         manual_type_matrix .= :Bernoulli
#         @timeit timer  "Bernoulli(400x400)" * "| rank="*string(input_rank) begin
#           completed_matrix, type_tracker, tracker = complete(A               = input_matrix,
#                                                              maxiter         = 200,
#                                                              ρ               = 0.3,
#                                                              use_autodiff    = false,
#                                                              gd_iter         = 3,
#                                                              debug_mode      = false,
#                                                              project_rank    = nothing,
#                                                              io              = io,
#                                                              type_assignment = manual_type_matrix)
#         end
#         log_simulation_result(Bernoulli(), completed_matrix, truth_matrix, type_tracker,tracker, io = io)
#         show(io, MIME("text/plain"), timer)
#       end
#     end
#   end
# end

# @testset "$(format("ADMM Algorithm: Medium Input Simulation [Bernoulli 2000 x 2000]"))" begin
#   if SIMULATION_STATUS_BERNOULLI_VARY_RANK_MEDIUM == false
#     let
#       Random.seed!(65536)
#       ROW = 2000
#       COL = 2000
#       # for input_rank in union(1, collect(10:10:500))
#       for input_rank in collect(300:10:500)
#         @printf("medium case: rank = %d\n", input_rank)
#         dist = Bernoulli()
#         timer = TimerOutput()
#         RESULTS_DIR    = GLOBAL_SIMULATION_RESULTS_DIR * "bernoulli/medium(2000x2000)(vary_rank)/" * "rank" * string(input_rank) * "/"
#         LOG_FILE_NAME  = "io.log"
#         DATA_FILE_NAME = "saved_variables.h5"
#         LOG_FILE_PATH  = RESULTS_DIR * LOG_FILE_NAME
#         DATA_FILE_PATH = RESULTS_DIR * DATA_FILE_NAME
#         Base.Filesystem.mkpath(RESULTS_DIR)
#         io = open(LOG_FILE_PATH, "w")
#         truth_matrix        = rand([(FixedRankMatrix(Distributions.Bernoulli(0.5), rank = input_rank), ROW, COL)])
#         sample_model        = provide(Sampler{BernoulliModel}(), rate = 0.8)
#         input_matrix        = sample_model.draw(truth_matrix)
#         manual_type_matrix  = Array{Symbol}(undef, ROW, COL)
#         manual_type_matrix .= :Bernoulli
#         @timeit timer  "Bernoulli(2000x2000)" * "| rank=" * string(input_rank) begin
#           completed_matrix, type_tracker, tracker = complete(A               = input_matrix,
#                                                              maxiter         = 200,
#                                                               ρ               = 0.3,
#                                                              use_autodiff    = false,
#                                                              gd_iter         = 3,
#                                                              debug_mode      = false,
#                                                              project_rank    = input_rank,
#                                                              io              = io,
#                                                              type_assignment = manual_type_matrix)
#         end

#         predicted_matrix = predict(MatrixCompletionModel(),
#                                    completed_matrix = completed_matrix,
#                                    type_tracker     = type_tracker)

#         summary_object   = summary(MatrixCompletionModel(),
#                                    predicted_matrix = predicted_matrix,
#                                    truth_matrix     = truth_matrix,
#                                    type_tracker     = type_tracker,
#                                    tracker          = tracker)
#         pickle(DATA_FILE_PATH,
#                "missing_idx"      => type_tracker[:Missing],
#                "completed_matrix" => completed_matrix,
#                "predicted_matrix" => predicted_matrix,
#                "truth_matrix"     => truth_matrix,
#                "summary"          => summary_object)

#         log_simulation_result(Bernoulli(), completed_matrix, truth_matrix, type_tracker,tracker, io = io)
#         print(io, JSON.json(summary_object, 4))
#         print(io, timer)
#         close(io)
#       end
#     end
#   else
#     @info("Already Completed Simualtion[Bernoulli vary rank][MEDIUM]")
#   end
# end

# @testset "$(format("ADMM Algorithm: Medium Input Simulation [Bernoulli 2000 x 2000]"))" begin
#   if SIMULATION_STATUS_BERNOULLI_VARY_RANK_MEDIUM == false
#     let
#       Random.seed!(65536)
#       ROW = 100
#       COL = 100
#       for input_rank in union(1, collect(10:10:500))
#         @printf("medium case: rank = %d\n", input_rank)
#         dist = Bernoulli()
#         timer = TimerOutput()
#         # Base.Filesystem.mkpath("./test_result/bernoulli/medium(2000x2000)")
#         # io = open("./test_result/bernoulli/medium(2000x2000)/rank"*string(input_rank)*".log", "w")
#         io = Base.stdout
#         truth_matrix        = rand([(FixedRankMatrix(Distributions.Bernoulli(0.5), rank = input_rank), ROW, COL)])
#         sample_model        = provide(Sampler{BernoulliModel}(), rate = 0.8)
#         input_matrix        = sample_model.draw(truth_matrix)
#         manual_type_matrix  = Array{Symbol}(undef, ROW, COL)
#         manual_type_matrix .= :Bernoulli
#         @timeit timer  "Bernoulli(2000x2000)" * "| rank="*string(input_rank) begin
#           completed_matrix, type_tracker, tracker = complete(A               = input_matrix,
#                                                              maxiter         = 200,
#                                                              ρ               = 0.3,
#                                                              use_autodiff    = false,
#                                                              gd_iter         = 3,
#                                                              debug_mode      = false,
#                                                              project_rank    = input_rank,
#                                                              io              = io,
#                                                              type_assignment = manual_type_matrix)
#         end
#         log_simulation_result(Bernoulli(), completed_matrix, truth_matrix, type_tracker,tracker, io = io)
#         show(io, MIME("text/plain"), timer)
#       end
#     end
#   end
# end


#===============================================================================

    ThermalConsistencyOutput.jl

    Author: Tom Price
    Date:   July 2019

===============================================================================#

struct ThermalConsistencyOutput
    tm_check            ::Vector{TmCheck1w}
    delta_Tm            ::Tuple{Float_T, Bool}
    valid               ::Bool
end


"""
$(TYPEDEF)

Introduce direct observations on all dimensions of a Pose2 variable:

Example:
--------
```julia
PriorPose2( MvNormal([10; 10; pi/6.0], Matrix(Diagonal([0.1;0.1;0.05].^2))) )
```
"""
mutable struct PriorPose2{T} <: IncrementalInference.AbstractPrior  where {T <: IncrementalInference.SamplableBelief}
  Z::T
  # empty constructor
  PriorPose2{T}() where T = new{T}()
  # regular constructor
  PriorPose2{T}(x::T) where {T <: IncrementalInference.SamplableBelief}  = new{T}(x)
end
# convencience and default object helper
PriorPose2(x::T) where {T <: IncrementalInference.SamplableBelief} = PriorPose2{T}(x)

function getSample(cfo::CalcFactor{<:PriorPose2}, N::Int=1)
  return (rand(cfo.factor.Z,N), )
end

function (s::CalcFactor{<:PriorPose2})(meas,
                                       wXi)
  #
  iXihat = SE2(meas) \ SE2(wXi)
  return se2vee(iXihat)
end

## Serialization support

"""
$(TYPEDEF)
"""
mutable struct PackedPriorPose2  <: IncrementalInference.PackedInferenceType
    str::String
    PackedPriorPose2() = new()
    PackedPriorPose2(x::String) = new(x)
end
function convert(::Type{PackedPriorPose2}, d::PriorPose2)
  return PackedPriorPose2(convert(PackedSamplableBelief, d.Z))
end
function convert(::Type{PriorPose2}, d::PackedPriorPose2)
  distr = convert(SamplableBelief, d.str)
  return PriorPose2{typeof(distr)}(distr)
end

## NOTE likely deprecated comparitors, see DFG compareFields, compareAll instead
function compare(a::PriorPose2,b::PriorPose2; tol::Float64=1e-10)
  compareDensity(a.Z, b.Z)
end


using PowerModels

function parse_prob_file(file::String, data::Dict{AbstractString,Any})

    file = string(split(file, ".m")[1], ".prob")
    data_string = readstring(open(file))
    data_lines = split(data_string, '\n')
    parsed_matrices = []
    last_index = length(data_lines)
    index = 1

    while index <= last_index
        line = strip(data_lines[index])

        if length(line) <= 0 || strip(line)[1] == '%'
            index += 1
            continue
        end

        if contains(line, "[")
            matrix = PowerModels.parse_matrix(data_lines, index)
            push!(parsed_matrices, matrix)
            index += matrix["line_count"] - 1
        end
        index += 1
    end

    parsed_matrix = parsed_matrices[1]
    prob = []
    if parsed_matrix["name"] == "mpc.branchprobabilities"
        for row in parsed_matrix["data"]
            push!(prob, parse(Float64, row[3]))
        end
    end

    for i in 1:length(prob)
        for branch in data["branch"]
            if branch["index"] == i
                branch["prob"] = prob[i]
                break
            end
        end
    end

    return
end


module Deprecated

using Test
using DataFramesMeta
using Statistics

const ≅ = isequal

@testset "@based_on" begin
    df = DataFrame(
        g = [1, 1, 1, 2, 2],
        i = 1:5,
        t = ["a", "b", "c", "c", "e"],
        y = [:v, :w, :x, :y, :z],
        c = [:g, :quote, :body, :transform, missing]
        )

    m = [100, 200, 300, 400, 500]

    gq = :g
    iq = :i
    tq = :t
    yq = :y
    cq = :c

    gr = "g"
    ir = "i"
    tr = "t"
    yr = "y"
    cr = "c"

    gd = groupby(df, :g)

    n_str = "new_column"
    n_sym = :new_column
    n_space = "new column"

    @test @based_on(gd, n = mean(:i)).n == [2.0, 4.5]
    @test @based_on(gd, n = mean(:i) + mean(:g)).n == [3.0, 6.5]
    @test @based_on(gd, n = first(:t .* string.(:y))).n == ["av", "cy"]
    @test @based_on(gd, n = first(Symbol.(:y, ^(:t)))).n == [:vt, :yt]
    @test @based_on(gd, n = first(Symbol.(:y, ^(:body)))).n == [:vbody, :ybody]
    @test @based_on(gd, body = :i).body == df.i
    @test @based_on(gd, transform = :i).transform == df.i
    @test @based_on(gd, (n1 = [first(:i)], n2 = [first(:y)])).n1 == [1, 4]

    @test @based_on(gd, n = mean(cols(iq))).n == [2.0, 4.5]
    @test @based_on(gd, n = mean(cols(iq)) + mean(cols(gq))).n == [3.0, 6.5]
    @test @based_on(gd, n = first(cols(tq) .* string.(cols(yq)))).n == ["av", "cy"]
    @test @based_on(gd, n = first(Symbol.(cols(yq), ^(:t)))).n == [:vt, :yt]
    @test @based_on(gd, n = first(Symbol.(cols(yq), ^(:body)))).n == [:vbody, :ybody]
    @test @based_on(gd, body = cols(iq)).body == df.i
    @test @based_on(gd, transform = cols(iq)).transform == df.i
    @test @based_on(gd, (n1 = [first(cols(iq))], n2 = [first(cols(yq))])).n1 == [1, 4]

    @test @based_on(gd, n = mean(cols(ir))).n == [2.0, 4.5]
    @test @based_on(gd, n = mean(cols(ir)) + mean(cols(gr))).n == [3.0, 6.5]
    @test @based_on(gd, n = first(cols(tr) .* string.(cols(yr)))).n == ["av", "cy"]
    @test @based_on(gd, n = first(Symbol.(cols(yr), ^(:t)))).n == [:vt, :yt]
    @test @based_on(gd, n = first(Symbol.(cols(yr), ^(:body)))).n == [:vbody, :ybody]
    @test @based_on(gd, body = cols(ir)).body == df.i
    @test @based_on(gd, transform = cols(ir)).transform == df.i
    @test @based_on(gd, (n1 = [first(cols(ir))], n2 = [first(cols(yr))])).n1 == [1, 4]
    @test @based_on(gd, n = mean(cols("i")) + 0 * first(cols(:g))).n == [2.0, 4.5]
    @test @based_on(gd, n = mean(cols(2)) + first(cols(1))).n == [3.0, 6.5]

    @test @based_on(gd, :i) == select(df, :g, :i)
    @test @based_on(gd, :i, :g) ≅ select(df, :g, :i)

    @test @based_on(gd, :i, n = 1).n == fill(1, nrow(df))

    @test @based_on(gd, cols("new_column") = 2).new_column == [2, 2]
    @test @based_on(gd, cols(n_str) = 2).new_column == [2, 2]
    @test @based_on(gd, cols(n_sym) = 2).new_column == [2, 2]
    @test @based_on(gd, cols(n_space) = 2)."new column" == [2, 2]
    @test @based_on(gd, cols("new" * "_" * "column") = 2)."new_column" == [2, 2]
end

@testset "where" begin
    df = DataFrame(A = [1, 2, 3, missing], B = [2, 1, 2, 1])

    x = [2, 1, 0, 0]

    @test @where(df, :A .> 1) == df[(df.A .> 1) .=== true,:]
    @test @where(df, :B .> 1) == df[df.B .> 1,:]
    @test @where(df, :A .> x) == df[(df.A .> x) .=== true,:]
    @test @where(df, :B .> x) ≅ df[df.B .> x,:]
    @test @where(df, :A .> :B, :B .> mean(:B)) == DataFrame(A = 3, B = 2)
    @test @where(df, :A .> 1, :B .> 1) == df[map(&, df.A .> 1, df.B .> 1),:]
    @test @where(df, :A .> 1, :A .< 4, :B .> 1) == df[map(&, df.A .> 1, df.A .< 4, df.B .> 1),:]

    @test @where(df, :A .> 1).A isa Vector{Union{Missing, Int}}

    @test @where(df, cols(:A) .> 1) == df[(df.A .> 1) .=== true,:]
    @test @where(df, cols(:B) .> 1) == df[df.B .> 1,:]
    @test @where(df, cols(:A) .> x) == df[(df.A .> x) .=== true,:]
    @test @where(df, cols(:B) .> x) ≅ df[df.B .> x,:]
    @test @where(df, cols(:A) .> :B, cols(:B) .> mean(:B)) == DataFrame(A = 3, B = 2)
    @test @where(df, cols(:A) .> 1, :B .> 1) == df[map(&, df.A .> 1, df.B .> 1),:]
    @test @where(df, cols(:A) .> 1, :A .< 4, :B .> 1) == df[map(&, df.A .> 1, df.A .< 4, df.B .> 1),:]

    @test @where(df, :A .> 1, :A .<= 2) == DataFrame(A = 2, B = 1)

    subdf = @view df[df.B .== 2, :]

    @test @where(subdf, :A .== 3) == DataFrame(A = 3, B = 2)
end

@testset "where with :block" begin
    df = DataFrame(A = [1, 2, 3, missing], B = [2, 1, 2, 1])

    d = @where df begin
        :A .> 1
        :B .> 1
    end
    @test d ≅ @where(df, :A .> 1, :B .> 1)

    d = @where df begin
        cols(:A) .> 1
        :B .> 1
    end
    @test d ≅ @where(df, :A .> 1, :B .> 1)

    d = @where df begin
        :A .> 1
        cols(:B) .> 1
    end
    @test d ≅ @where(df, :A .> 1, :B .> 1)

    d = @where df begin
        begin
            :A .> 1
        end
        :B .> 1
    end
    @test d ≅ @where(df, :A .> 1, :B .> 1)

    d = @where df begin
        :A .> 1
        @. :B > 1
    end
    @test d ≅ @where(df, :A .> 1, :B .> 1)
end

@testset "@where with a grouped data frame" begin
    df = DataFrame(
        g = [1, 1, 1, 2, 2],
        i = 1:5,
        t = ["a", "b", "c", "c", "e"],
        y = [:v, :w, :x, :y, :z],
        c = [:g, :quote, :body, :transform, missing]
    )

    gd = groupby(df, :g)

    @test @where(gd, :i .== first(:i)) ≅ df[[1, 4], :]
    @test @where(gd, cols(:i) .> mean(cols(:i)), :t .== "c") ≅ df[[3], :]
    @test @where(gd, :c .== :g) ≅ df[[], :]
end

@testset "Unquoted symbols on LHS" begin
    df = DataFrame(
        g = [1, 1, 1, 2, 2],
        i = 1:5,
        t = ["a", "b", "c", "c", "e"],
        y = [:v, :w, :x, :y, :z],
        c = [:g, :quote, :body, :transform, missing]
    )

    gd = groupby(df, :g)

    newdf = @transform df :n = :i

    @test (@transform df n = :i) ≅ newdf
    @test (@transform(df, n = identity(:i))) ≅ newdf
    @test (@transform df @byrow n = :i) ≅ newdf
    d = @transform df begin
        n = identity(:i)
    end
    @test d ≅ newdf

    d = @eachrow df begin
        @newcol n::Vector{Int}
        :n = :i
    end
    @test d ≅ newdf

    newdf = @select df :n = :i

    @test (@select df n = :i) ≅ newdf
    @test (@select(df, n = identity(:i))) ≅ newdf
    d = @select df begin
        n = identity(:i)
    end
    @test (@select df @byrow n = :i) ≅ newdf
    @test d ≅ newdf

    newdf = @combine gd :n = first(:i)
    @test (@combine gd n = first(:i)) ≅ newdf
    @test (@combine(gd, n = first(:i))) ≅ newdf
end

end # module


# This method is called when a SemidiscretizationHyperbolic is constructed.
# It constructs the basic `cache` used throughout the simulation to compute
# the RHS etc.
function create_cache(mesh::P4estMesh, equations::AbstractEquations, dg::DG, ::Any, ::Type{uEltype}) where {uEltype<:Real}
  # Make sure to balance the p4est before creating any containers
  # in case someone has tampered with the p4est after creating the mesh
  p4est_balance(mesh.p4est, P4EST_CONNECT_FACE, C_NULL)
  # Due to a bug in p4est, the forest needs to be rebalanced twice sometimes
  # See https://github.com/cburstedde/p4est/issues/112
  p4est_balance(mesh.p4est, P4EST_CONNECT_FACE, C_NULL)

  elements   = init_elements(mesh, equations, dg.basis, uEltype)
  interfaces = init_interfaces(mesh, equations, dg.basis, elements)
  boundaries = init_boundaries(mesh, equations, dg.basis, elements)
  mortars    = init_mortars(mesh, equations, dg.basis, elements)

  cache = (; elements, interfaces, boundaries, mortars)

  # Add specialized parts of the cache required to compute the volume integral etc.
  cache = (;cache..., create_cache(mesh, equations, dg.mortar, uEltype)...)

  return cache
end

# Extract outward-pointing normal vector (contravariant vector ±Ja^i, i = index) as SVector
# Note that this vector is not normalized
@inline function get_normal_vector(direction, cache, indices...)
  @unpack contravariant_vectors, inverse_jacobian = cache.elements

  # If the mapping is orientation-reversing, the contravariant vectors' orientation
  # is reversed as well
  sign_jacobian = sign(inverse_jacobian[indices...])

  orientation = div(direction + 1, 2)
  normal = sign_jacobian * get_contravariant_vector(orientation, contravariant_vectors, indices...)

  # Contravariant vectors at interfaces in negative coordinate direction are pointing inwards
  # (after normalizing with sign(J) above)
  if direction in (1, 3, 5)
    normal *= -1
  end

  return normal
end

@inline ndofs(mesh::P4estMesh, dg::DG, cache) = nelements(cache.elements) * nnodes(dg)^ndims(mesh)

include("containers.jl")
include("dg_2d.jl")