"""
Materialize a `Join` into a `DataFrame`. Currently only eager
materialization is allowed, which requires two steps:

1. Materialize the LHS and RHS source nodes into `DataFrame` objects.
2. Use `DataFrames.innerjoin` (or similar) to generate the `DataFrame` output
    based on the LHS and RHS inputs.
"""
function materialize(node::Join)
    lhs, rhs = node.sources

    # TODO: Rewrite when we don't require equi-join on existing columns.
    pairs = map(p -> _predicate_to_pair(p, lhs, rhs), node.predicates)

    # TODO: Handle generating source table prefixes.
    if node.kind === inner::JoinKind
        DataFrames.innerjoin(
            materialize(lhs.source),
            materialize(rhs.source),
            on = [pairs...],
        )
    elseif node.kind === left::JoinKind
        DataFrames.leftjoin(
            materialize(lhs.source),
            materialize(rhs.source),
            on = [pairs...],
        )
    elseif node.kind === right::JoinKind
        DataFrames.rightjoin(
            materialize(lhs.source),
            materialize(rhs.source),
            on = [pairs...],
        )
    elseif node.kind === outer::JoinKind
        DataFrames.outerjoin(
            materialize(lhs.source),
            materialize(rhs.source),
            on = [pairs...],
        )
    end
end


module ArgoTools

using Dates, YAML, NCDatasets, CSV, DataFrames, Interpolations, Printf
import ArgoData.ProfileNative

"""
    meta_init(fil::String)

Get parameters to call `MITprof.format` from yaml file (`fil`, e.g. "../examples/ArgoToMITprof.yml").
"""
function meta_init(fil="")

    if !isempty(fil)
        meta=YAML.load(open(fil))
    else
        z_std=[[5:10:185]... ; [200:20:500]... ; [550:50:1000]...; [1100:100:6000]...]
        meta=Dict("name" => "argo",
        "variables"   => ["depth", "T", "S"],
        "subset"      => Dict{Any, Any}("year"=>2016, "basin"=>"indian"),
        "depthLevels" => z_std,
        "dirOut"      => joinpath(tempdir(),"argo_example"),
        "depthRange"  => [0, 2000],
        "dirIn"       =>  joinpath(tempdir(),"argo_example"),
        )
    end

    d=meta["dirIn"]
    b=meta["subset"]["basin"]
    y=meta["subset"]["year"]

    list0=Array{Array,1}(undef,12)
    for m=1:12
        sd="$b"*"_ocean/$y/"*Printf.@sprintf("%02d/",m)
        if isdir(d*sd)
            tmp=readdir(d*sd)
            list0[m]=[sd*tmp[i] for i=1:length(tmp)]
        else
            list0[m]=[]
        end
    end

    nf=sum(length.(list0))
    list1=Array{String,1}(undef,nf)
    f=0
    for m=1:12
        for ff=1:length(list0[m])
            f+=1
            list1[f]=list0[m][ff]
        end
    end

    meta["fileInList"]=list1;

    #2. coordinate

    z_std=meta["depthLevels"]
    if length(z_std)>1
        tmp1=(z_std[2:end]+z_std[1:end-1])/2
        z_top=[z_std[1]-(z_std[2]-z_std[1])/2;tmp1]
        z_bot=[tmp1;z_std[end]+(z_std[end]-z_std[end-1])/2]
    else
        z_top=0.9*z_std
        z_bot=1.1*z_std
    end

    meta["z_std"]=z_std
    meta["z_top"]=z_top
    meta["z_bot"]=z_bot

    #3. various other specs

    meta["inclZ"] = false
    meta["inclT"] = true
    meta["inclS"] = true
    meta["inclU"] = false
    meta["inclV"] = false
    meta["inclPTR"] = false
    meta["inclSSH"] = false
    meta["TPOTfromTINSITU"] = true

    meta["doInterp"] = true
    meta["addGrid"] = true
    meta["outputMore"] = false
    meta["method"] = "interp"
    meta["fillval"] = -9999.0
    meta["buffer_size"] = 10000

    meta["fileOut"]=meta["name"]*"_"*meta["subset"]["basin"]*"_"*string(meta["subset"]["year"])*".nc"
    meta["var_out"]=meta["variables"]

    return meta
end

"""
    meta(input_file,output_file)

Get parameters to call `MITprof.format` which will read from `input_file` to create `output_file`.
"""
function meta(input_file,output_file)
    meta=ArgoTools.meta_init()
    #f=1
    #input_file=meta["dirIn"]*meta["fileInList"][f]
    meta["fileOut"]=output_file

    meta
end

"""
    GetOneProfile(m)

Get one profile from netcdf file.
"""
function GetOneProfile(ds,m)

    #
    t=ds["JULD"][m]
    if !ismissing(t)
        ymd=Dates.year(t)*1e4+Dates.month(t)*1e2+Dates.day(t)
        hms=Dates.hour(t)*1e4+Dates.minute(t)*1e2+Dates.second(t)

        prof_date=t-DateTime(0)
        prof_date=prof_date.value/86400/1000+1 #+1 is to match Matlab's datenum
    else
        ymd=0
        hms=0
        prof_date=0.0
    end

    lat=ds["LATITUDE"][m]
    lon=ds["LONGITUDE"][m]
    (!ismissing(lon))&&(lon < 0.0) ? lon=lon+360.0 : nothing

    direction=ds["DIRECTION"][m]
    direc=0
    direction=='A' ? direc=1 : nothing
    direction=='D' ? direc=2 : nothing

    #
    pnum_txt=ds["PLATFORM_NUMBER"][:,m]
    #pnum_txt=pnum_txt[findall((!ismissing).(pnum_txt))]
    ii=findall(skipmissing(in.(pnum_txt,"0123456789")))
    ~isempty(ii) ? pnum_txt=String(vec(Char.(pnum_txt[ii]))) : pnum_txt="9999"
    pnum=parse(Int,pnum_txt)

    #
    p=ds["PRES_ADJUSTED"][:,m]
    p_QC=ds["PRES_ADJUSTED_QC"][:,m]
    if sum((!ismissing).(p))==0
        p=ds["PRES"][:,m]
        p_QC=ds["PRES_QC"][:,m]
    end

    #set qc to 5 if missing
    p_QC[findall(ismissing.(p))].='5'
    #avoid potential duplicates
    for n=1:length(p)-1
        if ~ismissing(p[n])
            tmp1=( (!ismissing).(p[n+1:end]) ).&( p[n+1:end].==p[n] )
            tmp1=findall(tmp1)
            p[n.+tmp1].=missing
            p_QC[n.+tmp1].='5'
        end
    end

    #position and date
    isBAD=0
    ismissing(ds["POSITION_QC"][m])||~in(ds["POSITION_QC"][m],"1258") ? isBAD=1 : nothing
    ismissing(ds["JULD_QC"][m])||~in(ds["JULD_QC"][m],"1258") ? isBAD=1 : nothing

    #pressure
    tmp1=findall( (!in).(p_QC,"1258") )
    if (length(tmp1)<=5)&&(length(tmp1)>0)
        #omit these few bad points but keep the profile
        p[tmp1].=missing
    elseif length(tmp1)>5
        #flag the profile (will be omitted later)
        #but keep the bad points (for potential inspection)
        isBAD=1
    end

    #temperature
    t=ds["TEMP_ADJUSTED"][:,m]
    t_QC=ds["TEMP_ADJUSTED_QC"][:,m]
    t_ERR=ds["TEMP_ADJUSTED_ERROR"][:,m]
    t_ERR[findall( (ismissing).(t_ERR) )].=0.0

    if sum((!ismissing).(t))==0
        t=ds["TEMP"][:,m]
        t_QC=ds["TEMP_QC"][:,m]
    end

    #salinity
    if haskey(ds,"PSAL")
        s=ds["PSAL_ADJUSTED"][:,m]
        s_QC=ds["PSAL_ADJUSTED_QC"][:,m]
        s_ERR=ds["PSAL_ADJUSTED_ERROR"][:,m]
        s_ERR[findall( (ismissing).(t_ERR) )].=0.0
        if sum((!ismissing).(t))==0
            s=ds["PSAL"][:,m]
            s_QC=ds["PSAL_QC"][:,m]
        end
    else
        s=fill(missing,size(t))
        s_QC=Char.(32*ones(size(t_QC)))
    end

    if ismissing(t[1]) #this file does not contain temperature data...
        t=fill(missing,size(t))
        t_ERR=fill(0.0,size(t))
    else #apply QC
        tmp1=findall(skipmissing((!in).(t_QC,"1258")))
        t[tmp1].=missing
    end

    if ismissing(s[1]) #this file does not contain salinity data...
        s=fill(missing,size(s))
        s_ERR=fill(0.0,size(s))
    else #apply QC
        tmp1=findall(skipmissing((!in).(s_QC,"1258")))
        s[tmp1].=missing
    end

    prof=ProfileNative(
        convert(Array{Union{Float64,Missing}},[lon]),
        convert(Array{Union{Float64,Missing}},[lat]),
        convert(Union{Float64,Missing},prof_date),
        convert(Array{Union{Int,Missing}},[ymd]),
        convert(Array{Union{Int,Missing}},[hms]),
        convert(Array{Union{Float64,Missing}},t),
        convert(Array{Union{Float64,Missing}},s),
        convert(Array{Union{Float64,Missing}},p),
        convert(Array{Union{Float64,Missing}},p), #place holder for depth
        convert(Array{Union{Float64,Missing}},t_ERR),
        convert(Array{Union{Float64,Missing}},s_ERR),
        convert(Array{Union{String,Missing}},[pnum_txt]),
        convert(Array{Union{Int,Missing}},[direc]),
        convert(Array{Union{Char,Missing}},[ds["DATA_MODE"][m]]),
        convert(Array{Union{Int,Missing}},[isBAD])
        )

    return prof
end

## ArgoProfile data structure

"""
    prof_PtoZ!(prof)

Convert prof["p"] to prof["depth"]
"""
function prof_PtoZ!(prof)
    l=prof.lat[1]
    k=findall((!ismissing).(prof.pressure))
    prof.depth[k].=[sw_dpth(Float64(prof.pressure[kk]),Float64(l)) for kk in k]
end

"""
    prof_TtoΘ!(prof)

Convert prof["T"] to potential temperature
"""
function prof_TtoΘ!(prof)
    T=prof.T
    P=0.981*1.027*prof.depth
    S=35.0*ones(size(T))
    k=findall( ((!ismissing).(T)).*((!ismissing).(P)) )
    T[k]=[sw_ptmp(Float64(S[kk]),Float64(T[kk]),Float64(P[kk])) for kk in k]
end

"""
    prof_interp!(prof,meta)

Interpolate from prof["depth"] to meta["z_std"]
"""
function prof_interp!(prof,prof_std,meta)
    z_std=meta["z_std"]
    for ii=2:length(meta["var_out"])
        v=meta["var_out"][ii]
        v_e=v*"_ERR"

        t_std=similar(z_std,Union{Missing,Float64})
        e_std=similar(z_std,Union{Missing,Float64})

        z=prof.depth
        v=="T" ? t=prof.T : t=prof.S
        do_e=true #haskey(prof,v_e)
        v=="T" ? e=prof.T_ERR : e=prof.S_ERR

        kk=findall((!ismissing).(z.*t))
        if (meta["doInterp"])&&(length(kk)>1)
            z_in=z[kk]; t_in=t[kk]
            do_e ? e_in=e[kk] : nothing
            k=sort(1:length(kk),by= i -> z_in[i])
            z_in=z_in[k]; t_in=t_in[k]
            do_e ? e_in=e_in[k] : nothing
            #omit values outside observed range:
            D0=minimum(skipmissing(z_in))
            D1=maximum(skipmissing(z_in))
            msk1=findall( (z_std.<D0).|(z_std.>D1) )
            #avoid duplicates:
            msk2=findall( ([false;(z_in[1:end-1]-z_in[2:end]).==0.0]).==true )
            if length(kk)>5
                interp_linear_extrap = LinearInterpolation(Float64.(z_in), Float64.(t_in), extrapolation_bc=Line())
                t_std[:] = interp_linear_extrap(z_std)
                t_std[msk1].=missing
                t_std[msk2].=missing
                if do_e
                    e_in[findall(ismissing.(e_in))].=0.0
                    interp_linear_extrap = LinearInterpolation(Float64.(z_in), Float64.(e_in), extrapolation_bc=Line())
                    e_std[:] = interp_linear_extrap(z_std)
                    e_std[msk1].=missing
                    e_std[msk2].=missing
                end
            else
                t_std = []
                e_std = []
            end
        end
        if v=="T"&&!isempty(t_std)
            prof_std.T .=t_std
            prof_std.T_ERR .=e_std
        elseif v=="S"&&!isempty(t_std)
            prof_std.S .=t_std
            prof_std.S_ERR .=e_std
        end
    end
end

function prof_test_set1!(prof,prof_std,meta)

    #test for 'not enough data near standard level'
    z=prof.depth
    nz1=length(z)
    nz2=length(meta["z_top"])
    tmp1=[meta["z_top"][j]-z[i] for i in 1:nz1, j in 1:nz2]
    tmp2=[meta["z_bot"][j]-z[i] for i in 1:nz1, j in 1:nz2]
    tmp1=1.0*(tmp1.<0.0)
    tmp2=1.0*(tmp2.>=0.0)
    tmp3=tmp1.*tmp2
    if isa(tmp3,Matrix{Missing})
        tmp3=convert(Array{Union{Float64,Missing}},tmp3)
    end
    tmp3[findall(ismissing.(tmp3))].=0.0
    tmp3=sum(tmp3,dims=1)

    prof_std.Ttest.=0.0
    prof_std.Ttest[findall( (tmp3[:].<=0.0).&((!ismissing).(prof_std.T[:])) )].=1.0
    prof_std.Stest.=0.0
    prof_std.Stest[findall( (tmp3[:].<=0.0).&((!ismissing).(prof_std.S[:])) )].=1.0

    #test for "absurd" salinity values :
    prof_std.Stest[findall( (prof_std.S[:].>42.0).&((!ismissing).(prof_std.S[:])) )].=
    10*prof_std.Stest[findall( (prof_std.S[:].>42.0).&((!ismissing).(prof_std.S[:])) )] .+ 2
    prof_std.Stest[findall( (prof_std.S[:].<15.0).&((!ismissing).(prof_std.S[:])) )].=
    10*prof_std.Stest[findall( (prof_std.S[:].<15.0).&((!ismissing).(prof_std.S[:])) )] .+ 2

    #bad pressure flag:
    if prof.isBAD[1]>0
        prof_std.Ttest.=10*prof_std.Ttest .+ 6
        prof_std.Stest.=10*prof_std.Stest .+ 6
    end;

    #Argo grey list:
    if haskey(meta,"greylist") && !isempty(meta["greylist"])
        test1=!(prof.DATA_MODE[1].=='D') #true = real time profile ('R' or 'A')
        test2=sum(parse(Int,prof.pnum_txt[1]).==meta["greylist"][:,"PLATFORM_CODE"]) #is in grey list
        if test1&(test2>0)
            II=findall(parse(Int,prof.pnum_txt[1]).==meta["greylist"][:,"PLATFORM_CODE"])
            timeP=prof.ymd[1]
            for ii in II
                time0=meta["greylist"][ii,"START_DATE"]
                time1=meta["greylist"][ii,"END_DATE"]
                if (time0<timeP)&&(ismissing(time1)||(time1>timeP))
                    prof_std.Ttest.=10*prof_std.Ttest .+ 4
                    prof_std.Stest.=10*prof_std.Stest .+ 4
                end
            end
        end
    end

end

function prof_test_set2!(prof_std,meta)
    max_cost=50

    tmp_cost=prof_std.Tweight.*((prof_std.T-prof_std.Testim).^2)
    tmp_cost=convert(Array{Union{Float64,Missing}},tmp_cost)
    tmp_cost[findall(ismissing.(tmp_cost))].=0.0
    ii=findall( (tmp_cost.>max_cost) )
    prof_std.Ttest[ii].=10*prof_std.Ttest[ii] .+5

    tmp_cost=prof_std.Sweight.*((prof_std.S-prof_std.Sestim).^2)
    tmp_cost=convert(Array{Union{Float64,Missing}},tmp_cost)
    tmp_cost[findall(ismissing.(tmp_cost))].=0.0
    ii=findall( (tmp_cost.>max_cost) )
    prof_std.Stest[ii].=10*prof_std.Stest[ii] .+5

    ii=findall(
        ((ismissing).(prof_std.T)).+((ismissing).(prof_std.Testim)).+(prof_std.Ttest.>0).>0
        )
    prof_std.Tweight[ii].=0

    ii=findall(
        ((ismissing).(prof_std.S)).+((ismissing).(prof_std.Sestim)).+(prof_std.Stest.>0).>0
        )
    prof_std.Sweight[ii].=0

end

"""
    prof_convert!(prof,meta)

Appply conversions to variables (lon,lat,depth,temperature) in `prof` if specified in `meta`.
"""
function prof_convert!(prof,meta)
    lonlatISbad=false
    ismissing(prof.lat[1])||(prof.lat[1]<-90.0)||(prof.lat[1]>90.0) ? lonlatISbad=true : nothing
    ismissing(prof.lon[1])||(prof.lon[1]<-180.0)||(prof.lon[1]>360.0) ? lonlatISbad=true : nothing

    #if needed then reset lon,lat after issuing a warning
    lonlatISbad==true ? println("warning: out of range lon/lat was reset to 0.0,-89.99") : nothing
    lonlatISbad ? (prof.lon[1],prof.lat[1])=(0.0,-89.99) : nothing

    #if needed then fix longitude range to 0-360
    (~lonlatISbad)&(prof.lon[1]>180.0) ? prof.lon[1]-=360.0 : nothing

    #if needed then convert pressure to depth
    (~meta["inclZ"])&(~lonlatISbad) ? ArgoTools.prof_PtoZ!(prof) : nothing

    #if needed then convert T to potential temperature θ
    meta["TPOTfromTINSITU"] ? ArgoTools.prof_TtoΘ!(prof) : nothing
end

function interp_z(x,y,xi)
    jj=findall(isfinite.(y))
    interp_linear_extrap = LinearInterpolation(Float64.(x[jj]), Float64.(y[jj]), extrapolation_bc=Flat())
    return interp_linear_extrap(xi)
end

"""
    sw_dpth(P,LAT)

Calculate depth in meters from pressure (P; in decibars) and
latitude (LAT; in °N)

```
d = sw_dpth(100.0,20.0)
```
"""
function sw_dpth(P,LAT)
    # Original author:  Phil Morgan 92-04-06  (morgan@ml.csiro.au)
    # Reference: Unesco 1983. Algorithms for computation of fundamental properties of
    # seawater, 1983. _Unesco Tech. Pap. in Mar. Sci._, No. 44, 53 pp. Eqn 25, p26

    c1 = 9.72659
    c2 = -2.2512E-5
    c3 = 2.279E-10
    c4 = -1.82E-15
    gam_dash = 2.184e-6

    LAT = abs.(LAT)
    X   = sin.(deg2rad.(LAT))
    X   = X.*X
    bot_line = 9.780318*(1.0+(5.2788E-3+2.36E-5*X).*X) + gam_dash*0.5*P
    top_line = (((c4*P+c3).*P+c2).*P+c1).*P
    DEPTHM   = top_line./bot_line

    return DEPTHM
end

"""
    sw_ptmp(S,T,P,PR)

Calculate potential temperature as per UNESCO 1983 report from salinity (S;
in psu), in situ temperature (T; in °C), and pressure (P; in decibar)
relative to PR (in decibar; 0 by default).

```
ptmp = sw_ptmp(S,T,P,PR=missing)
```
"""
function sw_ptmp(S,T,P,PR=0.0)
# Original author:  Phil Morgan
# Reference: Fofonoff, P. and Millard, R.C. Jr
#    Unesco 1983. Algorithms for computation of fundamental properties of
#    seawater. _Unesco Tech. Pap. in Mar. Sci._, No. 44. Eqn.(31) p.39
#    Bryden, H. 1973.
#    "New Polynomials for thermal expansion, adiabatic temperature gradient
#    and potential temperature of sea water." DEEP-SEA RES., 1973, Vol 20

# theta1
del_P  = PR - P
del_th = del_P.*sw_adtg(S,T,P);
th     = T + 0.5*del_th;
q      = del_th;

# theta2
del_th = del_P.*sw_adtg(S,th,P+0.5*del_P);
th     = th + (1 - 1/sqrt(2))*(del_th - q);
q      = (2-sqrt(2))*del_th + (-2+3/sqrt(2))*q;

# theta3
del_th = del_P.*sw_adtg(S,th,P+0.5*del_P);
th     = th + (1 + 1/sqrt(2))*(del_th - q);
q      = (2 + sqrt(2))*del_th + (-2-3/sqrt(2))*q;

# theta4
del_th = del_P.*sw_adtg(S,th,P+del_P);
PT     = th + (del_th - 2*q)/6;

return PT
end

"""
    sw_adtg(S,T,P)

Calculate adiabatic temperature gradient as per UNESCO 1983 routines from salinity
(S; in psu), in situ temperature (T; in °C), and pressure (P; in decibar)
```
adtg = sw_adtg(S,T,P)
```
"""
function sw_adtg(S,T,P)
    # Original author:  Phil Morgan
    # Reference: Fofonoff, P. and Millard, R.C. Jr
    #    Unesco 1983. Algorithms for computation of fundamental properties of
    #    seawater. _Unesco Tech. Pap. in Mar. Sci._, No. 44. Eqn.(31) p.39
    #    Bryden, H. 1973.
    #    "New Polynomials for thermal expansion, adiabatic temperature gradient
    #    and potential temperature of sea water." DEEP-SEA RES., 1973, Vol 20

    a0 =  3.5803E-5;
    a1 =  8.5258E-6;
    a2 = -6.836E-8;
    a3 =  6.6228E-10;

    b0 =  1.8932E-6;
    b1 = -4.2393E-8;

    c0 =  1.8741E-8;
    c1 = -6.7795E-10;
    c2 =  8.733E-12;
    c3 = -5.4481E-14;

    d0 = -1.1351E-10;
    d1 =  2.7759E-12;

    e0 = -4.6206E-13;
    e1 =  1.8676E-14;
    e2 = -2.1687E-16;

    ADTG =      a0 + (a1 + (a2 + a3.*T).*T).*T + (b0 + b1.*T).*(S-35.0) +
    ( (c0 + (c1 + (c2 + c3.*T).*T).*T) + (d0 + d1.*T).*(S-35.0) ).*P +
    (  e0 + (e1 + e2.*T).*T ).*P.*P

    return ADTG
end

"""
    monthly_climatology_factors(date)

if `date` is a date in `days since DateTime(0)`

and `ff(rec)` returns a value for month `rec` in `1:12`

then compute the factors to interpolate from `rec[1],rec[2]` to `date`.

For example :

```
ff(x)=sin((x-0.5)/12*2pi)
fac,rec=monthly_climatology_factors(prof["date"])

gg=fac[1]*ff(rec[1])+fac[2]*ff(rec[2])
(ff(rec[1]),gg,ff(rec[2]))
```
"""
function monthly_climatology_factors(date)

    tmp2=ones(13,1)*[1991 1 1 0 0 0]; tmp2[1:12,2].=(1:12); tmp2[13,1]=1992.0;
    tmp2=[DateTime(tmp2[i,:]...) for i in 1:13]

    tim_fld=tmp2 .-DateTime(1991,1,1);
    tim_fld=1/2*(tim_fld[1:12]+tim_fld[2:13])
    tim_fld=[tim_fld[i].value for i in 1:12]/86400/1000

    tim_fld=[tim_fld[12]-365.0;tim_fld...;tim_fld[1]+365.0]
    rec_fld=[12;1:12;1]

    year0=year(DateTime(0,1,1)+Day(Int(floor(date))))
    date0=DateTime(year0,1,1)-DateTime(0)

    date0=date0.value/86400/1000+1 #+1 is to match Matlab's datenum
    tim_prof=date-date0
    tim_prof>365.0 ? tim_prof=365.0 : nothing

    tt=maximum(findall(tim_fld.<=tim_prof))
    a0=(tim_prof-tim_fld[tt])/(tim_fld[tt+1]-tim_fld[tt])

    return (1-a0,a0),(rec_fld[tt],rec_fld[tt+1])
end

end #module ArgoTools

module GriddedFields

using MeshArrays, OceanStateEstimation, Statistics

function NaN_mask(Γ)
    msk=write(Γ.hFacC)
    msk[findall(msk.>0.0)].=1.0
    msk[findall(msk.==0.0)].=NaN
    msk=MeshArrays.read(msk,Γ.hFacC)
end

function MonthlyClimatology(fil,msk)
    fid = open(fil)
    tmp = Array{Float32,4}(undef,(90,1170,50,12))
    read!(fid,tmp)
    tmp = hton.(tmp)
    close(fid)

    T=Array{MeshArray,1}(undef,12)
    for tt=1:12
        T[tt]=msk*MeshArrays.read(tmp[:,:,:,tt],msk.grid)
    end

    return T
end

function AnnualClimatology(fil,msk)
    fid = open(fil)
    tmp=Array{Float32,3}(undef,(90,1170,50))
    read!(fid,tmp)
    tmp = hton.(tmp)
    close(fid)

    T=MeshArray(msk.grid,Float64,50)
    T=msk*MeshArrays.read(convert(Array{Float64},tmp),T)
    return T
end

"""
    load()

Load gridded fields from files (and download the files if needed).
Originally this function returned `(Γ=Γ,msk,T=T,S=S,σT=σT,σS=σS)`.
"""
function load()
    γ=GridSpec("LatLonCap",MeshArrays.GRID_LLC90)
    Γ=GridLoad(γ,option="full")

    msk=NaN_mask(Γ)

    pth=MITPROFclim_path
    OceanStateEstimation.MITPROFclim_download()

    T=MonthlyClimatology(pth*"T_OWPv1_M_eccollc_90x50.bin",msk)
    S=MonthlyClimatology(pth*"S_OWPv1_M_eccollc_90x50.bin",msk)
    σT=AnnualClimatology(pth*"sigma_T_nov2015.bin",msk)
    σS=AnnualClimatology(pth*"sigma_S_nov2015.bin",msk)

    tmp=σT.grid.write(σT)
    for kk in 1:size(tmp,3)
        tmp1=tmp[:,:,kk]; tmp2=tmp1[findall((!isnan).(tmp1))];
        tmp3=max(quantile(tmp2,0.05),1e-3)
        tmp1[findall(tmp1.<tmp3)].=tmp3
        tmp[:,:,kk].=tmp1
    end
    σT=σT.grid.read(tmp,σT.grid)

    tmp=σS.grid.write(σS)
    for kk in 1:size(tmp,3)
        tmp1=tmp[:,:,kk]; tmp2=tmp1[findall((!isnan).(tmp1))];
        tmp3=max(quantile(tmp2,0.05),1e-3)
        tmp1[findall(tmp1.<tmp3)].=tmp3
        tmp[:,:,kk].=tmp1
    end
    σS=σS.grid.read(tmp,σS.grid)

    (Γ=Γ,msk,T=T,S=S,σT=σT,σS=σS)
end

function interp_h(z_in::MeshArray,f,i,j,w,z_out)
    for k in 1:50
        z_out[k]=NaN
        if !isnan(sum(w[1,:]))
            x=[z_in[f[1,ii],k][i[1,ii],j[1,ii]] for ii=1:4]
            kk=findall(isfinite.(x))
            ~isempty(kk) ? z_out[k]=sum(w[1,kk].*x[kk])/sum(w[1,kk]) : nothing
        end
    end
end

end #module GriddedFields


using Base.Test, JuLIP

if hasase
   println("Testing `jbulk == bulk` ...")
   for sym in [:Fe, :W, :Al, :Cu, :Si, :C, :Mg, :Be]
      @test isapprox(bulk(sym), Atoms(ASE.bulk(string(sym))), tol = 1e-12)
      @test isapprox(bulk(sym, cubic=true), Atoms(ASE.bulk(string(sym), cubic=true)), tol = 1e-12)
   end
end


struct HospitalInfectionParams
    ishealthcare::BitVector
    hospital_staff_ids::Vector{PersonIdx}
    kernel_constant::Float64

    healthcare_detection_prob::Float64
    healthcare_detection_delay::Float64
  end

ishealthcare(params::HospitalInfectionParams, person_id::Integer) = params.ishealthcare[person_id]

function saveparams(dict, p::HospitalInfectionParams, prefix::AbstractString="")
  dict[prefix*"kernel_constant"] = p.kernel_constant
  dict[prefix*"detection_prob"] = p.healthcare_detection_prob
  dict[prefix*"healthcare_detection_delay"] = p.healthcare_detection_delay
end


#
# This file is a part of MolecularGraph.jl
# Licensed under the MIT License http://opensource.org/licenses/MIT
#

export
    sssr, sssrmembership,
    fusedrings, fusedringmembership,
    sssrsizes, sssrcount,
    isringatom, isringbond,
    atomsymbol, charge, multiplicity, bondorder,
    nodedegree, valence, lonepair,
    heavyatomconnected, explicithconnected, implicithconnected,
    hydrogenconnected, connectivity,
    ishdonor, hacceptorcount, ishacceptor, hdonorcount,
    isrotatable, rotatablecount,
    atomcounter, heavyatomcount, molecularformula, empiricalformula,
    pielectron, hybridization,
    isaromaticring, isaromatic, isaromaticbond,
    precalculate!

const LONEPAIR_COUNT = Dict(
    :H => 0, :B => -1, :C => 0, :N => 1, :O => 2, :F => 3,
    :Si => 0, :P => 1, :S => 2, :Cl => 3,
    :As => 1, :Se => 2, :Br => 3, :I => 3
)

# Molecular graph topology

"""
    sssr(mol::UndirectedGraph) -> Vector{Vector{Int}}

Calculate Small set of smallest rings (SSSR).

This returns a vector of rings represented as a vector of atom node index along with the cycle path.
"""
sssr = Graph.mincycles

"""
    sssrmembership(mol::UndirectedGraph) -> Vector{Set{Int}}

Return size n vector of SSSR membership set where n is the number of atom nodes.

The numbers in the set correspond to the index of [`sssr`](@ref) that the node belongs.
"""
sssrmembership = Graph.mincyclemembership

"""
    fusedrings(mol::UndirectedGraph) -> Vector{Set{Int}}

Return the size ``n`` vector of ring labels within the molecule that have ``n``
fused rings (2-edge connected components in the context of graph theory).
"""
@cachefirst function fusedrings(mol::UndirectedGraph)
    cobr = setdiff(edgeset(mol), bridges(mol))
    subg = plaingraph(edgesubgraph(mol, cobr))
    return Graph.connectedcomponents(subg)
end

"""
    fusedringmembership(mol::UndirectedGraph) -> Vector{Int}

Return the membership of fused rings. See [`fusedrings`](@ref).
"""
@cachefirst function fusedringmembership(mol::UndirectedGraph)
    arr = [Set{Int}() for i in 1:nodecount(mol)]
    for (i, conn) in enumerate(fusedrings(mol))
        for n in conn
            push!(arr[n], i)
        end
    end
    return arr
end

@cachefirst function sssrsizes(mol::UndirectedGraph)
    ssrs_ = sssr(mol)
    arr = Set{Int}[]
    for ks in sssrmembership(mol)
        push!(arr, Set{Int}([length(ssrs_[k]) for k in ks]))
    end
    return arr
end

@cachefirst sssrcount(mol::UndirectedGraph) = length.(sssrmembership(mol))

@cachefirst isringatom(mol::UndirectedGraph) = .!isempty.(sssrmembership(mol))
isringatom(view::SubgraphView) = isringatom(view.graph)

@cachefirst isringbond(mol::UndirectedGraph
    ) = .!isempty.(edgemincyclemembership(mol))
isringbond(view::SubgraphView) = isringbond(view.graph)

# Elemental properties

"""
    atomsymbol(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of atom symbols within the molecule that have ``n``
atoms.
"""
@cachefirst atomsymbol(mol::GraphMol) = getproperty.(nodeattrs(mol), :symbol)
atomsymbol(view::SubgraphView) = atomsymbol(view.graph)

"""
    charge(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of atom charges within the molecule that have ``n``
atoms.
"""
@cachefirst charge(mol::GraphMol) = getproperty.(nodeattrs(mol), :charge)
charge(view::SubgraphView) = charge(view.graph)

"""
    multiplicity(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of atom multiplicities within the molecule that
have ``n`` atoms (1: non-radical, 2: radical, 3: biradical).
"""
@cachefirst multiplicity(mol::GraphMol
    ) = getproperty.(nodeattrs(mol), :multiplicity)
multiplicity(view::SubgraphView) = multiplicity(view.graph)

"""
    bondorder(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of bond orders within the molecule that have ``n``
bonds.
"""
@cachefirst bondorder(mol::GraphMol) = getproperty.(edgeattrs(mol), :order)
bondorder(view::SubgraphView) = bondorder(view.graph)

"""
    nodedegree(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of node degrees within the molecular graph that
have ``n`` atom nodes.

`nodedegree` values correspond to SMARTS `D` property.
"""
@cachefirst nodedegree(mol::GraphMol
    ) = [degree(mol, n) for n in 1:nodecount(mol)]
nodedegree(view::SubgraphView) = nodedegree(view.graph)

# Valence

"""
    lonepair(mol::GraphMol) -> Vector{Union{Int,Nothing}}

Return the size ``n`` vector of the number of lone pairs within the molecule that
have ``n`` atoms.

Note that implicit hydrogens are available for only organic atoms. The lonepair
value of inorganic atoms would be `nothing`. The result can take negative value if the atom has empty valence shells.
"""
@cachefirst function lonepair(mol::GraphMol)
    vec = Union{Int,Nothing}[]
    atomsymbol_ = atomsymbol(mol)
    charge_ = charge(mol)
    for i in 1:nodecount(mol)
        num = get(LONEPAIR_COUNT, atomsymbol_[i], nothing)
        v = num === nothing ? nothing : num - charge_[i]
        push!(vec, v)
    end
    return vec
end

lonepair(view::SubgraphView) = lonepair(view.graph)

@cachefirst function apparentvalence(mol::GraphMol)
    vec = zeros(Int, nodecount(mol))
    bondorder_ = bondorder(mol)
    for i in 1:nodecount(mol)
        for (inc, adj) in neighbors(mol, i)
            vec[i] += bondorder_[inc]
        end
    end
    return vec
end

apparentvalence(view::SubgraphView) = apparentvalence(view.graph)

"""
    valence(mol::GraphMol) -> Vector{Union{Int,Nothing}}

Return the size ``n`` vector of intrinsic valences (with considering implicit
hydrogens) within the molecule that have ``n`` atoms.

`valence` values correspond to SMARTS `v` property. `valence` values of inorganic atoms would be `nothing`.
"""
@cachefirst function valence(mol::GraphMol)
    vec = Union{Int,Nothing}[]
    atomsymbol_ = atomsymbol(mol)
    lonepair_ = lonepair(mol)
    for i in 1:nodecount(mol)
        if lonepair_[i] === nothing
            push!(vec, nothing)
        elseif atomsymbol_[i] === :H
            push!(vec, 1)
        else
            push!(vec, 4 - abs(lonepair_[i]))
        end
    end
    return vec
end

valence(view::SubgraphView) = valence(view.graph)

"""
    explicithconnected(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of the number of adjacent explicit hydrogen nodes within the molecule that have ``n`` atoms.
"""
@cachefirst function explicithconnected(mol::GraphMol)
    vec = zeros(Int, nodecount(mol))
    atomsymbol_ = atomsymbol(mol)
    for i in 1:nodecount(mol)
        for (inc, adj) in neighbors(mol, i)
            if atomsymbol_[adj] === :H
                vec[i] += 1
            end
        end
    end
    return vec
end

explicithconnected(view::SubgraphView) = explicithconnected(view.graph)

"""
    implicithconnected(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of the number of implicit hydrogens within
the molecule that have ``n`` atoms.
"""
@cachefirst function implicithconnected(mol::GraphMol)
    hcnt = (v, av) -> v === nothing ? 0 : max(0, v - av)
    return hcnt.(valence(mol), apparentvalence(mol))
end

implicithconnected(view::SubgraphView) = implicithconnected(view.graph)

"""
    heavyatomconnected(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of the number of adjacent non-hydrogen atoms within
the molecule that have ``n`` atoms.
"""
@cachefirst heavyatomconnected(mol::GraphMol
    ) = nodedegree(mol) - explicithconnected(mol)
heavyatomconnected(view::SubgraphView) = heavyatomconnected(view.graph)

"""
    hydrogenconnected(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of the total number of hydrogens attached to the atom within the molecule that have ``n`` atoms.

`hydrogenconnected` values correspond to SMARTS `H` property.
"""
@cachefirst hydrogenconnected(mol::GraphMol
    ) = explicithconnected(mol) + implicithconnected(mol)
hydrogenconnected(view::SubgraphView) = hydrogenconnected(view.graph)

"""
    connectivity(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of the total number of adjacent atoms (including implicit hydrogens) within the molecule that have ``n`` atoms.

`connectivity` values correspond to SMARTS `X` property.
"""
@cachefirst connectivity(mol::GraphMol
    ) = nodedegree(mol) + implicithconnected(mol)
connectivity(view::SubgraphView) = connectivity(view.graph)

# Hydrogen bond donor/acceptor

@cachefirst function ishacceptor(mol::GraphMol)
    ac = (sym, lp) -> lp === nothing ? false : sym in (:N, :O, :F) && lp > 0
    return ac.(atomsymbol(mol), lonepair(mol))
end

ishacceptor(view::SubgraphView) = ishacceptor(view.graph)

"""
    hacceptorcount(mol::GraphMol) -> Int

Return the number of hydrogen bond acceptors (N, O and F).
"""
hacceptorcount(mol::GraphMol) = reduce(+, ishacceptor(mol); init=0)

@cachefirst function ishdonor(mol::GraphMol)
    dc = (sym, h) -> sym in (:N, :O) && h > 0
    return dc.(atomsymbol(mol), hydrogenconnected(mol))
end

ishdonor(view::SubgraphView) = ishdonor(view.graph)

"""
    hdonorcount(mol::GraphMol) -> Int

Return the number of hydrogen bond donors (O and N attached to hydrogens).
"""
hdonorcount(mol::GraphMol) = reduce(+, ishdonor(mol); init=0)

# Rotatable bonds

"""
    isrotatable(mol::GraphMol)

Return whether the bonds are rotatable or not.
"""
@cachefirst function isrotatable(mol::GraphMol)
    nodedegree_ = nodedegree(mol)
    isringbond_ = isringbond(mol)
    bondorder_ = bondorder(mol)
    vec = Bool[]
    for (i, (u, v)) in enumerate(edgesiter(mol))
        rot = (!isringbond_[i] && bondorder_[i] == 1
            && nodedegree_[u] != 1 && nodedegree_[v] != 1)
        push!(vec, rot)
    end
    return vec
end

isrotatable(view::SubgraphView) = isrotatable(view.graph)

"""
    rotatablecount(mol::GraphMol) -> Int

Return the number of rotatable bonds.
"""
rotatablecount(mol::GraphMol) = reduce(+, isrotatable(mol); init=0)

# Composition

"""
    atomcounter(mol::GraphMol) -> Dict{Symbol,Int}

Count the number of atoms and return symbol => count dict.
"""
@cachefirst function atomcounter(mol::GraphMol)
    counter = Dict{Symbol,Int}()
    for sym in atomsymbol(mol)
        if !haskey(counter, sym)
            counter[sym] = 1
        else
            counter[sym] += 1
        end
    end
    hcnt = reduce(+, implicithconnected(mol); init=0)
    if hcnt > 0
        if !haskey(counter, :H)
            counter[:H] = hcnt
        else
            counter[:H] += hcnt
        end
    end
    return counter
end

"""
    heavyatomcount(mol::GraphMol) -> Int

Return the total number of non-hydrogen atoms.
"""
function heavyatomcount(mol::GraphMol)
    counter = atomcounter(mol)
    delete!(counter, :H)
    return reduce(+, values(counter); init=0)
end

function writeformula(counter::Dict{Symbol,Int})
    strs = []
    if haskey(counter, :C)
        push!(strs, "C")
        c = pop!(counter, :C)
        c > 1 && push!(strs, string(c))
        if haskey(counter, :H)
            push!(strs, "H")
            h = pop!(counter, :H)
            h > 1 && push!(strs, string(h))
        end
    end
    for sym in sort(collect(keys(counter)))
        push!(strs, string(sym))
        cnt = counter[sym]
        cnt > 1 && push!(strs, string(cnt))
    end
    return join(strs)
end

"""
    molecularformula(mol::GraphMol) -> String

Return the molecular formula in Hill system.
"""
function molecularformula(mol::GraphMol)
    counter = atomcounter(mol)
    return writeformula(counter)
end

"""
    empiricalformula(mol::GraphMol) -> String

Return the empirical formula in Hill system.
"""
function empiricalformula(mol::GraphMol)
    counter = atomcounter(mol)
    if length(counter) > 1
        dv = reduce(gcd, values(counter))
        for k in keys(counter)
            counter[k] = div(counter[k], dv)
        end
    end
    return writeformula(counter)
end

# Hybridization

"""
    pielectron(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of the number of ``\\pi`` electrons (including implicit hydrogens) within the molecule that have ``n`` atoms.

The ``\\pi`` electron enumelation is based on the following simple rules.

- Any atom incident to a double bond -> +1
- Any atom incident to two double bond -> +2
- Any atom incident to a triple bond -> +1
- Uncharged N and O not incident to any multiple bonds and adjacent to atoms described above -> +2

These rules are applied for only typical organic atoms. The values for inorganic atoms will be 0.
"""
@cachefirst function pielectron(mol::GraphMol)
    atomsymbol_ = atomsymbol(mol)
    charge_ = charge(mol)
    pie_ = apparentvalence(mol) - nodedegree(mol)
    vec = zeros(Int, nodecount(mol))
    for i in 1:nodecount(mol)
        vec[i] = pie_[i]
        atomsymbol_[i] in (:N, :O) || continue
        for (inc, adj) in neighbors(mol, i)
            if pie_[i] == 0 && pie_[adj] > 0 && charge_[i] == 0
                vec[i] = 2
                break
            end
        end
    end
    return vec
end

pielectron(view::SubgraphView) = pielectron(view.graph)

"""
    hybridization(mol::GraphMol) -> Vector{Int}

Return the size ``n`` vector of orbital hybridization symbols (`:sp3`, `:sp2`, `:sp`, `:none`) within the molecule that have ``n`` atoms.

These hybridizations are for only typical organic atoms. Inorganic atoms and other orbitals like s, sp3d and sp3d2 will be `:none`.
"""
@cachefirst function hybridization(mol::GraphMol)
    atomsymbol_ = atomsymbol(mol)
    pielectron_ = pielectron(mol)
    connectivity_ = connectivity(mol)
    lonepair_ = lonepair(mol)
    vec = Symbol[]
    for i in 1:nodecount(mol)
        if lonepair_[i] === nothing || atomsymbol_[i] === :H
            push!(vec, :none)
            continue
        end
        orbitals = connectivity_[i] + lonepair_[i]
        if orbitals == 4
            if (atomsymbol_[i] in [:N, :O] && pielectron_[i] == 2)
                push!(vec, :sp2)  # adjacent to conjugated bonds
            else
                push!(vec, :sp3)
            end
        elseif orbitals == 3
            push!(vec, :sp2)
        elseif orbitals == 2
            push!(vec, :sp)
        else
            push!(vec, :none)
        end
    end
    return vec
end

hybridization(view::SubgraphView) = hybridization(view.graph)

# Aromatic rings

@cachefirst function isaromaticring(mol::GraphMol)
    atomsymbol_ = atomsymbol(mol)
    nodedegree_ = nodedegree(mol)
    pie_ = apparentvalence(mol) - nodedegree(mol)
    lonepair_ = lonepair(mol)
    carbonylO = findall(
        (atomsymbol_ .=== :O) .* (nodedegree_ .== 1) .* (pie_ .== 1))
    carbonylC = falses(nodecount(mol))
    for o in carbonylO
        c = pop!(adjacencies(mol, o))
        if atomsymbol_[c] === :C
            carbonylC[c] = true
        end
    end
    arr = falses(circuitrank(mol))
    for (i, ring) in enumerate(sssr(mol))
        cnt = 0
        for r in ring
            if carbonylC[r]
                continue
            elseif pie_[r] == 1
                cnt += 1
            elseif lonepair_[r] === nothing
                cnt = 0
                break
            elseif lonepair_[r] > 0
                cnt += 2
            elseif lonepair_[r] < 0
                continue
            else
                cnt = 0
                break
            end
        end
        if cnt % 4 == 2
            # Huckel rule check
            arr[i] = true
            continue
        end
        if nodeattrtype(mol) === SmilesAtom
            # SMILES aromatic atom
            sub = nodesubgraph(mol, Set(ring))
            if all(nodeattr(mol, n).isaromatic for n in nodeset(sub))
                arr[i] = true
            end
        end
    end
    return arr
end

"""
    isaromatic(mol::GraphMol)

Return the vector whether the atom belongs to an aromatic ring.

Note that aromaticity described here means simplified binary descriptor
(aromatic or not) based on classical Huckel's rule. This is intended for use in
some kind of pharmaceutical researches. Non-classical aromaticity such as
Moebius aromaticity is not considered.
"""
@cachefirst function isaromatic(mol::GraphMol)
    aromatic = falses(nodecount(mol))
    for ring in sssr(mol)[isaromaticring(mol)]
        sub = nodesubgraph(mol, Set(ring))
        for n in nodeset(sub)
            aromatic[n] = true
        end
    end
    return aromatic
end

isaromatic(view::SubgraphView) = isaromatic(view.graph)

@cachefirst function isaromaticbond(mol::GraphMol)
    aromaticbond = falses(edgecount(mol))
    for ring in sssr(mol)[isaromaticring(mol)]
        sub = nodesubgraph(mol, Set(ring))
        for e in edgeset(sub)
            aromaticbond[e] = true
        end
    end
    return aromaticbond
end

isaromaticbond(view::SubgraphView) = isaromaticbond(view.graph)

"""
    precalculate!(mol::GraphMol)

Convenient method to pre-calculate and cache performance bottleneck descriptors.
"""
function precalculate!(mol)
    @cache edgemincycles(mol)
    @cache mincycles(mol)
    @cache lonepair(mol)
    @cache apparentvalence(mol)
    @cache valence(mol)
    nodeattrtype(mol) === SmilesAtom && @cache coordgen(mol)
end


__precompile__(true)

module FiniteDiff
    include("modes.jl")
    include("step_size.jl")
    include("derivative.jl")
    include("second_derivative.jl")
    include("gradient.jl")
    include("hessian.jl")
end


"""
The **pcl_surface** library deals with reconstructing the original surfaces
from 3D scans.

http://docs.pointclouds.org/trunk/group__surface.html

## Exports

$(EXPORTS)
"""
module PCLSurface

using DocStringExtensions
using LibPCL
using PCLCommon
using Cxx

const libpcl_surface = LibPCL.find_library_e("libpcl_surface")
try
    Libdl.dlopen(libpcl_surface, Libdl.RTLD_GLOBAL)
catch e
    warn("You might need to set DYLD_LIBRARY_PATH to load dependencies proeprty.")
    rethrow(e)
end

end # module


function LeastSquaresBuilder()
    return LeastSquaresBuilder(())
end

function build(obj::LeastSquaresBuilder)
    return jcall(obj, "build", LeastSquaresProblem, ())
end

function checker(obj::LeastSquaresBuilder, arg0::ConvergenceChecker)
    return jcall(obj, "checker", LeastSquaresBuilder, (ConvergenceChecker,), arg0)
end

function checker_pair(obj::LeastSquaresBuilder, arg0::ConvergenceChecker)
    return jcall(obj, "checkerPair", LeastSquaresBuilder, (ConvergenceChecker,), arg0)
end

function lazy_evaluation(obj::LeastSquaresBuilder, arg0::jboolean)
    return jcall(obj, "lazyEvaluation", LeastSquaresBuilder, (jboolean,), arg0)
end

function max_evaluations(obj::LeastSquaresBuilder, arg0::jint)
    return jcall(obj, "maxEvaluations", LeastSquaresBuilder, (jint,), arg0)
end

function max_iterations(obj::LeastSquaresBuilder, arg0::jint)
    return jcall(obj, "maxIterations", LeastSquaresBuilder, (jint,), arg0)
end

function model(obj::LeastSquaresBuilder, arg0::MultivariateJacobianFunction)
    return jcall(obj, "model", LeastSquaresBuilder, (MultivariateJacobianFunction,), arg0)
end

function model(obj::LeastSquaresBuilder, arg0::MultivariateVectorFunction, arg1::MultivariateMatrixFunction)
    return jcall(obj, "model", LeastSquaresBuilder, (MultivariateVectorFunction, MultivariateMatrixFunction), arg0, arg1)
end

function parameter_validator(obj::LeastSquaresBuilder, arg0::ParameterValidator)
    return jcall(obj, "parameterValidator", LeastSquaresBuilder, (ParameterValidator,), arg0)
end

function start(obj::LeastSquaresBuilder, arg0::RealVector)
    return jcall(obj, "start", LeastSquaresBuilder, (RealVector,), arg0)
end

function start(obj::LeastSquaresBuilder, arg0::Vector{jdouble})
    return jcall(obj, "start", LeastSquaresBuilder, (Vector{jdouble},), arg0)
end

function target(obj::LeastSquaresBuilder, arg0::RealVector)
    return jcall(obj, "target", LeastSquaresBuilder, (RealVector,), arg0)
end

function target(obj::LeastSquaresBuilder, arg0::Vector{jdouble})
    return jcall(obj, "target", LeastSquaresBuilder, (Vector{jdouble},), arg0)
end

function weight(obj::LeastSquaresBuilder, arg0::RealMatrix)
    return jcall(obj, "weight", LeastSquaresBuilder, (RealMatrix,), arg0)
end


function writeStamp(dirname::String, t::Float64, surf::ThreeDSurfSimple, curfield::ThreeDFieldSimple)
    dirvec = readdir()
    if dirname in dirvec
        rm(dirname, recursive=true)
    end
    mkdir(dirname)
    cd(dirname)

    f = open("time", "w")
    Serialization.Serialization.serialize(f, ["#time \t"])
    DelimitedFiles.writedlm(f, [t]')
    close(f)

    for i = 1:surf.nspan
        f = open("kinem-$i", "w")
        Serialization.Serialization.serialize(f, ["alpha (deg) \t", "h/c \t", "u/uref \t"])
        DelimitedFiles.writedlm(f, [surf.s2d[i].kinem.alpha, surf.s2d[i].kinem.h, surf.s2d[i].kinem.u]')
        close(f)

        f = open("FourierCoeffs-$i", "w")
        Serialization.serialize(f, ["#Fourier coeffs (0-n) \t", "d/dt (Fourier coeffs) \n"])
        matfour = zeros(surf.s2d[i].naterm+1, 2)
        matfour[:,1] = [surf.s2d[i].a0[1]; surf.s2d[i].aterm[:]]
        matfour[1:4,2] = [surf.s2d[i].a0dot[1];surf.s2d[i].adot[:]]
        DelimitedFiles.writedlm(f, matfour)
        close(f)

        f = open("tev-$i", "w")
        Serialization.serialize(f, ["#strength \t", "x-position \t", "z-position \n"])
        tevmat = zeros(length(curfield.f2d[i].tev), 3)
        for iv = 1:length(curfield.f2d[i].tev)
            tevmat[iv,:] = [curfield.f2d[i].tev[iv].s curfield.f2d[i].tev[iv].x curfield.f2d[i].tev[iv].z]
        end
        DelimitedFiles.writedlm(f, tevmat)
        close(f)

        f = open("lev-$i", "w")
        Serialization.serialize(f, ["#strength \t", "x-position \t", "z-position \n"])
        levmat = zeros(length(curfield.f2d[i].lev), 3)
        for iv = 1:length(curfield.f2d[i].lev)
            levmat[iv,:] = [curfield.f2d[i].lev[iv].s curfield.f2d[i].lev[iv].x curfield.f2d[i].lev[iv].z]
        end
        DelimitedFiles.writedlm(f, levmat)
        close(f)

        f = open("boundv-$i", "w")
        Serialization.serialize(f, ["#strength \t", "x-position \t", "z-position \n"])
        bvmat = zeros(length(surf.s2d[i].bv), 3)
        for iv = 1:length(surf.s2d[i].bv)
            bvmat[iv,:] = [surf.s2d[i].bv[iv].s surf.s2d[i].bv[iv].x surf.s2d[i].bv[iv].z]
        end
        DelimitedFiles.writedlm(f, bvmat)
        close(f)
    end

    #Write spanwise properties of interest in a file
    f = open("spanwise-var", "w")
    Serialization.serialize(f, ["#y_le \t", "shed TEV str \t", "shed LEV str \t", "Gamma \t", "A0 2D \t", "A03D \t", "A0 tot \n"])
    mat = zeros(0,7)
    mat = [mat; [-surf.AR/2, 0. , 0. , 0. , 0. , 0. , 0. ]']
    for i = 1:surf.nspan
        yle = surf.yle[i]
        tev_s = curfield.f2d[i].tev[end].s
        if surf.s2d[i]. levflag == 1
            lev_s = curfield.f2d[i].lev[end].s
        else
            lev_s = 0.
        end
        gam = surf.s2d[i].uref*surf.s2d[i].c*pi*(surf.s2d[i].a0[1] + 0.5*surf.s2d[i].aterm[1])
        a02d = surf.s2d[i].a0[1] - surf.a03d[i]

        mat = [mat; [yle, tev_s, lev_s, gam, a02d, surf.a03d[i], surf.s2d[i].a0[1]]']
    end
    DelimitedFiles.writedlm(f, mat)
    close(f)

    cd("..")
end

function calc_forces(surf::ThreeDSurfSimple, field :: ThreeDFieldSimple, dt :: Float64)

    cl = zeros(surf.nspan)
    cd = zeros(surf.nspan)
    cm = zeros(surf.nspan)

    lev_s = zeros(surf.nspan)
    for i = 1:surf.nspan
        if surf.s2d[i].levflag[1] == 1
            lev_s[i] = field.f2d[i].lev[end].s
        end
    end

    for i = 1:surf.nspan
        # First term in eqn (2.30) Ramesh et al. in coefficient form
        wi = 0
        for n = 1:surf.nspan
            nn = 2*n -2
            wi += real(nn)*surf.bcoeff[n]*sin(nn*surf.psi[i])/sin(surf.psi[i])
        end

        cnc = 2*pi*(surf.s2d[i].kinem.u*cos(surf.s2d[i].kinem.alpha)/surf.s2d[i].uref + surf.s2d[i].kinem.hdot*sin(surf.s2d[i].kinem.alpha)/surf.s2d[i].uref + wi*sin(surf.s2d[i].kinem.alpha))*(surf.s2d[i].a0[1] + surf.s2d[i].aterm[1]/2.)

        # Second term in eqn (2.30) Ramesh et al. in coefficient form
        cnnc = 2*pi*(3*surf.s2d[i].c*surf.s2d[i].a0dot[1]/(4*surf.s2d[i].uref) + surf.s2d[i].c*surf.s2d[i].adot[1]/(4*surf.s2d[i].uref) + surf.s2d[i].c*surf.s2d[i].adot[2]/(8*surf.s2d[i].uref)) + (2*lev_s[i]/(dt*surf.s2d[i].uref^2))

        # Suction force given in eqn (2.31) Ramesh et al.
        cs = 2*pi*surf.s2d[i].a0[1]*surf.s2d[i].a0[1]

        #The components of normal force and moment from induced velocities are calulcated in dimensional units and nondimensionalized later
        nonl=0
        nonl_m=0
        for ib = 1:surf.s2d[i].ndiv-1
            nonl = nonl + (surf.s2d[i].uind[ib]*cos(surf.s2d[i].kinem.alpha) - surf.s2d[i].wind[ib]*sin(surf.s2d[i].kinem.alpha))*surf.s2d[i].bv[ib].s
            nonl_m = nonl_m + (surf.s2d[i].uind[ib]*cos(surf.s2d[i].kinem.alpha) - surf.s2d[i].wind[ib]*sin(surf.s2d[i].kinem.alpha))*surf.s2d[i].x[ib]*surf.s2d[i].bv[ib].s
        end
        nonl = nonl*2. /(surf.s2d[i].uref*surf.s2d[i].uref*surf.s2d[i].c)
        nonl_m = nonl_m*2. /(surf.s2d[i].uref*surf.s2d[i].uref*surf.s2d[i].c*surf.s2d[i].c)

        # Normal force coefficient
        cn = cnc + cnnc + nonl

        # Lift and drag coefficients
        cl[i] = cn*cos(surf.s2d[i].kinem.alpha) + cs*sin(surf.s2d[i].kinem.alpha)
        cd[i] = cn*sin(surf.s2d[i].kinem.alpha) - cs*cos(surf.s2d[i].kinem.alpha)

        #Pitching moment is clockwise or nose up positive
        cm[i] = cn*surf.s2d[i].pvt - 2*pi*((surf.s2d[i].kinem.u*cos(surf.s2d[i].kinem.alpha)/surf.s2d[i].uref + surf.s2d[i].kinem.hdot*sin(surf.s2d[i].kinem.alpha)/surf.s2d[i].uref)*(surf.s2d[i].a0[1]/4. + surf.s2d[i].aterm[1]/4. - surf.s2d[i].aterm[2]/8.) + (surf.s2d[i].c/surf.s2d[i].uref)*(7. *surf.s2d[i].a0dot[1]/16. + 3. *surf.s2d[i].adot[1]/16. + surf.s2d[i].adot[2]/16. - surf.s2d[i].adot[3]/64. )) - nonl_m
    end

    cl3d = 0
    cd3d = 0
    cm3d = 0

    for is = 1:surf.nspan
        if is == 1
            yl = -0.5*surf.AR*surf.cref
            yu = 0.5*(surf.yle[is] + surf.yle[is+1])
        elseif is == surf.nspan
            yl = 0.5*(surf.yle[is] + surf.yle[is-1])
            yu = 0.
        else
            yl = 0.5*(surf.yle[is] + surf.yle[is-1])
            yu = 0.5*(surf.yle[is] + surf.yle[is+1])
        end
        cl3d += cl[is]*(yu-yl)
        cd3d += cd[is]*(yu-yl)
        cm3d += cm[is]*(yu-yl)
    end
    cl3d = cl3d/(0.5*surf.AR*surf.cref)
    cd3d = cd3d/(0.5*surf.AR*surf.cref)
    cm3d = cm3d/(0.5*surf.AR*surf.cref)

    return cl3d, cd3d, cm3d, cl, cd, cm
end


"""
A package for fitting data from auditory evidence accumulation task (Poisson clicks task)
to evidence accumulation model.
"""
__precompile__()

module PBupsModel

# 3rd party
import Base.convert

using MAT
using ForwardDiff
using Optim
using GeneralUtils

import ForwardDiff.DiffBase
# using DiffBase

export

# data_handle
    LoadData,
    TrialData,
    WriteData,

# model_likelihood
    logProbRight,
    LogLikelihood,

# all_trials
    ComputeLL,
    ComputeLL_bbox,
    ComputeGrad,
    ComputeHess,
    TrialsLikelihood,

# optimization
    InitParams,
    ModelFitting,
    FitSummary

include("data_handle.jl")
include("model_likelihood.jl")
include("all_trials.jl")
include("model_optimization.jl")

end # module


using SimpleSMC
using ProgressMeter

function _ccXpf!(model::SMCModel, ccsmcio::CCSMCIO{Particle}, lM::F,
  rngCouple::Bool) where {Particle, F<:Function}
  zetas1 = ccsmcio.smcio1.zetas
  zetaAncs1 = ccsmcio.smcio1.internal.zetaAncs
  lws1 = ccsmcio.smcio1.internal.lws
  ws1 = ccsmcio.smcio1.ws
  as1 = ccsmcio.smcio1.internal.as
  pScratch = ccsmcio.smcio1.internal.particleScratch
  logZhats1 = ccsmcio.smcio1.logZhats
  lZ1 = 0.0

  zetas2 = ccsmcio.smcio2.zetas
  zetaAncs2 = ccsmcio.smcio2.internal.zetaAncs
  lws2 = ccsmcio.smcio2.internal.lws
  ws2 = ccsmcio.smcio2.ws
  as2 = ccsmcio.smcio2.internal.as
  logZhats2 = ccsmcio.smcio2.logZhats
  lZ2 = 0.0

  for p = 1:ccsmcio.n
    if p > 1
      _copyParticles!(zetaAncs1, zetas1, as1)
      _copyParticles!(zetaAncs2, zetas2, as2)
    end

    if rngCouple
      _rngCoupledMutateParticles!(zetas1, zetas2, model.M!, p, zetaAncs1,
        zetaAncs2, pScratch, ccsmcio.ref1[p], ccsmcio.ref2[p], ccsmcio.extraRNG)
    else
      _indexCoupledMutateParticles!(zetas1, zetas2, model.M!, p, zetaAncs1,
        zetaAncs2, pScratch, ccsmcio.ref1[p], ccsmcio.ref2[p])
    end

    _logWeightParticles!(lws1, p, model.lG, zetas1, pScratch)
    _logWeightParticles!(lws2, p, model.lG, zetas2, pScratch)

    maxlw::Float64 = maximum(lws1)
    ws1 .= exp.(lws1 .- maxlw)
    mws::Float64 = mean(ws1)
    lZ1 += maxlw + log(mws)
    ws1 ./= mws
    @inbounds logZhats1[p] = lZ1

    maxlw = maximum(lws2)
    ws2 .= exp.(lws2 .- maxlw)
    mws = mean(ws2)
    lZ2 += maxlw + log(mws)
    ws2 ./= mws
    @inbounds logZhats2[p] = lZ2

    if p < ccsmcio.n
      _coupledResample!(ccsmcio)
      lM != error && _ancestorSample!(ccsmcio, p, lM)
    end

    _intermediateOutput!(ccsmcio.smcio1, p)
    _intermediateOutput!(ccsmcio.smcio2, p)
  end
end

function ccXpf!(model::SMCModel, lM::F, ccsmcio::CCSMCIO{Particle},
  algorithm::Symbol = :BS, rngCouple::Bool = false) where {F<:Function, Particle}
  if algorithm == :AS
    _ccXpf!(model, ccsmcio, lM, rngCouple)
    _pickParticles!(ccsmcio)
  else
    _ccXpf!(model, ccsmcio, error, rngCouple)
    if algorithm == :BS
      _pickParticlesBS!(ccsmcio, lM)
    else
      _pickParticles!(ccsmcio)
    end
  end
end

function ccXpf!(model::SMCModel, ccsmcio::CCSMCIO{Particle},
  rngCouple::Bool = false) where Particle
  ccXpf!(model, error, ccsmcio, :AT, rngCouple)
end

function initializeCCSMC(model::SMCModel, lM::F, ccsmcio::CCSMCIO{Particle},
  algorithm::Symbol = :BS, independent::Bool = false) where
  {F<:Function, Particle}
  ref1 = ccsmcio.ref1
  ref2 = ccsmcio.ref2

  smcio1 = ccsmcio.smcio1
  smcio2 = ccsmcio.smcio2

  SimpleSMC.smc!(model, smcio1)
  SimpleSMC.pickParticle!(ref1, smcio1)

  if independent
    SimpleSMC.smc!(model, smcio2)
    SimpleSMC.pickParticle!(ref2, smcio2)
  else
    SimpleSMC._copyParticles!(ref2, ref1)
  end
  if algorithm == :BS || algorithm == :AS
    SimpleSMC.csmc!(model, lM, smcio1, ref1, ref1)
  else
    SimpleSMC.csmc!(model, smcio1, ref1, ref1)
  end
end

function initializeCCSMC(model::SMCModel, ccsmcio::CCSMCIO{Particle},
  independent::Bool = false) where Particle
  initializeCCSMC(model, error, ccsmcio, :AT, independent)
end


###################################################################################
# Antony Schutz 2015, ANR - POLCA - 2016
# TOOLS for PAINTER
# Schutz, A., Ferrari, A., Mary, D., Soulez, F., Thiébaut, E., Vannier, M.
# PAINTER: a spatiospectral image reconstruction algorithm for optical interferometry
# J. Opt. Soc. Am. A, Vol. 31, N. 11, pp 2334--2345, 2014.
#
# PAINTER Outpt
#
###################################################################################
# using PyPlot
# verb
function painterplotfct(PDATA::PAINTER_Data,OIDATA::PAINTER_Input)
    x = PDATA.x
    if sum(PDATA.w)==0
      w = ones(PDATA.w)
    else
      w = PDATA.w
    end

    crit1 = PDATA.crit1
    crit2 = PDATA.crit2
    eps1 = OIDATA.eps1
    eps2 = OIDATA.eps2
    wvl = OIDATA.wvl
    nx = OIDATA.nx
    nw = OIDATA.nw
    FOV = OIDATA.FOV
    SubColumn,SubRow = createsubplotindex(nw)

    indpix = linspace(-(FOV / 2), (FOV / 2), nx)
    @printf("stop crit, primal: %e (%e) \t dual %e (%e)\n", crit1[end], eps1, crit2[end], eps2)
###################################################################################
# Reconstructed Cube
###################################################################################
    figure(1)
    clf()
    pos     = round(Int, [1, round(Int, nx / 4), round(Int, nx / 2), round(Int, nx * 3 / 4), nx])
    count_y = 0
    count_x = 0
    for n in 1:nw
        subplot(SubColumn, SubRow, n)
        imshow(x[:, :, n] .* max(0., w[:, :, n]), origin ="lower")
        titlestring = @sprintf("%2.4f µm", wvl[n] * 1e6)
        title(titlestring)
        xticks([])
        yticks([])
        if( n == (nw + 1 - SubRow + count_x) )
            xticks(pos - 1, round(Int, indpix[pos] * 100000) / 100)
            xlabel("FOV (mas)")
            count_x += 1
        end
        if(n == (1 + count_y * SubRow))
            yticks(pos - 1, round(Int, indpix[pos] * 100000) / 100)
            ylabel("FOV (mas)")
            count_y += 1
        end
    end
###################################################################################
# primal and dual residuals plot
###################################################################################
    figure(2)
    clf()
    label1 = "Primal Residual"
    label2 = "Dual Residual"
    label3 = "Primal Threshold"
    label4 = "Dual Threshold"
    figure(2)
    plt_1 = plot(crit1, color="blue", label = label1)
    plt_2 = plot(crit2, color="red" , label = label2)
    plt_3 = plot(eps1 * ones(crit1), linestyle = "--", color = "blue", label = label3)
    plt_4 = plot(eps2 * ones(crit2), linestyle = "--", color = "red" , label = label4)
    ax1 = gca()
    ax1[:set_yscale]("log")
    xlabel("Iteration")
    ylabel("Residual (semilog)")
    title( "Primal and Dual residual")
    legend([plt_1, plt_2, plt_3, plt_4],[label1, label2, label3, label4], loc = "upper right", fancybox = "true")
    grid("on")
    #  show()
end
###################################################################################
# PLOT TOOLS
# ---------------------------------------------------------------------------------
# Automatic Subplot index creation
# (SubColumn,SubRow)= SubIndexPlot(nw::Integer)
function createsubplotindex(nw::Integer)
    if(nw == 1)
        SubColumn = 1
        SubRow = 1
    elseif(nw == 2)
        SubColumn = 1
        SubRow = 2
    elseif(nw == 3)
        SubColumn = 1
        SubRow = 3
    elseif(nw == 4)
        SubColumn = 2
        SubRow = 2
    elseif(nw == 5)
        SubColumn = 2
        SubRow = 3
    elseif(nw == 6)
        SubColumn = 2
        SubRow = 3
    elseif(nw == 7)
        SubColumn = 2
        SubRow = 4
    elseif(nw == 8)
        SubColumn = 2
        SubRow = 4
    elseif(nw == 9)
        SubColumn = 3
        SubRow = 3
    elseif(nw == 10)
        SubColumn = 2
        SubRow = 5
    elseif(nw == 11)
        SubColumn = 3
        SubRow = 4
    elseif(nw == 12)
        SubColumn = 3
        SubRow = 4
    elseif(nw == 13)
        SubColumn = 3
        SubRow = 5
    elseif(nw == 14)
        SubColumn = 3
        SubRow = 5
    elseif(nw == 15)
        SubColumn = 3
        SubRow = 5
    elseif(nw == 16)
        SubColumn = 4
        SubRow = 4
    elseif(nw == 18)
        SubColumn = 3
        SubRow = 6
    else
        println(" Automatic subplot, can be")
        SubColumn = round(Int, sqrt(nw))
        SubRow = round(Int, .5+ sqrt(nw))
    end
    return int(SubColumn), int(SubRow)
end


using .LatticePhysics.LatPhysBase: from, to, numSites, bonds, sites, latticeVectors, unitcell,
                   point, AbstractSite
import .LatticePhysics.LatPhysBase, .LatticePhysics

struct LatPhysLattice{LT <: LatPhysBase.AbstractLattice} <: AbstractLattice
    lattice::LT
    neighs::Matrix{Int}
    bonds::Vector{Vector{Int}}
end
export LatPhysLattice

function LatPhysLattice(lattice::LatPhysBase.AbstractLattice)
    # Build lookup table for neighbors
    # neighs[:, site] = list of neighoring site indices
    nested_bonds = [Int[] for _ in 1:numSites(lattice)]
    for b in LatPhysBase.bonds(lattice)
        push!(nested_bonds[LatPhysBase.from(b)], LatPhysBase.to(b))
    end
    max_bonds = maximum(length(x) for x in nested_bonds)

    neighs = fill(-1, max_bonds, numSites(lattice))
    for (src, targets) in enumerate(nested_bonds)
        for (idx, trg) in enumerate(targets)
            neighs[idx, src] = trg
        end
    end
    any(x -> x == -1, neighs) && @debug(
        "neighs is padded with -1 to indicated the lack of a bond. This is " *
        "due to the lattice having an irregular number of bonds per site."
    )
    LatPhysLattice(lattice, neighs, nested_bonds)
end

@inline Base.length(l::LatPhysLattice) = numSites(l.lattice)
@inline Base.ndims(l::LatPhysLattice) = ndims(l.lattice)
function Base.size(l::LatPhysLattice)
    lattice_extend = latticeVectors(l.lattice)
    uc_extend = latticeVectors(unitcell(l.lattice))
    Ls = map(lattice_extend, uc_extend) do lv, ucv
        scaling = lv ./ ucv
        if all(x -> x ≈ scaling[1], scaling)
            return scaling[1]
        else
            @warn "Irregular scaling of unitcell vectors $scaling"
            NaN
        end
    end

    return tuple(Int64.(Ls)...)
end

@inline function neighbors(l::LatPhysLattice, directed::Val{true})
    ((LatPhysBase.from(b), LatPhysBase.to(b)) for b in LatPhysBase.bonds(l.lattice))
end
@inline function neighbors(l::LatPhysLattice, directed::Val{false})
    (
        (LatPhysBase.from(b), LatPhysBase.to(b))
        for b in LatPhysBase.bonds(l.lattice)
        if LatPhysBase.from(b) < LatPhysBase.to(b)
    )
end
@inline function neighbors(l::LatPhysLattice, site_index::Integer)
    l.bonds[site_index]
end

@inline neighbors_lookup_table(l::LatPhysLattice) = copy(l.neighs)

positions(lattice::LatPhysLattice) = point.(sites(lattice.lattice))
lattice_vectors(l::LatPhysLattice) = latticeVectors(l.lattice)

function reciprocal_vectors(lattice::AbstractLattice, L)
    LatPhysBase.latticeVectors(
        LatticePhysics.getReciprocalUnitcell(
            LatPhysBase.unitcell(lattice.lattice)
        )
    )
end

################################################################################
### Saving & Loading
################################################################################

function save_lattice(file::JLDFile, lattice::LatPhysLattice, entryname::String)
    write(file, entryname * "/VERSION", 0)
    write(file, entryname * "/tag", "LatPhysLattice")
    _save_lattice(file, lattice.lattice, entryname * "/lattice")
    write(file, entryname * "/neighs", lattice.neighs)
    write(file, entryname * "/bonds", lattice.bonds)
    nothing
end
function _save_lattice(file::JLDFile, lattice::LatPhysBase.AbstractLattice, entryname::String)
    write(file, entryname * "/type", typeof(lattice))
    write(file, entryname * "/lv/N", length(lattice.lattice_vectors))
    for (i, v) in enumerate(lattice.lattice_vectors)
        write(file, entryname * "/lv/$i", v)
    end

    write(file, entryname * "/sites/type", typeof(lattice.sites[1])) # eltype of lattice
    write(file, entryname * "/sites/points", [s.point for s in lattice.sites])
    write(file, entryname * "/sites/labels", [s.label for s in lattice.sites])

    write(file, entryname * "/bonds/type", typeof(lattice.bonds[1])) # eltype
    write(file, entryname * "/bonds/froms",  [b.from for b in lattice.bonds])
    write(file, entryname * "/bonds/tos",    [b.to for b in lattice.bonds])
    write(file, entryname * "/bonds/labels", [b.label for b in lattice.bonds])
    write(file, entryname * "/bonds/wraps",  [b.wrap for b in lattice.bonds])

    save_unitcell(file, lattice.unitcell, entryname * "/unitcell")
    nothing
end
function save_unitcell(file::JLDFile, uc::LatPhysBase.AbstractUnitcell, entryname::String)
    write(file, entryname * "/type", typeof(uc))
    write(file, entryname * "/lv", uc.lattice_vectors)

    write(file, entryname * "/sites/type", typeof(uc.sites[1])) # eltype of lattice
    write(file, entryname * "/sites/points", [s.point for s in uc.sites])
    write(file, entryname * "/sites/labels", [s.label for s in uc.sites])

    write(file, entryname * "/bonds/type", typeof(uc.bonds[1])) # eltype
    write(file, entryname * "/bonds/froms",  [b.from  for b in uc.bonds])
    write(file, entryname * "/bonds/tos",    [b.to    for b in uc.bonds])
    write(file, entryname * "/bonds/labels", [b.label for b in uc.bonds])
    write(file, entryname * "/bonds/wraps",  [b.wrap  for b in uc.bonds])
    nothing
end

function _load(data, ::Val{:LatPhysLattice})
    @assert data["VERSION"] == 0
    LatPhysLattice(load_lattice(data["lattice"]), data["neighs"], data["bonds"])
end
function load_lattice(data)
    sites = load_sites(data["sites"])
    bonds = load_bonds(data["bonds"])
    lattice_vectors = [data["lv"]["$i"] for i in 1:data["lv"]["N"]]
    unitcell = load_unitcell(data["unitcell"])

    data["type"](lattice_vectors, sites, bonds, unitcell)
end
function load_unitcell(data)
    sites = load_sites(data["sites"])
    bonds = load_bonds(data["bonds"])
    lattice_vectors = data["lv"]
    data["type"](lattice_vectors, sites, bonds)
end
load_sites(data) = data["type"].(data["points"], data["labels"])
load_bonds(data) = data["type"].(data["froms"], data["tos"], data["labels"], data["wraps"])

to_tag(::Type{<: LatPhysLattice}) = Val(:LatPhysLattice)

################################################################################
### Extensions
################################################################################

function to_angle(v)
    angle = acos(v[1] / norm(v))
    (v[2] ≥ 0) ? angle : -angle
end

function get_sorted_corners(lattice::LatPhysLattice)
    bz = LatticePhysics.getBrillouinZone(
        LatticePhysics.getReciprocalUnitcell(
            LatticePhysics.unitcell(lattice.lattice)
        )
    )
    cs = sort(LatticePhysics.corners(bz), by = to_angle)
end

################################################################################
### Deprecated
################################################################################

function DistanceMask(lattice::LatPhysLattice)
    wrap = generate_combinations(latticeVectors(lattice.lattice))
    VerboseDistanceMask(lattice, wrap)
end

function directions(mask::VerboseDistanceMask, lattice::LatPhysLattice)
    pos = positions(lattice)
    dirs = [pos[src] - pos[trg] for (src, trg) in first.(mask.targets)]
    wrap = generate_combinations(latticeVectors(lattice.lattice))
    map(dirs) do _d
        d = round.(_d .+ wrap[1], digits=8)
        for v in wrap[2:end]
            new_d = round.(_d .+ v, digits=8)
            if directed_norm(new_d, 1e-6) < directed_norm(d, 1e-6)
                d .= new_d
            end
        end
        d
    end
end


"""
Module defining random play.

This module is the reference implementation for game decisions that should be
    overridden when improving strategy.
"""
module RandomPlay

import PlayingCards: Card

import Cribbage: Player
import Cribbage.Rules: validhands, validplays

export discardtocrib, pickcardtoplay

struct RandomPlayer <: Player
    name :: String
end

"""
    discardtocrib(player, cards)
Divides `cards` into `hand`, `crib`
"""
function discardtocrib(player::Player, cards::Tuple{Vararg{Card}})
    subsets = validhands(cards)
    hand = rand(subsets)
    crib = setdiff(cards, hand)
    @debug "keeping $(string(hand)), discarding $(string(crib))"
    return hand, crib
end

"""
    pickcardtoplay(player, prior, cards)
Returns either
- a card from `cards` to play
- `missing` if there are no cards can be played
"""
function pickcardtoplay(player::Player, prior::Tuple{Vararg{Card}}, cards::Tuple{Vararg{Card}})::Union{Missing,Card}
    possibleplays = validplays(prior, cards)
    if length(possibleplays) == 0
        return missing
    else
        return rand(possibleplays)
    end
end

end  # module RandomPlay


module Omniscape
using Circuitscape
using DelimitedFiles
using Distributed

include("config.jl")
include("utils.jl")
include("functions.jl")
include("io.jl")
include("run_omniscape.jl")
export run_omniscape

end


function extract_surface(_dataset::AbstractVTKStructuredData)
    if dim(_dataset) == 2
        return VTKPolyData(_dataset)
    end

    dataset = VTKStructuredData(_dataset)
    cextents = cell_extents(dataset)
    faces = Vector{Int}[]

    _cell_data = empty(_dataset.cell_data)
    for m in keys(dataset.cell_data)
        _cell_data[m] = Float64[]
    end

    for cind in surface_cell_inds(dataset)
        hexa_cc = cell_connectivity(dataset, cind)
        if cind[1] == 1
            push!(faces, [hexa_cc[1], hexa_cc[4], hexa_cc[8], hexa_cc[5]])
            for m in keys(dataset.cell_data)
                add_cell_data!(dataset, _cell_data, m, cind)
            end
        end
        if cind[1] == cextents[1]
            push!(faces, [hexa_cc[2], hexa_cc[3], hexa_cc[7], hexa_cc[6]])
            for m in keys(dataset.cell_data)
                add_cell_data!(dataset, _cell_data, m, cind)
            end
        end

        if cind[2] == 1
            push!(faces, [hexa_cc[1], hexa_cc[2], hexa_cc[6], hexa_cc[5]])
            for m in keys(dataset.cell_data)
                add_cell_data!(dataset, _cell_data, m, cind)
            end
        end
        if cind[2] == cextents[2]
            push!(faces, [hexa_cc[3], hexa_cc[4], hexa_cc[8], hexa_cc[7]])
            for m in keys(dataset.cell_data)
                add_cell_data!(dataset, _cell_data, m, cind)
            end
        end

        if cind[3] == 1
            push!(faces, [hexa_cc[1], hexa_cc[2], hexa_cc[3], hexa_cc[4]])
            for m in keys(dataset.cell_data)
                add_cell_data!(dataset, _cell_data, m, cind)
            end
        end
        if cind[3] == cextents[3]
            push!(faces, [hexa_cc[5], hexa_cc[6], hexa_cc[7], hexa_cc[8]])
            for m in keys(dataset.cell_data)
                add_cell_data!(dataset, _cell_data, m, cind)
            end
        end
    end

    _cell_types = fill(9, length(faces))

    for m in keys(dataset.cell_data)
        _var_dim = var_dim(dataset, m, "Cell")
        if _var_dim > 1
            _cell_data[m] = reshape(_cell_data[m], (_var_dim, length(faces)))
        end
    end

    _point_coords = Float64[]
    point_inds = Int[]
    _point_data = empty(_dataset.point_data)
    for m in keys(dataset.point_data)
        _point_data[m] = Float64[]
    end
    pextents = extents(dataset)

    k = 0
    for _face in faces
        for (i,p) in enumerate(_face)
            _i = findfirst(point_inds, p)
            if _i > 0
                _face[i] = _i
            else
                k += 1
                append!(_point_coords, dataset.point_coords[:,ind2sub(pextents, p)...])
                push!(point_inds, p)
                for m in keys(dataset.point_data)
                    _var_dim = var_dim(dataset, m, "Point")
                    if _var_dim == 1
                        push!(_point_data[m], dataset.point_data[m][ind2sub(pextents, p)...])
                    else
                        append!(_point_data[m], dataset.point_data[m][:, ind2sub(pextents, p)...])
                    end
                end
                _face[i] = k
            end
        end
    end

    _point_coords = reshape(_point_coords, (3, k))
    for m in keys(dataset.point_data)
        _var_dim = var_dim(dataset, m, "Point")
        if _var_dim == 1
            _point_data[m] = reshape(_point_data[m], (k,))
        else
            _point_data[m] = reshape(_point_data[m], (_var_dim, k))
        end
    end

    return VTKPolyData(_point_coords, _cell_types, faces, _point_data, _cell_data)
    #=
    _point_coords = reshape(dataset.point_coords, (3, num_of_points(dataset)))
    _point_data = empty(_dataset.point_data)
    for m in keys(dataset.point_data)
        _var_dim = var_dim(dataset, m, "Point")
        if _var_dim == 1
            _point_data[m] = reshape(dataset.point_data[m], (num_of_points(dataset),))
        else
            _point_data[m] = reshape(dataset.point_data[m], (_var_dim, num_of_points(dataset)))
        end
    end
    return VTKPolyData(_point_coords, _cell_types, faces, _point_data, _cell_data)
    =#
end

function add_cell_data!(dataset, _cell_data, m, cind)
    _var_dim = var_dim(dataset, m, "Cell")
    if _var_dim == 1
        _cd = dataset.cell_data[m][cind...]
        push!(_cell_data[m], _cd)
    else
        _cd = dataset.cell_data[m][:, cind...]
        append!(_cell_data[m], _cd)
    end
    return
end

function extract_surface(dataset::T) where {T<:AbstractVTKUnstructuredData}
    if T <: VTKPolyData
        return dataset
    end
    if dim(dataset) == 2
        return VTKPolyData(dataset)
    end

    #Sorted inds and cell type are keys
    #Cell connectivity, cell count, and cell data are the values

    cell_register = Dict{Tuple{Vector{Int}, Int}, Tuple{Vector{Int}, _Counter, typeof(dataset.cell_data)}}()
    for i in 1:length(dataset.cell_connectivity)
        if dataset.cell_types[i] ∉ POLY_CELLS
            _cells, _types = decompose_cell(dataset.cell_connectivity[i], dataset.cell_types[i], target="Faces")
        else
            _cells, _types = [dataset.cell_connectivity[i]], [dataset.cell_types[i]]
        end
        for j in 1:length(_cells)
            _key = (sort(_cells[j]), _types[j])
            if haskey(cell_register, _key)
                cell_register[_key][2].a += 1
            else
                cell_register[_key] = (_cells[j], _Counter(1), empty(dataset.cell_data))
                for m in keys(dataset.cell_data)
                    _var_dim = var_dim(dataset, m, "Cell")
                    if _var_dim == 1
                        cell_register[_key][3][m] = [dataset.cell_data[m][i]]
                    else
                        cell_register[_key][3][m] = dataset.cell_data[m][:,i]
                    end
                end
            end
        end
    end

    _cell_data = empty(dataset.cell_data)
    _cell_connectivity = Vector{Int}[]
    _cell_types = Int[]

    for m in keys(dataset.cell_data)
        _cell_data[m] = Float64[]
    end

    for (i, kv) in enumerate(cell_register)
        k, v = kv
        if v[2].a == 1
            push!(_cell_connectivity, v[1])
            push!(_cell_types, k[2])
            for m in keys(dataset.cell_data)
                append!(_cell_data[m], v[3][m])
            end
        end
    end

    for m in keys(dataset.cell_data)
        _var_dim = var_dim(dataset, m, "Cell")
        if _var_dim > 1
            _cell_data[m] = reshape(_cell_data[m], (_var_dim, length(_cell_connectivity)))
        end
    end

    _point_coords = Float64[]
    point_inds = Int[]
    _point_data = empty(dataset.point_data)
    for m in keys(dataset.point_data)
        _point_data[m] = Float64[]
    end

    k = 0
    for _cc in _cell_connectivity
        for (i,p) in enumerate(_cc)
            _i = findfirst(point_inds, p)
            if _i > 0
                _cc[i] = _i
            else
                k += 1
                append!(_point_coords, dataset.point_coords[:,p])
                push!(point_inds, p)
                for m in keys(dataset.point_data)
                    _var_dim = var_dim(dataset, m, "Point")
                    if _var_dim == 1
                        push!(_point_data[m], dataset.point_data[m][p])
                    else
                        append!(_point_data[m], dataset.point_data[m][:,p])
                    end
                end
                _cc[i] = k
            end
        end
    end

    _point_coords = reshape(_point_coords, (3, k))
    for m in keys(dataset.point_data)
        _var_dim = var_dim(dataset, m, "Point")
        if _var_dim > 1
            _point_data[m] = reshape(_point_data[m], (_var_dim, k))
        end
    end

    return VTKPolyData(_point_coords, _cell_types, _cell_connectivity, _point_data, _cell_data)
end


"""
## Indexing
`pa` is an instance of `SeisForwExpt`.
* `pa[:data,i]` : get data for ith supersource (defaults to first), after `update!`
* `pa[:ageom]` : get `ageom` originally input while creating `pa`
* `pa[:srcwav]`: `srcwav` used to create `pa`
* `pa[:snaps,i]` : snapshots corresponding to `tsnaps` and ith source, after `update!`
"""
function Base.getindex(pa::PFdtd, s::Symbol, iss::Int=1)
	@assert iss≤length(pa.c.ageom[1])
	@assert s in [:snaps,:data, :ageom, :srcwav]
	if(s==:snaps)
		nzd,nxd=length.(pa.c.model.mgrid)
		snaps_all=zeros(nzd,nxd,length(pa.c.itsnaps))
		ip=findall(in.(iss,pa.sschunks))[1]
		issp=findall(x->x==iss,pa.sschunks[ip])[]
		@sync begin
			p=procs(pa.p)[ip]
			@sync remotecall_wait(p) do
				pap=localpart(pa.p)
				for i in eachindex(snaps_all)
					snaps_all[i]=pap.ss[issp].snaps[i]
				end
			end
		end
		return snaps_all
	elseif(s==:data)
		(iss==0) ?  (return pa.c.data[1]) :  (return pa.c.data[1][iss])
	elseif(s==:ageom)
		return pa.c.ageom
	elseif(s==:srcwav)
		return pa.c.srcwav
	end
end

function Base.show(io::Base.IO, pa::PFdtd)
	println(typeof(pa), "")
	println("pa[:snaps]")
	println("pa[:data] : modeled data after running `update!`")
	println("pa[:ageom]")
	println("pa[:srcwav]")
end

Base.show(io::Base.IO, pa::PFdtdp)=nothing
Base.show(io::Base.IO, pa::PFdtdc)=nothing
Base.show(io::Base.IO, pa::PFdtdss)=nothing


####
#### Useful kernels
####

function velocity_div!(grid::RegularCartesianGrid, u, v, w, div)
    @loop for k in (1:grid.Nz; (blockIdx().z - 1) * blockDim().z + threadIdx().z)
        @loop for j in (1:grid.Ny; (blockIdx().y - 1) * blockDim().y + threadIdx().y)
            @loop for i in (1:grid.Nx; (blockIdx().x - 1) * blockDim().x + threadIdx().x)
                @inbounds div[i, j, k] = div_f2c(grid, u, v, w, i, j, k)
            end
        end
    end
end

####
#### Horizontal averages
####

"""
    HorizontalAverage{F, R, P, I, Ω} <: AbstractDiagnostic

A diagnostic for computing horizontal average of a field.
"""
mutable struct HorizontalAverage{F, R, P, I, Ω, G} <: AbstractDiagnostic
          field :: F
         result :: P
      frequency :: Ω
       interval :: I
       previous :: Float64
    return_type :: R
           grid :: G
end

"""
    HorizontalAverage(model, field; frequency=nothing, interval=nothing, return_type=Array)

Construct a `HorizontalAverage` of `field`.

After the horizontal average is computed it will be stored in the `result` property.

The `HorizontalAverage` can be used as a callable object that computes and returns the
horizontal average.

A `frequency` or `interval` (or both) can be passed to indicate how often to run this
diagnostic if it is part of `model.diagnostics`. `frequency` is a number of iterations
while `interval` is a time interval in units of `model.clock.time`.

A `return_type` can be used to specify the type returned when the `HorizontalAverage` is
used as a callable object. The default `return_type=Array` is useful when running a GPU
model and you want to save the output to disk by passing it to an output writer.
"""
function HorizontalAverage(field; frequency=nothing, interval=nothing, return_type=Array)
    arch = architecture(field)
    result = zeros(arch, field.grid, 1, 1, field.grid.Tz)
    return HorizontalAverage(field, result, frequency, interval, 0.0, return_type, field.grid)
end

# Normalize a horizontal sum to get the horizontal average.
normalize_horizontal_sum!(havg, grid) = havg.result /= (grid.Nx * grid.Ny)

"""
    run_diagnostic(model, havg::HorizontalAverage{NTuple{1}})

Compute the horizontal average of `havg.field` and store the result in `havg.result`.
"""
function run_diagnostic(model, havg::HorizontalAverage)
    zero_halo_regions!(parent(havg.field), model.grid)
    sum!(havg.result, parent(havg.field))
    normalize_horizontal_sum!(havg, model.grid)
    return nothing
end

function (havg::HorizontalAverage{F, Nothing})(model) where F
    run_diagnostic(model, havg)
    return havg.result
end

function (havg::HorizontalAverage)(model)
    run_diagnostic(model, havg)
    return havg.return_type(havg.result)
end

####
#### NaN checker
####

"""
    NaNChecker{F} <: AbstractDiagnostic

A diagnostic that checks for `NaN` values and aborts the simulation if any are found.
"""
struct NaNChecker{F} <: AbstractDiagnostic
    frequency :: Int
       fields :: F
end

"""
    NaNChecker(model; frequency, fields)

Construct a `NaNChecker` for `model`. `fields` should be a `Dict{Symbol,Field}`. A
`frequency` should be passed to indicate how often to check for NaNs (in number of
iterations).
"""
function NaNChecker(model; frequency, fields)
    return NaNChecker(frequency, fields)
end

function run_diagnostic(model::Model, nc::NaNChecker)
    for (name, field) in nc.fields
        if any(isnan, field.data.parent)
            t, i = model.clock.time, model.clock.iteration
            error("time = $t, iteration = $i: NaN found in $name. Aborting simulation.")
        end
    end
end

####
#### Timeseries diagnostics
####

"""
    Timeseries{D, Ω, I, T, TT} <: AbstractDiagnostic

A diagnostic for collecting and storing timeseries.
"""
struct Timeseries{D, Ω, I, T, TT} <: AbstractDiagnostic
    diagnostic :: D
     frequency :: Ω
      interval :: I
          data :: T
          time :: Vector{TT}
end

"""
    Timeseries(diagnostic, model; frequency=nothing, interval=nothing)

A `Timeseries` `Diagnostic` that records a time series of `diagnostic(model)`.

Example
=======
```julia
julia> model = Model(grid=RegularCartesianGrid(size=(16, 16, 16), length=(1, 1, 1)));

julia> max_u = Timeseries(FieldMaximum(abs, model.velocities.u), model; frequency=1)

julia> model.diagnostics[:max_u] = max_u; data(model.velocities.u) .= π; time_step!(model, Nt=3, Δt=1e-16)

julia> max_u.data
3-element Array{Float64,1}:
 3.141592653589793
 3.1415926025389127
 3.1415925323439517
```
"""
function Timeseries(diagnostic, model; frequency=nothing, interval=nothing)
    TD = typeof(diagnostic(model))
    TT = typeof(model.clock.time)
    return Timeseries(diagnostic, frequency, interval, TD[], TT[])
end

@inline (timeseries::Timeseries)(model) = timeseries.diagnostic(model)

function run_diagnostic(model, diag::Timeseries)
    push!(diag.data, diag(model))
    push!(diag.time, model.clock.time)
    return nothing
end

"""
    Timeseries(diagnostics::NamedTuple, model; frequency=nothing, interval=nothing)

A `Timeseries` `Diagnostic` that records a `NamedTuple` of time series of
`diag(model)` for each `diag` in `diagnostics`.

Example
=======
```julia
julia> model = Model(grid=RegularCartesianGrid(size=(16, 16, 16), length=(1, 1, 1))); Δt = 1.0;

julia> cfl = Timeseries((adv=AdvectiveCFL(Δt), diff=DiffusiveCFL(Δt)), model; frequency=1);

julia> model.diagnostics[:cfl] = cfl; time_step!(model, Nt=3, Δt=Δt)

julia> cfl.data
(adv = [0.0, 0.0, 0.0, 0.0], diff = [0.0002688, 0.0002688, 0.0002688, 0.0002688])

julia> cfl.adv
4-element Array{Float64,1}:
 0.0
 0.0
 0.0
 0.0
```
"""
function Timeseries(diagnostics::NamedTuple, model; frequency=nothing, interval=nothing)
    TT = typeof(model.clock.time)
    TDs = Tuple(typeof(diag(model)) for diag in diagnostics)
    data = NamedTuple{propertynames(diagnostics)}(Tuple(T[] for T in TDs))
    return Timeseries(diagnostics, frequency, interval, data, TT[])
end

function (timeseries::Timeseries{<:NamedTuple})(model)
    names = propertynames(timeseries.diagnostic)
    return NamedTuple{names}(Tuple(diag(model) for diag in timeseries.diagnostics))
end

function run_diagnostic(model, diag::Timeseries{<:NamedTuple})
    ntuple(Val(length(diag.diagnostic))) do i
        Base.@_inline_meta
        push!(diag.data[i], diag.diagnostic[i](model))
    end
    push!(diag.time, model.clock.time)
    return nothing
end

Base.getproperty(ts::Timeseries{<:NamedTuple}, name::Symbol) = get_timeseries_field(ts, name)

"Returns a field of timeseries, or a field of `timeseries.data` when possible."
function get_timeseries_field(timeseries, name)
    if name ∈ propertynames(timeseries)
        return getfield(timeseries, name)
    else
        return getproperty(timeseries.data, name)
    end
end

"""
    FieldMaximum(mapping, field)

An object for calculating the maximum of a `mapping` function applied
element-wise to `field`.

Examples
=======
```julia
julia> model = Model(grid=RegularCartesianGrid(size=(16, 16, 16), length=(1, 1, 1)));

julia> max_abs_u = FieldMaximum(abs, model.velocities.u);

julia> max_w² = FieldMaximum(x->x^2, model.velocities.w);
```
"""
struct FieldMaximum{F, M}
    mapping :: M
      field :: F
end

(m::FieldMaximum)(args...) = maximum(m.mapping, m.field.data.parent)

(m::FieldMaximum{<:NamedTuple})(args...) =
    NamedTuple{propertynames(m.field)}(maximum(m.mapping, f.data.parent) for f in m.field)

"""
    CFL{D, S}

An object for computing the Courant-Freidrichs-Lewy (CFL) number.
"""
struct CFL{D, S}
           Δt :: D
    timescale :: S
end

"""
    CFL(Δt [, timescale=Oceananigans.cell_advection_timescale])

Returns an object for computing the Courant-Freidrichs-Lewy (CFL) number
associated with time step or `TimeStepWizard` `Δt` and `timescale`.

See also `AdvectiveCFL` and `DiffusiveCFL`
"""
CFL(Δt) = CFL(Δt, cell_advection_timescale)

(c::CFL{<:Number})(model) = c.Δt / c.timescale(model)
(c::CFL{<:TimeStepWizard})(model) = c.Δt.Δt / c.timescale(model)

"""
    AdvectiveCFL(Δt)

Returns an object for computing the Courant-Freidrichs-Lewy (CFL) number
associated with time step or `TimeStepWizard` `Δt` and the time scale
for advection across a cell.

Example
=======
```julia
julia> model = Model(grid=RegularCartesianGrid(size=(16, 16, 16), length=(8, 8, 8)));

julia> cfl = AdvectiveCFL(1.0);

julia> data(model.velocities.u) .= π;

julia> cfl(model)
6.283185307179586
```
"""
AdvectiveCFL(Δt) = CFL(Δt, cell_advection_timescale)

"""
    DiffusiveCFL(Δt)

Returns an object for computing the Courant-Freidrichs-Lewy (CFL) number
associated with time step or `TimeStepWizard` `Δt` and the time scale
for diffusion across a cell associated with `model.closure`.

Example
=======
```julia
julia> model = Model(grid=RegularCartesianGrid(size=(16, 16, 16), length=(1, 1, 1)));

julia> cfl = DiffusiveCFL(0.1);

julia> cfl(model)
2.688e-5
```
"""
DiffusiveCFL(Δt) = CFL(Δt, cell_diffusion_timescale)


# Benchmarks.jl
#
# Run some benchmarks for FluxArchitectures

# Settings
const RUN_GPU = true
const RUN_CPU = true

# Make sure all the required packages are available
cd(@__DIR__)
using Pkg; Pkg.activate(".")
Pkg.instantiate()

@info "Loading packages"
include("../src/FluxArchitectures.jl")
using .FluxArchitectures
using BenchmarkTools
using DataFrames
using Random

# Initialize results dataframe
results = DataFrame(Name=String[], CPU=Union{Missing,Float32}[], GPU=Union{Missing,Float32}[])

# Set up data and loss
poollength = 10
horizon = 6
datalength = 500
input_cpu, target_cpu = get_data(:solar, poollength, datalength, horizon)
input_gpu, target_gpu = gpu(input_cpu), gpu(target_cpu)
inputsize = size(input_cpu, 1)
function loss(x, y)
    Flux.reset!(model)
    return Flux.mse(model(x), permutedims(y))
end

## DARNN ----------------------------------
@info "DARNN"
push!(results,["DARNN", missing, missing])
encodersize = 10
decodersize = 10
if RUN_GPU
    model = DARNN(inputsize, encodersize, decodersize, poollength, 1) |> gpu
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_gpu, target_gpu), 5),
            ADAM(0.007))
    results[end,:GPU] = time
end
if RUN_CPU
    model = DARNN(inputsize, encodersize, decodersize, poollength, 1)
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_cpu, target_cpu), 5),
            ADAM(0.007))
    results[end,:CPU] = time
end

## DSANet ----------------------------------
@info "DSANet"
push!(results,["DSANet", missing, missing])
local_length = 3
n_kernels = 3
d_model = 4
hiddensize = 1
n_layers = 3
n_head = 2
if RUN_GPU
    Random.seed!(123)
    model = DSANet(inputsize, poollength, local_length, n_kernels, d_model,
               hiddensize, n_layers, n_head) |> gpu
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_gpu, target_gpu), 5),
            ADAM(0.007))
    results[end,:GPU] = time
end
if RUN_CPU
    Random.seed!(123)
    model = DSANet(inputsize, poollength, local_length, n_kernels, d_model,
               hiddensize, n_layers, n_head)
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_cpu, target_cpu), 5),
            ADAM(0.007))
    results[end,:CPU] = time
end

## LSTNet ----------------------------------
@info "LSTNet"
push!(results,["LSTNet", missing, missing])
convlayersize = 2
recurlayersize = 3
skiplength = 120
if RUN_GPU
    model = LSTnet(inputsize, convlayersize, recurlayersize, poollength, skiplength,
        init=Flux.zeros, initW=Flux.zeros) |> gpu
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_gpu, target_gpu), 5),
            ADAM(0.007))
    results[end,:GPU] = time
end
if RUN_CPU
    model = LSTnet(inputsize, convlayersize, recurlayersize, poollength, skiplength,
        init=Flux.zeros, initW=Flux.zeros)
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_cpu, target_cpu), 5),
            ADAM(0.007))
    results[end,:CPU] = time
end

## TPA-LSTM ----------------------------------
@info "TPA-LSTM"
push!(results,["TPA-LSTM", missing, missing])
hiddensize = 10
layers = 2
filternum = 32
filtersize = 1
if RUN_GPU
    model = TPALSTM(inputsize, hiddensize, poollength, layers, filternum, filtersize) |> gpu
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_gpu, target_gpu), 5),
            ADAM(0.007))
    results[end,:GPU] = time
end
if RUN_CPU
    model = TPALSTM(inputsize, hiddensize, poollength, layers, filternum, filtersize)
    time = @belapsed Flux.train!(loss, Flux.params(model),Iterators.repeated((input_cpu, target_cpu), 5),
            ADAM(0.007))
    results[end,:CPU] = time
end


FactCheck.setstyle(:compact)
FactCheck.onlystats(true)

facts("Semantics are still enforced" ) do

    nan = fill(NaN, 3)
    inf = fill(Inf, 3)
    neg = fill(-Inf, 3)

    context("NaN math adds up") do
        @fact isnan(sum(nan)) --> true
    end

    context("Inf math adds up") do
        @fact sum(inf) --> Inf
    end

    context("NaN math adds up") do
        @fact sum(neg) --> -Inf
    end
end

facts("Types correctly constructed" ) do

    here     = HereSomewhere(NaN)
    consumed = Consumed(NaN)
    nonsense = Nonsense(NaN)
    shifted  = Shifted(NaN)

    context("HereSomewhere") do
        @fact isnan(here.val) --> true
        @fact_throws HereSomewhere(1.0)
    end

    context("Consumed") do
        @fact isnan(consumed.val) --> true
        @fact_throws Consumed(1.0)
    end

    context("Nonsense") do
        @fact isnan(nonsense.val) --> true
        @fact_throws Nonsense(1.0)
    end

    context("Shifted") do
        @fact isnan(shifted.val) --> true
        @fact_throws Shifted(1.0)
    end
end


import DataFrames
# using InvertedIndices
import Luxor

import Base
import CSV
import Printf

import Combinatorics
import Statistics
import StatsBase
import Random
import Distributions
import KernelDensity
import SparseArrays

import LinearAlgebra, LinearAlgebra.I

include(joinpath(WATERFALL_DIR,"src","Figure","Color","Coloring.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Color","Blending.jl"))

abstract type Shape end
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Box.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Line.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Point.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Poly.jl"))

include(joinpath(WATERFALL_DIR,"src","Figure","Text","Label.jl"))

abstract type Sampling end
abstract type Geometry <: Sampling end
include(joinpath(WATERFALL_DIR,"src","Cascade","Data.jl"))
include(joinpath(WATERFALL_DIR,"src","Cascade","Cascade.jl"))
include(joinpath(WATERFALL_DIR,"src","Cascade","Geometry","Horizontal.jl"))
include(joinpath(WATERFALL_DIR,"src","Cascade","Geometry","Parallel.jl"))
include(joinpath(WATERFALL_DIR,"src","Cascade","Geometry","Vertical.jl"))
include(joinpath(WATERFALL_DIR,"src","Cascade","Geometry","Violin.jl"))

include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Arrow.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Box.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Line.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Point.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Shape","Poly.jl"))

abstract type Axis end
include(joinpath(WATERFALL_DIR,"src","Figure","XAxis.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","YAxis.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Annotation.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Handle.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Legend.jl"))
include(joinpath(WATERFALL_DIR,"src","Figure","Plot.jl"))

include(joinpath(WATERFALL_DIR,"src","definitions.jl"))
include(joinpath(WATERFALL_DIR,"src","options.jl"))

include(joinpath(WATERFALL_DIR,"src","utils","attributes.jl"))
include(joinpath(WATERFALL_DIR,"src","utils","io.jl"))
include(joinpath(WATERFALL_DIR,"src","utils","calc.jl"))
include(joinpath(WATERFALL_DIR,"src","utils","draw.jl"))
include(joinpath(WATERFALL_DIR,"src","utils","random.jl"))
include(joinpath(WATERFALL_DIR,"src","utils","scale.jl"))
include(joinpath(WATERFALL_DIR,"src","utils","utils.jl"))

include(joinpath(WATERFALL_DIR,"src","utils","define.jl"))


#InspectDR: Secondary annotation (Cairo layer)
#-------------------------------------------------------------------------------

#==Constants
===============================================================================#
const ALIGN_MAP = Dict{Symbol, CAlignment}(
	:tl => ALIGN_TOP | ALIGN_LEFT,
	:tc => ALIGN_TOP | ALIGN_HCENTER,
	:tr => ALIGN_TOP | ALIGN_RIGHT,

	:cl => ALIGN_VCENTER | ALIGN_LEFT,
	:cc => ALIGN_VCENTER | ALIGN_HCENTER,
	:cr => ALIGN_VCENTER | ALIGN_RIGHT,

	:bl => ALIGN_BOTTOM | ALIGN_LEFT,
	:bc => ALIGN_BOTTOM | ALIGN_HCENTER,
	:br => ALIGN_BOTTOM | ALIGN_RIGHT,
)

#==Rendering text annotation
===============================================================================#
function render(canvas::PCanvas2D, a::TextAnnotation, axes::Axes)
	const ctx = canvas.ctx
	const graphbb = canvas.graphbb
	align = get(ALIGN_MAP, a.align, ALIGN_BOTTOM | ALIGN_LEFT)
	angle = deg2rad(a.angle)

	Cairo.set_source(ctx, a.color)
	pt = _rescale(a.pt, axes)
	pt = ptmap(canvas.xf, pt)
	x = pt.x; y = pt.y
	if isnan(x); x = graphbb.xmin; end
	if isnan(y); y = graphbb.ymax; end
	x += a.xoffset * width(graphbb)
	y -= a.yoffset * height(graphbb)
	render(ctx, a.text, Point2D(x,y), a.font, angle=angle, align=align)
	return
end

render(canvas::PCanvas2D, alist::Vector{TextAnnotation}, axes::Axes) =
	map((a)->render(canvas, a, axes::Axes), alist)

#==Rendering markers
===============================================================================#

function render(canvas::PCanvas2D, mkr::HVMarker, axes::Axes)
	const ctx = canvas.ctx
	const graphbb = canvas.graphbb
	if :none == mkr.line.style
		return
	end

	Cairo.set_source(ctx, mkr.line.color)
	setlinestyle(ctx, mkr.line.style, Float64(mkr.line.width))
	pt = _rescale(Point2D(mkr.pos, mkr.pos), axes)
	pt = ptmap(canvas.xf, pt)

	if mkr.vmarker
		drawline(ctx, Point2D(pt.x, graphbb.ymin), Point2D(pt.x, graphbb.ymax))
	else #hmarker
		drawline(ctx, Point2D(graphbb.xmin, pt.y), Point2D(graphbb.xmax, pt.y))
	end

	return
end

render(canvas::PCanvas2D, mkrlist::Vector{HVMarker}, axes::Axes) =
	map((mkr)->render(canvas, mkr, axes::Axes), mkrlist)

#Last line