using .Abstract: returntype
using IRTools: Variable, returnvalue, blocks, isexpr
using IRTools.Inner: iscall

struct Trivial end

function infer(f, ::Trivial, tr = trace(typeof(f)))
  r = returntype(tr)
  r isa Abstract.Const && return Singleton(r.value)
  any(((v, st),) -> iscall(st.expr, observe), tr) && return
  r = returnvalue(blocks(tr)[end])
  r isa Variable || return
  ex = tr[r].expr
  iscall(ex, rand) && ex.args[2] isa Distribution && return ex.args[2]
  return
end


## This is the OLD version of Table. We are using the version in lists_new.jl

# We only do Table(expr,[i,imax])
# Our Table is rather slow. Slower than Maxima makelist.
# Table(a(i),[i,10000]) is 2 to 3 x slower than makelist( i, i, 1, 10000)
# If we cheat and only do a single evaluation,
# and setfixed() then the speed is the same.
# But, the 3rd party 'table' for Maxima is 4 times faster than makelist
# in this example.
# Mma 3.0  Timing[mm = Table[a[i],{i,10^5}];]  {0.05 Second, Null}
# 3rd pary maxima: table(a(i),[i,10^5]);  0.14 s
# At commit 3478f83ee403238704ad6af81b42e1c7f4d07cc8
# Table(a(i),[i,10000]) takes bout 0.45--0.49 s, makelist reports 0.503.

set_attribute(:TableOld, :HoldAll)

function apprules(mx::Mxpr{:TableOld})
    expr = mx[1]
    iter = mx[2]
    isym = get_localized_symbol(iter[1])
    imax = meval(iter[2])
#    issym = createssym(isym,Int)  ## Trying out Typed variable
    ex = replsym(deepcopy(expr),iter[1],isym) # takes no time, for simple expression
    args = do_table(imax,isym,ex) # creating a symbol is pretty slow
    removesym(isym)
    mx1 = mxpr(:List,args) # takes no time
    # This mergesyms no longer makes this much faster, because overall Table has slowed by more than 5x
    # syms are being merged alot somewhere else
    mergesyms(mx1,:nothing) # not correct, but stops the merging
    setcanon(mx1)
    setfixed(mx1)
    return mx1
end

# These tests concern not Table per se, but the efficiency
# of evaluation, in particular assigning values to symbols.
# commit 3478f83ee403238704ad6af81b42e1c7f4d07cc8
# Testing Table(a(i),[i,10^5])
# changes                   Time
# normal                    0.49
# disable setsymval         0.35
#
# commit 0d2332b17eb7e5c518a86b1b3044b691fddb5b87
# setsymval is now much faster because we changed data structure in SSJSym
# Testing Table(a(i),[i,10^5])
# normal                    0.36
# *sigh* Cannot reproduce the performance in the previous line. Even if
# I return to that commit. Time is closer to 0.5 than 0.35
# setsymval still seems to be an expensive operation
#
# 4f7d9f6dff207ae2c95736ab058627f503fbad26
# With this commit, Time is 1.83 s. For same code as above.  Table is
# becoming slower. We try using a stripped-down meval1 from
# alteval.jl, time is .43 with no gc.

function do_table(imax::T,isym,ex) where T<:Integer
    args = newargs(imax)
    sisym = getssym(isym)
#    setsymval(sisym,1)
    @inbounds for i in 1:imax
        setsymval(sisym,i)
#        args[i] = meval(ex)
        args[i] = doeval(ex)
#        args[i] = meval1(ex)
        setfixed(args[i])  # no difference ?
    end
    return args
end

# The speed penalty for val::Any instead of val::Int, is factor of 10^4.
# Setting isym.val is by far the slowest step in Table(i,[i,10^6])
# Much slower than Mma and SBCL
# If we us val::Array{Any,1}, the speed penalty is better by a couple
# orders of magnitude

# type NSYM
# #    val::Int
#     val::Array{Any,1}
# end

# function testset()
#     ss = NSYM(Any[0])
#     sum0 = 0
#     for i in 1:10^6
#         ss.val[1] = i
#         sum0 += ss.val[1]
#     end
#     sum0
# end


# ---------------------------------------------------------------------------- #
#
#   mesherBAMG.jl
#
#   BAMG-specific read and write
#
#   λυτέος
#   Fall 2017
#
#   Max Opgenoord
#
# ---------------------------------------------------------------------------- #

"""
    MesherBAMG

MesherBAMG type:
Type for BAMG meshes.
"""
struct MesherBAMG <: Mesher

end

"""
    writeMsh( mesh::Mesh2D, mesher::MesherBAMG, flname::String )

Writes 2D mesh in `mesh` to `flname` in format used by BAMG.
"""
function writeMsh( mesh::Mesh2D, mesher::MesherBAMG, flname::String )

    np = size(mesh.p,1)

    fid = open( flname, "w" )

    @printf( fid, "MeshVersionFormatted 0\n\n" )

    @printf(fid, "Dimension 2\n\n")

    # Vertices
    @printf(fid, "Vertices\n%i\n", np)
    for jj = 1:size(mesh.p,1)
        @printf(fid, "%16.15e\t%16.15e\t%i\n", mesh.p[jj,1], mesh.p[jj,2], 1)
    end
    @printf(fid,"\n")

    # Edges
    nbound = size(mesh.fb,1)
    @printf(fid, "Edges\n%i\n",nbound)
    for ii = 1:nbound
        indf = mesh.fb[ii,1]
        @printf(fid, "%i\t%i\t%i\n", mesh.f[indf,1], mesh.f[indf,2], mesh.fb[ii,2])
    end
    @printf(fid,"\n")

    # Triangles
    @printf(fid, "Triangles\n%i\n",size(mesh.t,1))
    for jj = 1:size(mesh.t,1)
        @printf(fid, "%i\t%i\t%i\t%i\n", mesh.t[jj,1], mesh.t[jj,2], mesh.t[jj,3], 1)
    end
    @printf(fid,"\n")

    close(fid)

    # TODO: Corners

end

"""
    readBAMG( flname::String )

Reads (2D) BAMG mesh from `flname`.
"""
function readBAMG( flname::String )

    dim = 2

    # Open file
    fid = open( flname, "r" )

    # Check dimension
    for il in 1:3
        x = readline(fid)
    end
    x = readline(fid)
    if parse(Int64, x) != dim
        error(" BAMG read-in: dimension should be 2")
    end

    # Read vertices
    while isempty(x) || x[1] != 'V'
        x = readline(fid)
    end
    x  = readline(fid)
    np = parse( Int64, x )
    p  = fill( 0.0, np, 2 )
    for il in 1:np
        x  = readline(fid)
        xx = split(x)
        p[il,1] = parse( Float64, xx[1] )
        p[il,2] = parse( Float64, xx[2] )
    end

    # Read boundary
    while isempty(x) || x[1] != 'E'
        x = readline(fid)
    end
    x   = readline(fid)
    nb  = parse( Int64, x )
    bel = fill( 0, nb, 3 )
    for il in 1:nb
        x  = readline(fid)
        xx = split(x)
        bel[il,1] = parse( Int64, xx[1] )
        bel[il,2] = parse( Int64, xx[2] )
        bel[il,3] = parse( Int64, xx[3] )
    end

    # Read triangles
    while isempty(x) || x[1] != 'T'
        x = readline(fid)
    end
    x  = readline(fid)
    nt = parse( Int64, x )
    t  = fill( 0, nt, 3 )
    for il in 1:nt
        x  = readline(fid)
        xx = split(x)
        t[il,1] = parse( Int64, xx[1] )
        t[il,2] = parse( Int64, xx[2] )
        t[il,3] = parse( Int64, xx[3] )
    end

    close(fid)

    # Add boundary tags
    nbounds = maximum( bel[:,3] )
    tags = Vector{String}( nbounds )
    for bb in 1:nbounds
        tags[bb] = string("bound", bb)
    end

    return( p, t, bel, tags )

end

"""
    runMesher( mesher::MesherBAMG, flMesh::String, flMetric::String, flMeshNew::String )

Runs BAMG mesher.
"""
function runMesher( mesher::MesherBAMG, flMesh::String, flMetric::String, flMeshNew::String )

    run( `bamg -b $flMesh -M $flMetric -o $flMeshNew` )

end


# MIT License

# Copyright (c) 2021 Fraunhofer-Chalmers Centre

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

########################################
# Runs the LR algorithm and Gurobi on all categories of instances in the "instance" folder and writes the result data to the "result" folder
# Results from our run is already in the results.7z folder
########################################
using DelimitedFiles
cd(@__DIR__ )
folders = [name  for  name in readdir("instances") if isnothing(findfirst('.',name))]
instance_names = Dict()
n_tot = 0
T_limit = 300
for folder in folders
    global  n_tot
    instance_name = []
    for file in readdir("instances/"*folder)
        if file[end-3:end] != ".csv"
            continue
        end
        inst_path = folder*"/"*file
        push!(instance_name,file[1:end-4])
        n_tot += 1
    end
    instance_names[folder]=instance_name
end
cp("../binary/RCmax.exe","RCmax.exe")
##
n_solved = 0
start = time()
for folder in folders
    global  n_solved
    out_path = "results/"*folder*"/res.txt"
    if isfile(out_path)
        rm(out_path)
    end

    for name in instance_names[folder]
        inst_path = "instances/"*folder*"/"*name*".csv"
        A = readdlm(inst_path,',')
        (m,n) = size(A)
        println(inst_path)
        if n <= 200
            run(`RCmax.exe $inst_path $T_limit -o $out_path`)
        else
            run(`RCmax.exe $inst_path $T_limit -o $out_path -nL -nS`)
        end
        n_solved += 1
        println(time()-start,"s ", n_solved,"/",n_tot)
    end
    name_v = vcat("name", instance_names[folder])
    A = readdlm(out_path, ',')
    writedlm(out_path, [name_v A],',')
end

## with Gurobi
using JuMP, Gurobi
function solveRCmax(inst_path)
    p = readdlm(inst_path,',')
    m,n = size(p)
    model = Model(Gurobi.Optimizer)
    @variable(model,z)
    @objective(model,Min,z)
    @variable(model, x[1:m,1:n], Bin)
    @constraint(model, [i in 1:m], p[i,:]'*x[i,:] <= z)
    @constraint(model, [j in 1:n], sum(x[:,j]) == 1)
    set_time_limit_sec(model,T_limit)
    set_silent(model)
    set_optimizer_attribute(model,"Threads", 1)
    set_optimizer_attribute(model, "Cuts", 2)
    set_optimizer_attribute(model,"MIPGap", 1e-8)
    set_optimizer_attribute(model,"MIPGapAbs", 0.999) # integer costs, ensure that it is not a round off error
    optimize!(model)

    zv = objective_value(model)
    zb = objective_bound(model)
    st = solve_time(model)
    si = simplex_iterations(model)
    nc = node_count(model)
    solved = st < T_limit

    unset_binary.(x)
    set_lower_bound.(x,0.0)
    optimize!(model)
    z_lp = objective_value(model)

    gap = (zv-zb)/zv
    lb_root_gap = (zv-z_lp)/zv

    return [m n nc si st solved gap lb_root_gap zv]
end

start = time()
n_solved = 0
for folder in folders
    global  n_solved
    out_path = "results/"*folder*"/res_gur_cut.txt"
    A = ["name" "m" "n" "nodes" "iter" "time" "solved" "gap" "lbrootgap" "z"]
    for name in instance_names[folder]
        inst_path = "instances/"*folder*"/"*name*".csv"
        println(inst_path)
        res = solveRCmax(inst_path)
        n_solved += 1
        println(time()-start,"s ", n_solved,"/",n_tot)
        A = vcat(A, hcat(name, res))
        writedlm(out_path, A, ",")
    end
end


export Representation, HRepresentation, VRepresentation, fulldim
export RepIterator
export Rep

"""
    Representation{T<:Real}

Supertype for H-(or V-)representations with coefficient type `T`.
"""
abstract type Representation{T <: Real} end
"""
    HRepresentation{T<:Real}

Supertype for H-representations with coefficient type `T`.
"""
abstract type HRepresentation{T} <: Representation{T} end
"""
    VRepresentation{T<:Real}

Supertype for V-representations coefficient type `T`.
"""
abstract type VRepresentation{T} <: Representation{T} end

export Rep, HRep, VRep

const  Rep{T} = Union{ Representation{T}, Polyhedron{T}}
const HRep{T} = Union{HRepresentation{T}, Polyhedron{T}}
const VRep{T} = Union{VRepresentation{T}, Polyhedron{T}}

Base.broadcastable(x::Union{Rep, Library}) = Ref(x)

"""
    coefficient_type(rep::Rep)

Returns the type of the coefficients used in the representation of `rep`.
"""
coefficient_type(rep::Union{Rep{T}, Type{<:Rep{T}}}) where {T} = T

FullDim(rep::Type{<:VRep}) = FullDim(vvectortype(rep))
FullDim(rep::Type{<:HRep}) = FullDim(hvectortype(rep))
FullDim(rep::Type{<:Polyhedron}) = FullDim(hvectortype(rep))

# Check that it is either empty or it has a point
vconsistencyerror() = error("Non-empty V-representation must contain at least one point. If it is a polyhedral cone, the origin should be added.")
function checkvconsistency(vrep::VRep)
    if !haspoints(vrep) && hasallrays(vrep)
        vconsistencyerror()
    end
end
checkvconsistency(p::Polyhedron) = vrepiscomputed(p) && checkvconsistency(vrep(p))

# This method solves the ambiguity with the following methods and the general method
# Base.convert{T}(::Type{T}, p::T) = p
Base.convert(::Type{T}, p::T) where {T<:HRepresentation} = p
Base.convert(::Type{T}, p::T) where {T<:VRepresentation} = p

Base.convert(RepT::Type{<:HRep}, p::HRepresentation)            = hconvert(RepT, p)
Base.convert(RepT::Type{<:HRepresentation}, p::HRep)            = hconvert(RepT, p)
# avoid ambiguity
Base.convert(RepT::Type{<:HRepresentation}, p::HRepresentation) = hconvert(RepT, p)

Base.copy(rep::HRepresentation) = typeof(rep)(FullDim_hreps(rep)...)
Base.copy(rep::VRepresentation) = typeof(rep)(FullDim_vreps(rep)...)

function Polyhedron{S}(p::Polyhedron{T}) where {S, T}
    RepT = similar_type(typeof(p), S)
    if !hrepiscomputed(p) && vrepiscomputed(p)
        vconvert(RepT, p)
    else
        hconvert(RepT, p)
    end
end

Base.convert(RepT::Type{<:VRep}, p::VRepresentation)            = vconvert(RepT, p)
Base.convert(RepT::Type{<:VRepresentation}, p::VRep)            = vconvert(RepT, p)
# avoid ambiguity
Base.convert(RepT::Type{<:VRepresentation}, p::VRepresentation) = vconvert(RepT, p)

# Used by VRepPolyhedraModel
Base.convert(::Type{VRep}, p::VRepresentation) = p

change_coefficient_type(p::Rep{T}, ::Type{T}) where {T} = p
change_coefficient_type(p::Rep, T::Type) = convert(similar_type(typeof(p), T), p)

VRepresentation{T}(v::VRepresentation) where {T} = convert(similar_type(typeof(v), FullDim(v), T), v)
HRepresentation{T}(h::HRepresentation) where {T} = convert(similar_type(typeof(h), FullDim(h), T), h)

VRep{T}(v::VRepresentation) where {T} = VRepresentation{T}(v)
VRep{T}(p::Polyhedron) where {T} = Polyhedron{T}(p)
HRep{T}(h::HRepresentation) where {T} = HRepresentation{T}(h)
HRep{T}(p::Polyhedron) where {T} = Polyhedron{T}(p)

# FIXME it does not get called. The calls always go throug vconvert and hconvert. Use changecoefficient_type instead
#function Base.convert{T, RepT<:Representation}(::Type{Representation{T}}, rep::RepT)
#  if fulldim(RepT) != N
#    error("Cannot convert representations of the same dimension")
#  end
#  Base.convert(changecoefficient_type(RepT, T), rep)
#end
#function Base.convert{T, RepT<:HRepresentation}(::Type{HRepresentation{T}}, rep::RepT)
#  if fulldim(RepT) != N
#    error("Cannot convert representations of the same dimension")
#  end
#  Base.convert(changecoefficient_type(RepT, T), rep)
#end
#function Base.convert{S, RepT<:VRepresentation}(::Type{VRepresentation{S}}, rep::RepT)
#  if fulldim(RepT) != M
#    error("Cannot convert representations of the same dimension")
#  end
#  Base.convert(changecoefficient_type(RepT, S), rep)
#end


### Instruction:
# Run before main.jl.
# CSV files must be in the input folder matching the simulatin model.
# For the list of input parameters, see readData function in Helper.jl
# For the list of input files, check all *.csv files in Helper.jl
using Distributed
@everywhere include("incl.jl")
@everywhere sayhello()
@everywhere push!(LOAD_PATH,pwd())

@everywhere begin
using TypeDef, Helper, Tools, Compute, Solve, FOCs, UserFileIO, MonteCarlo
using Post
using JLD, Printf
end

model =   0
mcid  =  20
b1    =   1
b2    = 200

### Monte Carlo analysis
for vInt=[false,true]
  @printf("\n=================== Starting Monte Carlo ===================\n")
  @printf("  Model: %2d  MCid: %2d  Vertical Int: %d\n",model,mcid,vInt)
  dat,dim,env,mat,par0,set,sim,indx,mask = readData(model,mcid,b1,b2,vInt,true)
  eqa = simMonteCarloMulti!(b1,b2,dat,dim,env,mat,par0,set,indx,mask,vInt)
end


export ExperienceBasedSamplingModel, sample

using Random
import StatsBase: sample

"""
    ExperienceBasedSamplingModel

Randomly generate a transition of (s, a, r, t, s′) based on previous experiences
in each sampling.
"""
Base.@kwdef mutable struct ExperienceBasedSamplingModel{R} <: AbstractEnvironmentModel
    experiences::Dict{Any,Dict{Any,Any}} = Dict{Any,Dict{Any,Any}}()
    sample_count::Int = 0
    rng::R = Random.GLOBAL_RNG
end

function RLBase.update!(
    m::ExperienceBasedSamplingModel,
    t::AbstractTrajectory,
    ::AbstractPolicy,
    ::AbstractEnv,
    ::Union{PreActStage,PostEpisodeStage},
)
    if length(t[:terminal]) > 0
        transition = (
            t[:state][end-1],
            t[:action][end-1],
            t[:reward][end],
            t[:terminal][end],
            t[:state][end],
        )
        update!(m, transition)
    end
end

function RLBase.update!(m::ExperienceBasedSamplingModel, transition::Tuple)
    s, a, r, d, s′ = transition
    if haskey(m.experiences, s)
        m.experiences[s][a] = (reward = r, terminal = d, nextstate = s′)
    else
        m.experiences[s] = Dict(a => (reward = r, terminal = d, nextstate = s′))
    end
end

sample(model::ExperienceBasedSamplingModel) = sample(model.rng, model)

function sample(rng::AbstractRNG, model::ExperienceBasedSamplingModel)
    if length(model.experiences) > 0
        s = rand(rng, keys(model.experiences))
        a = rand(rng, keys(model.experiences[s]))
        model.sample_count += 1
        s, a, model.experiences[s][a]...
    else
        nothing
    end
end


using MRIReco
using Test
using HTTP

@testset "ISMRMRD" begin

filename = joinpath(datadir, "simple_gre.h5")

f = ISMRMRDFile(filename)
acq = RawAcquisitionData(f)

@test acq.profiles[1].head.version == Int16(0)
@test acq.profiles[1].head.measurement_uid == Int32(37)
@test acq.profiles[1].head.scan_counter == Int32(1)
@test acq.profiles[1].head.number_of_samples == Int16(256)
@test acq.profiles[1].head.available_channels == Int16(32)
@test acq.profiles[1].head.active_channels == Int16(32)

params = Dict{Symbol, Any}()
params[:reco] = "direct"
params[:reconSize] = (256,128) #this should be clear from context

Ireco = reconstruction(AcquisitionData(acq), params)
exportImage(joinpath(tmpdir,"recogre.png"), abs.(Ireco[:,:,1,1,1]))

filenameCopy = joinpath(tmpdir, "simple_gre_copy.h5")
fCopy = ISMRMRDFile(filenameCopy)
# store data in another ISMRMRD file
save(fCopy, acq)
acqCopy = RawAcquisitionData(fCopy)

@test convert(AcquisitionHeaderImmutable, acq.profiles[1].head) ==
       convert(AcquisitionHeaderImmutable, acqCopy.profiles[1].head)
@test acqCopy.profiles[1].data == acq.profiles[1].data

IrecoCopy = reconstruction(AcquisitionData(acqCopy), params)

@test IrecoCopy == Ireco

filename = joinpath(datadir, "simple_spiral.h5")

f = ISMRMRDFile(filename)

params = Dict{Symbol, Any}()
params[:reco] = "direct"
params[:reconSize] = (128,128) #this should be clear from context

Ireco = reconstruction(AcquisitionData(f), params)
exportImage(joinpath(tmpdir,"recospiral.png"), abs.(Ireco[:,:,1,1,1]))

filenameCopy = joinpath(tmpdir, "simple_spiral_copy.h5")
fCopy = ISMRMRDFile(filenameCopy)
# store data in another ISMRMRD file
acq = RawAcquisitionData(f)
save(fCopy, acq)
acqCopy = RawAcquisitionData(f)

@test convert(MRIReco.AcquisitionHeaderImmutable, acq.profiles[1].head) ==
       convert(MRIReco.AcquisitionHeaderImmutable, acqCopy.profiles[1].head)
@test acqCopy.profiles[1].traj == acq.profiles[1].traj
@test acqCopy.profiles[1].data == acq.profiles[1].data

# 3D_partial_fourier.h5

#=filename = "3D_partial_fourier.h5"
if !isfile(filename)
  HTTP.open("GET", "https://netix.dl.sourceforge.net/project/ismrmrd/data/3D_partial_fourier.h5") do http
    open(filename, "w") do file
      write(file, http)
    end
  end
end

f = ISMRMRDFile(filename)
=#

#@test size(f.data) == (909, 32, 160)

end

#=

For our own reference: Here is code that can be used with Python to load an
ISMRMRD file

import h5py
f = h5py.File('simple_gre.h5', 'r')
dset = f['/dataset/data']
dset[0]['head']

=#


# Asset constructors

"""
    @asset asset_type code_symb name

Create a dimension and a reference unit for an asset.

The macros `@dimension` and `@refunit` are called with arguments derived
from `asset_type`, `code_symb` and `name`.

# Examples

```jldoctest
julia> @asset Cash USD "USD"
USD

julia> @asset Metal GOLD "GOLD"
GOLD

julia> @asset Stock DIS "DIS"
DIS
```
"""
macro asset(asset_type, code_symb, name)
    asset_str = titlecase(string(asset_type), strict=true)
    code_abbr = string(code_symb)

    code_abbr_bold = boldify(code_abbr)
    dimension = Symbol(code_abbr_bold)
    dim_abbr = asset_str * "{" * code_abbr_bold * "}"
    dim_name = Symbol(dim_abbr)

    esc(quote
        Unitful.@dimension($dimension, $dim_abbr, $dim_name)
        Unitful.@refunit($code_symb, $code_abbr, $name, $dimension, true)
    end)
end


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

################################################################################
# Common models used in testing
################################################################################
function rand_dims(max_ndim=6)
  tuple(rand(1:10, rand(1:max_ndim))...)
end

function mlp2()
  data = mx.Variable(:data)
  out = mx.FullyConnected(data, name=:fc1, num_hidden=1000)
  out = mx.Activation(out, act_type=:relu)
  out = mx.FullyConnected(out, name=:fc2, num_hidden=10)
  return out
end

function mlpchain()
  mx.@chain mx.Variable(:data) =>
            mx.FullyConnected(name=:fc1, num_hidden=1000) =>
            mx.Activation(act_type=:relu) =>
            mx.FullyConnected(name=:fc2, num_hidden=10)
end

"""
execution helper of SymbolicNode
"""
function exec(x::mx.SymbolicNode; feed...)
  ks, vs = zip(feed...)
  vs′ = mx.NDArray.(vs)

  e = mx.bind(x, context = mx.cpu(), args = Dict(zip(ks, vs′)))
  mx.forward(e)
  e.outputs
end


using Dates;
using Printf;
using Distributions;

root_path = raw"D:\AppliRTE\repo\scopf-quanti";
push!(LOAD_PATH, root_path);
cd(root_path);
include(joinpath(root_path, "AmplTxt.jl"));
include(joinpath(root_path, "Workflow.jl"));
include(joinpath(root_path, "DataManager.jl"));

# test_name = "5buses_wind";
test_name = "2buses"
dir_path = joinpath(root_path, test_name);

flexibility = read_flexibility(dir_path);

N_SCENARIO = 5;
df = Dates.DateFormat("Y-m-d-H:M");
h = Dates.DateTime("2015-01-01-11:00", df);
h_1 = Dates.DateTime("2015-01-01-10:00", df);
h_2 = Dates.DateTime("2015-01-01-09:00", df);

# TIME_STEP =  [Minute(0), Minute(15), Minute(30), Minute(45)];
# TIME_STEP =  [Minute(0), Minute(15)];
TIME_STEP =  [Minute(0)];
HORIZON = [h_2, h_1, h];

SCENARIO = [@sprintf("S%d", x) for x in 1:N_SCENARIO];

println("TIME_STEP : ", TIME_STEP);
println("SCENARIO : ", SCENARIO);
println("HORIZON : ", HORIZON);

id_fuel = read_id_fuel(dir_path);
amplTxt = AmplTxt.read(test_name);
num_to_name, bus_load = read_bus_load(amplTxt);
uncertainties, pmax_uncertain = read_uncertainties(dir_path);

println(bus_load);

println(id_fuel);
println(flexibility);

output_file_path = joinpath(dir_path, "pscopf_uncertainties.txt");
write_pscopf_uncertainties(output_file_path, uncertainties, SCENARIO, HORIZON, TIME_STEP);

output_file_path = joinpath(dir_path, "pscopf_previsions.txt");
write_pscopf_previsions(output_file_path, uncertainties, SCENARIO, HORIZON, TIME_STEP);

output_file_path = joinpath(dir_path, "pscopf_units.txt");
write_pscopf_units(output_file_path, amplTxt, id_fuel, flexibility);

output_file_path = joinpath(dir_path, "pscopf_gen_type_bus.txt");
write_pscopf_gen_type_bus(output_file_path, amplTxt, id_fuel, num_to_name);

println(id_fuel);


"""
Applies a function to each timeseries of a datacube present in a mask.
# Example:
```
julia> datacube = rand(1:10.0, 10,10,10)
julia> mask = rand(0:1, 10, 10)
julia> long_apply(sum, datacube)
```
"""
function long_apply(f, datacube::Array, mask = ones(size(datacube)[1], size(datacube)[2]))
    if ndims(f(datacube[1, 1, :])) ==0
        new_matrix = Array{Union{Missing, Float16}, 3}(undef, size(datacube, 1), size(datacube, 2))
        for i in 1:size(datacube)[1]
            for j in 1:size(datacube)[2]
                new_matrix[i, j] = f(datacube[i, j, :])
            end
        end
        return new_matrix
    else
        new_matrix = Array{Union{Missing, Float16}}(undef, size(datacube, 1), size(datacube, 2),size(datacube)[3])
        @showprogress for i in 1:size(datacube)[1]
            for j in 1:size(datacube)[2]
                if mask[i, j] == 1
                    new_matrix[i, j, :] = f(datacube[i, j, :])
                else
                    new_matrix[i, j, :] = datacube[i, j, :]
                end
            end
        end
        return new_matrix
    end
end
"""
Applies a function to each cross section of a datacube present.
# Example:
```
julia> datacube = rand(1:10.0, 10,10,10)
julia> mask = rand(0:1, 10, 10)
julia> long_apply(sum, datacube)
```
"""
function cross_apply(f, datacube, mask = ones(size(datacube)[1], size(datacube)[2]))
    if ndims(f(datacube[:, :, 1])) == 2
        new_matrix = Array{Union{Missing, Float16}}(undef, size(datacube, 1), size(datacube, 2),size(datacube)[3])
        for i in 1:size(datacube)[3]
            new_matrix[:, :, i] = f(datacube[:, :, i])
        end
        return new_matrix
    else
        array = []
        for i in 1:size(datacube)[3]
            push!(array, f(datacube[:, :, i]))
        end
        return array
    end
end


###################
# ForwardOptimize #
###################

for (M, f, arity) in DiffRules.diffrules()
    if arity == 1
        @eval @inline $M.$(f)(t::TrackedReal) = ForwardOptimize($M.$(f))(t)
    elseif arity == 2
        @eval @inline $M.$(f)(a::TrackedReal, b::TrackedReal) = ForwardOptimize($M.$(f))(a, b)
        for R in REAL_TYPES
            @eval begin
                @inline $M.$(f)(a::TrackedReal, b::$R) = ForwardOptimize($M.$(f))(a, b)
                @inline $M.$(f)(a::$R, b::TrackedReal) = ForwardOptimize($M.$(f))(a, b)
            end
        end
    end
end

################
# SkipOptimize #
################

# unary #
#-------#

for f in SKIPPED_UNARY_SCALAR_FUNCS
    @eval @inline Base.$(f)(t::TrackedReal) = SkipOptimize($(f))(t)
end

# binary #
#--------#

for f in SKIPPED_BINARY_SCALAR_FUNCS
    @eval @inline Base.$(f)(a::TrackedReal, b::TrackedReal) = SkipOptimize($(f))(a, b)
    for R in REAL_TYPES
        @eval begin
            @inline Base.$(f)(a::$R, b::TrackedReal) = SkipOptimize($(f))(a, b)
            @inline Base.$(f)(a::TrackedReal, b::$R) = SkipOptimize($(f))(a, b)
        end
    end
end

###########
# reverse #
###########

@noinline function scalar_reverse_exec!(instruction::ScalarInstruction{F,I,O,C}) where {F,I,O,C}
    f = instruction.func
    input = instruction.input
    output = instruction.output
    partials = instruction.cache[]
    if istracked(input)
        increment_deriv!(input, deriv(output) * partials)
    else
        a, b = input
        output_deriv = deriv(output)
        a_partial, b_partial = partials
        if istracked(a) && istracked(b)
            increment_deriv!(a, output_deriv * a_partial)
            increment_deriv!(b, output_deriv * b_partial)
        elseif istracked(a)
            increment_deriv!(a, output_deriv * a_partial)
        else
            increment_deriv!(b, output_deriv * b_partial)
        end
    end
    unseed!(output)
    return nothing
end

###########
# forward #
###########

@noinline function scalar_forward_exec!(instruction::ScalarInstruction{F,I,O,C}) where {F,I,O,C}
    f = instruction.func
    input = instruction.input
    output = instruction.output
    cache = instruction.cache
    if istracked(input)
        unary_scalar_forward_exec!(f, output, input, cache)
    else
        binary_scalar_forward_exec!(f, output, input, cache)
    end
    return nothing
end

@noinline function unary_scalar_forward_exec!(f::F, output::O, input, cache) where {F,O}
    pull_value!(input)
    result1 = DiffResult(zero(valtype(O)), zero(valtype(O)))
    result1 = ForwardDiff.derivative!(result1, f, value(input))
    value!(output, DiffResults.value(result1))
    cache[] = DiffResults.derivative(result1)
    return nothing
end

@noinline function binary_scalar_forward_exec!(f::F, output::O, input, cache) where {F,O}
    a, b = input
    pull_value!(a)
    pull_value!(b)
    if istracked(a) && istracked(b)
        result2 = DiffResults.GradientResult(SVector(zero(valtype(O)), zero(valtype(O))))
        result2 = ForwardDiff.gradient!(result2, x -> f(x[1], x[2]), SVector(value(a), value(b)))
        value!(output, DiffResults.value(result2))
        cache[] = DiffResults.gradient(result2)
    else
        result1 = DiffResult(zero(valtype(O)), zero(valtype(O)))
        if istracked(a)
            result1 = ForwardDiff.derivative!(result1, va -> f(va, b), value(a))
        else
            result1 = ForwardDiff.derivative!(result1, vb -> f(a, vb), value(b))
        end
        value!(output, DiffResults.value(result1))
        partial = DiffResults.derivative(result1)
        cache[] = SVector(partial, partial)
    end
    return nothing
end


"""
Coordinates of the QNode object in up to three spatial dimensions
"""
mutable struct Coords
    x::Float64
    y::Float64
    z::Float64

    Coords() = new(0,0,0)
    Coords(x,y) = new(x,y,0)
    Coords(x,y,z) = new(x,y,z)
end

"""
The default QNode object. Nothing special, but nothing unspecial either ;-)"
"""
mutable struct BasicNode <: QNode
    name::String
    costs::Dict
    has_memory::Bool
    memory_costs::Dict
    id::Int64
    time::Int64
    active::Bool
    location::Coords

    BasicNode() = new("", zero_costvector(), true, zero_costvector(), 0, 0, true, Coords())
    BasicNode(name) = new(string(name), zero_costvector(), true, zero_costvector(), 0, 0, true, Coords())
end

"""
Add a QNode object to the network.

Example:

```
using QuNet
Q = QNetwork()
A = BasicNode("A")
add(Q, A)
```
"""
function add(network::QNetwork, node::QNode)
    push!(network.nodes, node)
    node.id = length(network.nodes)
end

"""
Remove a Qnode object from the network.

The convention for node removal in QuNet echos that of LightGraphs.jl.
Suppose a given network has N nodes, and we want to remove the node with the
id v:

1. Check if v < N . If false, simply pop the node from QNetwork.nodes
2. Else, swap the nodes v and N, then pop from QNetworks.nodes
"""
function remove(network::QNetwork, node::QNode)
    node_id = node.id
    if node_id != length(network.nodes)
        # Swap the node to be removed with the last node
        # (Same removal strategy as SimpleGraphs)
        tmp_node = deepcopy(node)
        N = last(network.nodes)
        node = N
        node.id = node_id
        N = tmp_node
    end
    pop!(network.nodes)
end

"""
```function update(node::QNode)```

Does nothing
"""
function update(node::QNode)
end

function update(node::QNode, old_time::Float64, new_time::Float64)
end

# Identical structure to Coords, but using a different name for distrinction
"""
```
mutable struct Velocity
    x::Float64
    y::Float64
    z::Float64

    Velocity() = new(0,0,0)
    Velocity(x,y) = new(x,y,0)
    Velocity(x,y,z) = new(x,y,z)
```

Type for Cartesian Velocity in up to 3 spatial coordinates
"""
mutable struct Velocity
    x::Float64
    y::Float64
    z::Float64

    Velocity() = new(0,0,0)
    Velocity(x,y) = new(x,y,0)
    Velocity(x,y,z) = new(x,y,z)
end

"""
```
mutable struct PlanSatNode <: QNode
    name::String
    costs::Dict
    memory::Dict
    id::Int64
    time::Int64
    active::Bool
    location::Coords
    velocity::Velocity
```

The `PlanSatNode` type has all the functionality of a `BasicNode` but can move
according to a fixed velocity.
"""
mutable struct PlanSatNode <: QNode
    name::String
    costs::Dict
    has_memory::Bool
    memory_costs::Dict
    id::Int64
    time::Int64
    active::Bool
    location::Coords
    velocity::Velocity
    PlanSatNode() = new("", zero_costvector(), true, unit_costvector(),
                    0, 0, true, Coords(), Velocity())
    PlanSatNode(name) = new(string(name), zero_costvector(), true, zero_costvector(),
                        0, 0, true, Coords(), Velocity())
end

"""
```function update(sat::PlanSatNode)```

Update the position of a planar satellite node by incrementing its current
location with the distance covered by its velocity in `TIME_STEP` seconds.
"""
function update(sat::PlanSatNode, old_time::Float64, new_time::Float64)
    # Calculate new position from velocity
    sat.location.x += sat.velocity.x * (new_time - old_time)
    sat.location.y += sat.velocity.y * (new_time - old_time)
    sat.location.z += sat.velocity.z * (new_time - old_time)
    return
end


# MIT License
#
# Copyright (c) 2021 Juha Tapio Heiskala
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

module ITerSOLvers

using SparseArrays
import ITSOL_2_jll
import ZITSOL_1_jll

const libitsol2 = ITSOL_2_jll.libITSOL_2
const libzitsol1 = ZITSOL_1_jll.libZITSOL_1

# Sparse matrix data structures used by ITSOL libraries (ITSOL_2, ZITSOL_1)
mutable struct SpaFmt{T}
    n::Int32
    nzcount::Ptr{Cint}
    ja::Ptr{Ptr{Cint}}
    ma::Ptr{Ptr{T}}

    ja_ptrs::Array{Ptr{Cint},1}
    ma_ptrs::Array{Ptr{T},1}

    nzc_data::Array{Cint,1}
    ja_data::Array{Cint,1}
end

mutable struct ILUfac{T}
    n::Int32
    L::Ptr{Cvoid}
    D::Ptr{T}
    U::Ptr{Cvoid}
    work::Ptr{Cint}
end

mutable struct IluSpar{T}
    n::Cint
    C::Ptr{Cvoid}
    L::Ptr{Cvoid}
    U::Ptr{Cvoid}

    rperm::Ptr{Cint}
    perm::Ptr{Cint}
    perm2::Ptr{Cint}

    D1::Ptr{T}
    D2::Ptr{T}
    wk::Ptr{T}
end

# ILU0 solver

function ilu0(A::SparseMatrixCSC{T}, milu) where T <: Number
    itsolA = make_SpaFmt(A)
    iLU = ILUfac{T}(A.n, C_NULL, C_NULL, C_NULL, C_NULL)

    if T <: Real
        f = ccall((:itsol_pc_ilukC, libitsol2), Int32, (Int32, Ref{SpaFmt}, Ref{ILUfac}, Cint, Ptr{Cvoid}),
                  Cint(0), itsolA, iLU, Cint(milu), C_NULL)
    else
        f = ccall((:zilukC, libzitsol1), Int32, (Int32, Ref{SpaFmt}, Ref{ILUfac}, Cint, Ptr{Cvoid}),
                  Cint(0), itsolA, iLU, Cint(milu), C_NULL)
    end

    L = _make_jl_sparse(T, iLU.L)
    U = _make_jl_sparse(T, iLU.U)

    D = Array{T, 1}(undef, iLU.n)
    unsafe_copyto!(pointer(D), iLU.D, iLU.n)

    return (L, D, U)
end

# ILUC solver

function iluc(A::SparseMatrixCSC{T}, droptol, milu) where T <: Number
    itsolA = make_SpaFmt(A)

    inputILU = ILUfac{T}(A.n, C_NULL, C_NULL, C_NULL, C_NULL)

    if T<:Real
        f = ccall((:itsol_CSClumC, libitsol2), Int32, (Ref{SpaFmt}, Ref{ILUfac}, Cint), itsolA, inputILU, Cint(0))

        iLU = ILUfac{T}(A.n, C_NULL, C_NULL, C_NULL, C_NULL)

        f = ccall((:itsol_pc_ilutc, libitsol2), Cint, (Ref{ILUfac}, Ref{ILUfac}, Cint, Cdouble, Cint, Cint, Ptr{Cvoid}),
                  inputILU, iLU, Cint(A.n), Cdouble(droptol), Cint(5), Cint(milu), C_NULL)
    else
        f = ccall((:zCSClumC, libzitsol1), Int32, (Ref{SpaFmt}, Ref{ILUfac}, Cint), itsolA, inputILU, Cint(0))

        iLU = ILUfac{T}(A.n, C_NULL, C_NULL, C_NULL, C_NULL)

        f = ccall((:zilutc, libzitsol1), Cint, (Ref{ILUfac}, Ref{ILUfac}, Cint, Cdouble, Cint, Cint, Ptr{Cvoid}),
                  inputILU, iLU, Cint(A.n), Cdouble(droptol), Cint(5), Cint(milu), C_NULL)
    end

    L = _make_jl_sparse(T, iLU.L)
    U = _make_jl_sparse(T, iLU.U)

    D = Array{T, 1}(undef, iLU.n)
    unsafe_copyto!(pointer(D), iLU.D, iLU.n)

    return (L, D, U)
end

# ILUtp solver

function ilutp(A::SparseMatrixCSC{T}, droptol, thresh, milu, udiag) where T<:Number
    itsolA = make_SpaFmt(A)
    iLU = IluSpar{T}(A.n, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL,
                  C_NULL, C_NULL, C_NULL, C_NULL)

    droptolA = zeros(Cdouble, 7)
    droptolA[6] = droptol
    droptolA[7] = droptol

    lfilA = zeros(Cint, 7)
    lfilA[6] = A.n
    lfilA[7] = A.n

    if T<:Real
        f = ccall((:itsol_pc_ilutpC, libitsol2), Int32,
                  (Ref{SpaFmt}, Ptr{Cdouble}, Ptr{Cint}, Cdouble, Cint, Ref{IluSpar}, Cint),
                  itsolA, pointer(droptolA), pointer(lfilA), thresh, Cint(A.n), iLU, Cint(udiag))
    else
        f = ccall((:zilutpC, libzitsol1), Int32,
                  (Ref{SpaFmt}, Ptr{Cdouble}, Ptr{Cint}, Cdouble, Cint, Ref{IluSpar}, Cint),
                  itsolA, pointer(droptolA), pointer(lfilA), thresh, Cint(A.n), iLU, Cint(udiag))
    end

    L = _make_jl_sparse(T, iLU.L)
    U = _make_jl_sparse(T, iLU.U)

    P = Array{Cint, 1}(undef, iLU.n)
    unsafe_copyto!(pointer(P), iLU.perm2, iLU.n)

    return (L,U,P)
end

# convert from Julia SparseArray to ITSOL sparse array
function make_SpaFmt(A::SparseMatrixCSC{T}) where T <: Number

    nz_data = Array{Cint,1}(undef, A.n)
    ja_data = Array{Cint,1}(undef, nnz(A))

    ja_ptrs = Array{Ptr{Cint},1}(undef, A.n)
    ma_ptrs = Array{Ptr{T},1}(undef, A.n)

    cum_nnz = 0
    for c_ix = 1:A.n
        col_nnz = Cint(length(nzrange(A,c_ix)))
        nz_data[c_ix] = col_nnz

        for r_ix = A.colptr[c_ix]:(A.colptr[c_ix]+col_nnz-1)
            ja_data[r_ix]  = Cint(A.rowval[r_ix]-1)
        end
        ja_ptrs[c_ix] = pointer(ja_data) + cum_nnz*sizeof(Cint)
        ma_ptrs[c_ix] = pointer(A.nzval) + cum_nnz*sizeof(T)
        cum_nnz += col_nnz
    end

    itsolA = SpaFmt{T}(Cint(A.n), pointer(nz_data), pointer(ja_ptrs), pointer(ma_ptrs),
                    ja_ptrs, ma_ptrs, nz_data, ja_data)

    return itsolA
end

# convert from ITSOL SparseArray to Julia sparse array
function  _make_jl_sparse(T, A::Ptr{Cvoid})
    n = unsafe_load(Ptr{Cint}(A))

    row_nz_cnt = Array{Int32,1}(undef, n+1)
    row_nz_cnt[1] = Int32(1)
    unsafe_copyto!(pointer(row_nz_cnt)+sizeof(Int32), unsafe_load(Ptr{Ptr{Cint}}(A+8)), n)
    colptr = cumsum(row_nz_cnt)

    n_nz = colptr[end]-1

    rowval = Vector{Int32}(undef, n_nz)
    nzval = Vector{T}(undef, n_nz)

    r_ix_array_ptr = unsafe_load(Ptr{Ptr{Cvoid}}(A+16))
    val_array_ptr = unsafe_load(Ptr{Ptr{Cvoid}}(A+24))
    for c = 1:n
        unsafe_copyto!(pointer(rowval)+(colptr[c]-1)*sizeof(Cint),
                       Ptr{Cint}(unsafe_load(Ptr{Ptr{Cvoid}}(r_ix_array_ptr), c)), row_nz_cnt[c+1])
        unsafe_copyto!(pointer(nzval)+(colptr[c]-1)*sizeof(T),
                       Ptr{T}(unsafe_load(Ptr{Ptr{Cvoid}}(val_array_ptr), c)), row_nz_cnt[c+1])

    end
    jlA = SparseMatrixCSC{T,Int64}(n, n, Int64.(colptr), (Int64.(rowval)).+Int64(1), nzval)
    return jlA
end

end # module


function updateDictionary(optr_t::Type{AG}, opt_params, cb::DT, x::AbstractArray, hy::AbstractArray, params; shape::Shapes.AbstractShape=Shapes.Vec(size(x)), vlevel::Integer=0, kwargs...) where {DT<:LearningTarget,TT<:AbstractArray,TM<:AbstractArray,AG<:Optimizers.AbstractGradientDescent}
    vecpm, pminfo = decompose_params(DT, params)
    f(t) = lossfcn(cb, x, hy, compose_params(DT, t, pminfo); shape=shape)
    ∇f(t) = ForwardDiff.gradient(f, t)
    optr = optr_t(∇f; opt_params...)

    vecpm_opt = optr(vecpm; kwargs...)

    params_opt = compose_params(DT, vecpm_opt, pminfo)
    loss_opt = f(vecpm_opt)
    return (params_opt, loss_opt,)
end

function updateDictionary(nlopt_t::Type{Optimizers.CRS}, opt_params, cb::DT, x::AbstractArray, hy::AbstractArray, params; shape=Shapes.Vec(size(x)), vlevel::Integer=0, kwargs...) where {DT<:LearningTarget}
    vecpm, pminfo = decompose_params(DT, params)
    f(t, grad=nothing) = lossfcn(cb, x, hy, compose_params(DT, t, pminfo); shape=shape)

    nlopt = nlopt_t(f; opt_params...)

    vecpm_opt = nlopt(vecpm)
    minf = f(vecpm_opt)
    params_opt = compose_params(DT, vecpm_opt, pminfo)
    # vlevel >= 0 && println("fopt(θ) = $minf, f(θ) = $(f(minx))")
    return (params_opt, minf)
end

function updateDictionary(nlopt_t::Type{Optimizers.GlobalOpt}, opt_params, cb::DT, x::AbstractArray, hy::AbstractArray, params; shape=Shapes.Vec(size(x)), vlevel::Integer=0, kwargs...) where {DT<:LearningTarget}
    vecpm, pminfo = decompose_params(DT, params)
    f(t, grad=nothing) = lossfcn(cb, x, hy, compose_params(DT, t, pminfo); shape=shape)

    nlopt = nlopt_t(f; opt_params...)

    vecpm_opt = nlopt(vecpm)
    minf = f(vecpm_opt)
    params_opt = compose_params(DT, vecpm_opt, pminfo)
    # vlevel >= 0 && println("fopt(θ) = $minf, f(θ) = $(f(minx))")
    return (params_opt, minf)
end

function lossfcn(cb::LearningTarget, x::AbstractArray, y::AbstractArray, params; shape=Shapes.Vec(size(x))) where {TT<:AbstractArray,TM<:AbstractArray}
    cpcb = similar_dictionary(cb, params)
    syn = createTransform(cpcb, shape)
    norm(x - synthesize(syn, y))^2/2
end

function similar_dictionary(nsolt::AbstractNsolt, (θ, μ)::Tuple{TT,TM}) where {TT<:AbstractArray,TM<:AbstractArray}
    setrotations!(similar(nsolt, eltype(θ)), θ, μ)
end

similar_dictionary(target::Multiscale, params::NTuple{N}) where {N} = Multiscale(similar_dictionary.(target.filterbanks, params)...)

getParamsDictionary(nsolt::AbstractNsolt) = getrotations(nsolt)
setParamsDictionary!(nsolt::AbstractNsolt, pm::NTuple{2}) = setrotations!(nsolt, pm...)

function getParamsDictionary(nsolt::Cnsolt)
    θ, μ = getrotations(nsolt)
    ([angle.(diag(nsolt.Φ)); θ], μ)
end

function setParamsDictionary(nsolt::Cnsolt, (ϕnθ, μ))
    P = nchannels(nsolt)
    nsolt.Φ .= Diagonal(cis.(ϕnθ[1:P]))
    return setrotations!(nsolt, ϕnθ[(P+1):end], μ)
end

getParamsDictionary(targets::Multiscale) = map(t->getParamsDictionary(t), targets.filterbanks)
setParamsDictionary!(targets::Multiscale, pm::NTuple) = map((t,p)->setParamsDictionary!(t, p), targets.filterbanks, pm)

getOperatorType_scs(::Type, ::Val) = AbstractOperator
getOperatorType_scs(::Type{NS}, ::Val{:convolution}) where {NS<:AbstractNsolt} = ConvolutionalOperator

decompose_params(::Type, params) = (params, ())
compose_params(::Type, params, ()) = params

decompose_params(::Type{NS}, (θ, μ)) where {NS<:AbstractNsolt} = (θ, μ)
compose_params(::Type{NS}, θ, μ) where {NS<:AbstractNsolt} = (θ, μ)

function decompose_params(::Type{Multiscale}, params::NTuple{N}) where {N} # T<:AbstractNsolt
    θs, μs = map(t->t[1], params), map(t->t[2], params)
    vcat(θs...), (length.(θs), μs)
end

function compose_params(::Type{Multiscale}, vp::AbstractArray, pminfo::Tuple) where {N}
    arrpms = map(rng->vp[rng],intervals(pminfo[1]))
    map((lhs,rhs)->(lhs,rhs,), arrpms, pminfo[2])
end


#debug fullLocalGraphCopy!
using Caesar, RoME, IncrementalInference
using CloudGraphs, Neo4j
# using Distributions
using JSON

using KernelDensityEstimate

using Base.Test

## Uncomment out for command line operation
# cloudGraph, addrdict = standardcloudgraphsetup(nparticles=true)
# session = addrdict["session"]
# Nparticles = parse(Int,addrdict["num particles"])

# interactive operation
session = "SESSROX"
Nparticles = 100
include(joinpath(dirname(@__FILE__),"blandauthremote.jl"))

fg = Caesar.initfg(sessionname=session, cloudgraph=cloudGraph)

# setDBAllReady!(fg)
setBackendWorkingSet!(session)

fullLocalGraphCopy!(fg)

IDs = getAllExVertexNeoIDs(conn, sessionname=fg.sessionname)

tosee = Dict()
tosee2 = Dict()
for i in 1:length(IDs)
  vid, nid = IDs[i]
  tosee[vid] = nid
  tosee2[nid] = vid
end

tosee2[149088]

writeGraphPdf(fg)
# @async run(`evince fg.pdf`)

tree = wipeBuildNewTree!(fg,drawpdf=true)
# @async run(`evince bt.pdf`)

spyCliqMat(tree, :l200050)

inferOverTreeR!(fg, tree, N=N, dbg=true)

# Some inspection methods to see what is going on
spyCliqMat(tree, :l200050)

plotMCMC(tree, :l200050)

plotPriorsAtCliq(tree, :l200050, :l200050, dims=[1;2], levels=1)

plotUpMsgsAtCliq(tree, :l200050, :l200050, marg=[1;2])

# basic plotting of Pose2D and landmarks
drawPosesLandms(fg ,spscale=0.5)

lbll=:l200050
cliq = whichCliq(tree, string(lbll))
cliqdbg = cliq.attributes["debug"]
vidx = 1
for lb in cliqdbg.mcmc[1].lbls
  if lb == lbll
    break;
  else
    vidx += 1
  end
end
ppp = kde!(cliqdbg.mcmc[3].prods[vidx].product)
plotKDE(ppp)

vc = startdefaultvisualization(draworigin=true)

drawposepoints!(vc, fg, :x4)

visualizeallposes!(vc, fg, drawlandms=false, drawtype=:fit)

drawPoses(fg,from=4,to=4)

# prior on :x1 issue

fc = getVert(fg, 100007)

vid = fg.IDs[:l200050]
vid = fg.IDs[:x4]

fc

Profile.clear()

@profile pts = evalFactor2(fg, fc, vid)
plotKDE(kde!(pts[1:2,:]))

getSample( getData(fc).fnc.usrfnc!, 100 )

plotKDE(getData(fc).fnc.usrfnc!.range)

#


# This file is a part of BAT.jl, licensed under the MIT License (MIT).

# ToDo: Add `density_logvalgrad!` to support HMC, etc.

@doc doc"""
    AbstractDensity

Subtypes of `AbstractDensity` must implement the function

* `BAT.density_logval`

For likelihood densities this is typically sufficient, since shape, and
bounds of parameters will be inferred from the prior.

Densities with known parameters bounds may also implement

* `BAT.param_bounds`

If the parameter bounds are unkown, but the number of parameters is known,
the function

* `BAT.nparams`

may be implemented directly (usually it is inferred from the bounds).

!!! note

    The functions `BAT.param_bounds` and `BAT.params_shape` are currently not
    part of the stable public BAT-API, their names and arguments may change
    without notice.

Densities that support named parameters should also implement

* `BAT.params_shape`
"""
abstract type AbstractDensity end
export AbstractDensity

@doc doc"""
    BAT.density_logval(density::AbstractDensity, params::Any)

Compute log of value of a multivariate density function at the given
parameter values.

Input:

* `density`: density function
* `params`: parameter values

Note: If `density_logval` is called with out-of-bounds parameters (see
`param_bounds`), the behaviour is undefined. The result for parameters that
are not within bounds is *implicitly* `-Inf`, but it is the caller's
responsibility to handle these cases.
"""
function density_logval end

@doc doc"""
    param_bounds(
        density::AbstractDensity
    )::Union{AbstractParamBounds,Missing}

Get the parameter bounds of `density`. See `density_logval` for the
implications and handling of the bounds. If the bounds are missing,
`density_logval` must be prepared to handle any parameter values.
"""
param_bounds(density::AbstractDensity) = missing

@doc doc"""
    nparams(density::AbstractDensity)::Union{Int,Missing}

Get the number of parameters of `density`. May return `missing`, if the
density supports a variable number of parameters.
"""
function nparams(density::AbstractDensity)
    bounds = param_bounds(density)
    ismissing(bounds) ? missing : nparams(bounds)
end
export nparams

@doc doc"""
    params_shape(
        density::AbstractDensity
    )::Union{ValueShapes.AbstractValueShape,Missing}

    params_shape(
        density::DistLikeDensity
    )::ValueShapes.AbstractValueShape

    params_shape(
        distribution::Distributions.Distribution
    )::ValueShapes.AbstractValueShape

*BAT-internal, not part of stable public API.*

Get the shapes of parameters of `density`.

For prior densities, the result must not be `missing`, but may be `nothing` if
the prior only supports flat parameter vectors.
"""
function params_shape end

params_shape(density::AbstractDensity) = missing

params_shape(dist::Distribution) = valshape(dist)

@doc doc"""
    eval_density_logval(
        density::AbstractDensity,
        params::AbstractVector{<:Real},
        parshapes::ValueShapes.AbstractValueShape
    )

*BAT-internal, not part of stable public API.*

Evaluates density log-value via `density_logval`.

`parshapes` *must* be compatible with `params_shape(density)`.

Checks that:

* The number of parameters of `density` (if known) matches the length of
  `params`.
* The return value of `density_logval` is not `NaN`.
* The return value of `density_logval` is less than `+Inf`.
"""
function eval_density_logval(
    density::AbstractDensity,
    params::AbstractVector{<:Real},
    parshapes::ValueShapes.AbstractValueShape
)
    npars = nparams(density)
    ismissing(npars) || (length(eachindex(params)) == npars) || throw(ArgumentError("Invalid length of parameter vector"))

    r = float(density_logval(density, _apply_parshapes(params, parshapes)))
    isnan(r) && throw(ErrorException("Return value of density_logval must not be NaN, density has type $(typeof(density))"))
    r < convert(typeof(r), +Inf) || throw(ErrorException("Return value of density_logval must not be posivite infinite, density has type $(typeof(density))"))

    r
end

_apply_parshapes(params::AbstractVector{<:Real}, parshapes::AbstractValueShape) = parshapes(params)

@doc doc"""
    DistLikeDensity <: AbstractDensity

A density that implements part of the `Distributions.Distribution` interface.
Such densities are suitable to be used as a priors.

Typically, custom priors should be implemented as subtypes of
`Distributions.Distribution`. BAT will automatically wrap them in a subtype of
`DistLikeDensity`.

Subtypes of `DistLikeDensity` are required to support more functionality
than a `AbstractDensity`, but less than a
`Distribution{Multivariate,Continuous}`.

The following functions must be implemented for subtypes:

* `BAT.density_logval`

* `BAT.param_bounds`

* `BAT.params_shape`

* `Distributions.sampler`

* `Statistics.cov`

!!! note

    The functions `BAT.param_bounds` and `BAT.params_shape` are currently not
    part of the stable public BAT-API, their names and arguments may change
    without notice.

Prior densities that support non-flat parameters should also implement

* `BAT.params_shape`

A `d::Distribution{Multivariate,Continuous}` can be converted into (wrapped
in) an `DistLikeDensity` via `conv(DistLikeDensity, d)`.
"""
abstract type DistLikeDensity <: AbstractDensity end
export DistLikeDensity

@doc doc"""
    param_bounds(density::DistLikeDensity)::AbstractParamBounds

*BAT-internal, not part of stable public API.*

Get the parameter bounds of `density`. Must not be `missing`.
"""
function param_bounds end

@doc doc"""
    nparams(density::DistLikeDensity)::Int

Get the number of parameters of prior density `density`. Must not be
`missing`, prior densities must have a fixed number of parameters. By default,
the number of parameters is inferred from the parameter bounds.
"""
nparams(density::DistLikeDensity) = nparams(param_bounds(density))


# Copyright (c) 2015, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# FIXME: This should be moved into the Latte core as it uses special internal
# handling.
export ConcatLayer

function ConcatEnsemble(name::Symbol, inputs::Vector)
    # shape = (size(inputs[1])..., length(inputs))
    shape = [size(inputs[1])...]
    inner_size = prod(shape)
    shape[end] *= length(inputs)
    neurons = Array(ConcatNeuron, shape...)
    for i in 1:length(neurons)
        neurons[i] = ConcatNeuron(0.0, 0.0)
    end
    ConcatEnsemble(name, neurons, [], TrainTest, inputs, inner_size, 1)
end

function init(ensemble::ConcatEnsemble, net::Net)
    set_buffer(net, symbol(ensemble.name, :value), Array(Float32, size(ensemble)..., net.batch_size))
    set_buffer(net, symbol(ensemble.name, :∇), Array(Float32, size(ensemble)..., net.batch_size))
end

function ConcatLayer(name::Symbol, net::Net, input_ensembles...)
    @assert length(input_ensembles) > 1 "Must concatenate at least two ensembles"
    for ens in input_ensembles[2:end]
        @assert(size(input_ensembles[1]) == size(input_ensembles[2]),
                "All inputs to ConcatLayer must have the same size")
    end

    ens = ConcatEnsemble(name, [input_ensembles...])
    add_ensemble(net, ens)
    ens
end


abstract type Proposal{P} end

struct StaticProposal{issymmetric,P} <: Proposal{P}
    proposal::P
end
const SymmetricStaticProposal{P} = StaticProposal{true,P}

StaticProposal(proposal) = StaticProposal{false}(proposal)
function StaticProposal{issymmetric}(proposal) where {issymmetric}
    return StaticProposal{issymmetric,typeof(proposal)}(proposal)
end

struct RandomWalkProposal{issymmetric,P} <: Proposal{P}
    proposal::P
end
const SymmetricRandomWalkProposal{P} = RandomWalkProposal{true,P}

RandomWalkProposal(proposal) = RandomWalkProposal{false}(proposal)
function RandomWalkProposal{issymmetric}(proposal) where {issymmetric}
    return RandomWalkProposal{issymmetric,typeof(proposal)}(proposal)
end

# Random draws
Base.rand(p::Proposal, args...) = rand(Random.GLOBAL_RNG, p, args...)
Base.rand(rng::Random.AbstractRNG, p::Proposal{<:Distribution}) = rand(rng, p.proposal)
function Base.rand(rng::Random.AbstractRNG, p::Proposal{<:AbstractArray})
    return map(x -> rand(rng, x), p.proposal)
end

# Densities
Distributions.logpdf(p::Proposal{<:Distribution}, v) = logpdf(p.proposal, v)
function Distributions.logpdf(p::Proposal{<:AbstractArray}, v)
    # `mapreduce` with multiple iterators requires Julia 1.2 or later
    return mapreduce(((pi, vi),) -> logpdf(pi, vi), +, zip(p.proposal, v))
end

###############
# Random Walk #
###############

function propose(
    rng::Random.AbstractRNG,
    proposal::RandomWalkProposal{issymmetric,<:Union{Distribution,AbstractArray}},
    ::DensityModel
) where {issymmetric}
    return rand(rng, proposal)
end

function propose(
    rng::Random.AbstractRNG,
    proposal::RandomWalkProposal{issymmetric,<:Union{Distribution,AbstractArray}},
    model::DensityModel,
    t
) where {issymmetric}
    return t + rand(rng, proposal)
end

function q(
    proposal::RandomWalkProposal{issymmetric,<:Union{Distribution,AbstractArray}},
    t,
    t_cond
) where {issymmetric}
    return logpdf(proposal, t - t_cond)
end

##########
# Static #
##########

function propose(
    rng::Random.AbstractRNG,
    proposal::StaticProposal{issymmetric,<:Union{Distribution,AbstractArray}},
    model::DensityModel,
    t=nothing
) where {issymmetric}
    return rand(rng, proposal)
end

function q(
    proposal::StaticProposal{issymmetric,<:Union{Distribution,AbstractArray}},
    t,
    t_cond
) where {issymmetric}
    return logpdf(proposal, t)
end

############
# Function #
############

# function definition with abstract types requires Julia 1.3 or later
for T in (:StaticProposal, :RandomWalkProposal)
    @eval begin
        function (p::$T{issymmetric,<:Function})() where {issymmetric}
            return $T{issymmetric}(p.proposal())
        end
        function (p::$T{issymmetric,<:Function})(t) where {issymmetric}
            return $T{issymmetric}(p.proposal(t))
        end
    end
end

function propose(
    rng::Random.AbstractRNG,
    proposal::Proposal{<:Function},
    model::DensityModel
)
    return propose(rng, proposal(), model)
end

function propose(
    rng::Random.AbstractRNG,
    proposal::Proposal{<:Function},
    model::DensityModel,
    t
)
    return propose(rng, proposal(t), model)
end

function q(
    proposal::Proposal{<:Function},
    t,
    t_cond
)
    return q(proposal(t_cond), t, t_cond)
end

####################
# Multiple proposals
####################

function propose(
    rng::Random.AbstractRNG,
    proposals::AbstractArray{<:Proposal},
    model::DensityModel,
)
    return map(proposals) do proposal
        return propose(rng, proposal, model)
    end
end
function propose(
    rng::Random.AbstractRNG,
    proposals::AbstractArray{<:Proposal},
    model::DensityModel,
    ts,
)
    return map(proposals, ts) do proposal, t
        return propose(rng, proposal, model, t)
    end
end

@generated function propose(
    rng::Random.AbstractRNG,
    proposals::NamedTuple{names},
    model::DensityModel,
) where {names}
    isempty(names) && return :(NamedTuple())
    expr = Expr(:tuple)
    expr.args = Any[:($name = propose(rng, proposals.$name, model)) for name in names]
    return expr
end

@generated function propose(
    rng::Random.AbstractRNG,
    proposals::NamedTuple{names},
    model::DensityModel,
    ts,
) where {names}
    isempty(names) && return :(NamedTuple())
    expr = Expr(:tuple)
    expr.args = Any[
        :($name = propose(rng, proposals.$name, model, ts.$name)) for name in names
    ]
    return expr
end

"""
    logratio_proposal_density(proposal, state, candidate)

Compute the log-ratio of the proposal densities in the Metropolis-Hastings algorithm.

The log-ratio of the proposal densities is defined as
```math
\\log \\frac{g(x | x')}{g(x' | x)},
```
where ``x`` is the current state, ``x'`` is the proposed candidate for the next state,
and ``g(y' | y)`` is the conditional probability of proposing state ``y'`` given state
``y`` (proposal density).
"""
function logratio_proposal_density(proposal::Proposal, state, candidate)
    return q(proposal, state, candidate) - q(proposal, candidate, state)
end

# ratio is always 0 for symmetric proposals
logratio_proposal_density(::RandomWalkProposal{true}, state, candidate) = 0
logratio_proposal_density(::StaticProposal{true}, state, candidate) = 0

# type stable implementation for `NamedTuple`s
function logratio_proposal_density(
    proposals::NamedTuple{names}, states::NamedTuple, candidates::NamedTuple
) where {names}
    if @generated
        args = map(names) do name
            :(logratio_proposal_density(
                proposals[$(QuoteNode(name))],
                states[$(QuoteNode(name))],
                candidates[$(QuoteNode(name))],
            ))
        end
        return :(+($(args...)))
    else
        return sum(names) do name
            return logratio_proposal_density(
                proposals[name], states[name], candidates[name]
            )
        end
    end
end

# use recursion for `Tuple`s to ensure type stability
logratio_proposal_density(proposals::Tuple{}, states::Tuple, candidates::Tuple) = 0
function logratio_proposal_density(
    proposals::Tuple{<:Proposal}, states::Tuple, candidates::Tuple
)
    return logratio_proposal_density(first(proposals), first(states), first(candidates))
end
function logratio_proposal_density(proposals::Tuple, states::Tuple, candidates::Tuple)
    valfirst = logratio_proposal_density(first(proposals), first(states), first(candidates))
    valtail = logratio_proposal_density(
        Base.tail(proposals), Base.tail(states), Base.tail(candidates)
    )
    return valfirst + valtail
end

# fallback for general iterators (arrays etc.) - possibly not type stable!
function logratio_proposal_density(proposals, states, candidates)
    return sum(zip(proposals, states, candidates)) do (proposal, state, candidate)
        return logratio_proposal_density(proposal, state, candidate)
    end
end


function creatematerialstable!(data::Dict{Symbol, Any})
  push!(data, :materials=>Dict{Symbol, Dict{Symbol, Float64}}())
  #
  # Insert default values for :steel
  #
  merge!(data[:materials], Dict(:steel=>Dict{Symbol, Float64}()))
  push!(data[:materials][:steel], :sg=>0.283)
  push!(data[:materials][:steel], :ez => 29.0e6)
  push!(data[:materials][:steel], :er => 29.0e6)
  push!(data[:materials][:steel], :g=>11.5e6)

  merge!(data[:materials], Dict(:monel=>Dict{Symbol, Float64}()))
  push!(data[:materials][:monel], :sg=>0.319)
  push!(data[:materials][:monel], :ez => 26.0e6)
  push!(data[:materials][:monel], :er => 26.0e6)
  push!(data[:materials][:monel], :g=>9.9e6)

  merge!(data[:materials], Dict(:titanium=>Dict{Symbol, Float64}()))
  push!(data[:materials][:titanium], :sg=>0.164)
  push!(data[:materials][:titanium], :ez => 15.5e6)
  push!(data[:materials][:titanium], :er => 15.5e6)
  push!(data[:materials][:titanium], :g=>5.9e6)

  merge!(data[:materials], Dict(:aluminium=>Dict{Symbol, Float64}()))
  push!(data[:materials][:aluminium], :sg=>0.1)
  push!(data[:materials][:aluminium], :ez => 10.3e6)
  push!(data[:materials][:aluminium], :er => 10.3e6)
  push!(data[:materials][:aluminium], :g=>3.9e6)
end


import Base:in, +, -, *, ==, convert, isempty, isequal, length, push!, sizeof, append!
import SeisIO: GeoLoc, PZResp
export FFTData

const fftfields = [:name, :loc, :fs, :gain, :freqmin, :freqmax, :cc_step,
                   :cc_len, :whitened, :time_norm, :resp]

"""
   FFTData

A structure for fourier transforms (FFT) of ambient noise data.

## Fields: FFTData
| **Field** | **Description** |
|:--------------------|:--------------------|
| :name       | Freeform channel names |
| :id         | Channel ids. use NET.STA.LOC.CHAN format when possible. |
| :loc        | Location (position) object |
| :fs         | Sampling frequency in Hz. |
| :gain       | Scalar gain; divide data by the gain to convert to units  |
| :freqmin    | Minimum frequency for whitening.  |
| :freqmax    | Maximum frequency for whitening. |
| :cc_len     | Length of each correlation in seconds. |
| :cc_step    | Spacing between correlation windows in seconds. |
| :whitened   | Whitening applied.
| :time_norm  | Apply one-bit whitening with "one_bit". |
| :resp       | Instrument response object, format [zeros poles] |
| :misc       | Dictionary for non-critical information. |
| :notes      | Timestamped notes; includes automatically-logged acquisition and |
|             | processing information. |
| :t          | Starttime of each FFT. |
| :fft        | FFTs stored in columns. |
"""
mutable struct FFTData
  name::String                                # name [Net.Sta.Loc.Chan]
  id::String                                  # id [Y-mm-dd] this is date of fft
  loc::GeoLoc                                 # loc
  fs::Float64                                 # sampling rate [Hz]
  gain::Float64                               # gain
  freqmin::Float64                            # minumum frequency [Hz]
  freqmax::Float64                            # maximum frequency [Hz]
  cc_len::Int                                 # window_length [s]
  cc_step::Int                                # step between windows [s]
  whitened::Bool                              # whitening applied
  time_norm::Union{Bool,String}               # time normaliation
  resp::PZResp                         # response poles/zeros
  misc::Dict{String,Any}                      # misc
  notes::Array{String,1}                      # notes
  t::Array{Float64,1}                         # time
  fft::Array{<:Union{Complex{Float32},Complex{Float64}},2}  # fft data

  function FFTData(
      name     ::String,
      id       ::String,
      loc      ::GeoLoc,
      fs       ::Float64,
      gain     ::Float64,
      freqmin  ::Float64,
      freqmax  ::Float64,
      cc_len   ::Int,
      cc_step  ::Int,
      whitened ::Bool,
      time_norm::Union{Bool,String},
      resp     ::PZResp,
      misc     ::Dict{String,Any},
      notes    ::Array{String,1},
      t        ::Array{Float64,1},
      fft      ::Array{<:Union{Complex{Float32},Complex{Float64}},2}
      )

      return new(name, id, loc, fs, gain, freqmin, freqmax, cc_len, cc_step,
                 whitened, time_norm, resp, misc, notes, t, fft)
    end
end

FFTData(;
          name     ::String                    = "",
          id       ::String                    = "",
          loc      ::GeoLoc                    = GeoLoc(),
          fs       ::Float64                   = zero(Float64),
          gain     ::Float64                   = one(Float64),
          freqmin  ::Float64                   = zero(Float64),
          freqmax  ::Float64                   = zero(Float64),
          cc_len   ::Int                       = zero(Int),
          cc_step  ::Int                       = zero(Int),
          whitened ::Bool                      = false,
          time_norm::Union{Bool,String}        = false,
          resp     ::PZResp                    = PZResp(),
          misc     ::Dict{String,Any}          = Dict{String,Any}(),
          notes    ::Array{String,1}           = Array{String,1}(undef, 0),
          t        ::Array{Float64,1}          = Array{Float64,1}(undef, 0),
          fft      ::Array{<:Union{Complex{Float32},Complex{Float64}},2} =
                     Array{Complex{Float32},2}(undef, 0, 2)
          ) = FFTData(name, id, loc, fs, gain, freqmin, freqmax, cc_len, cc_step,
                     whitened, time_norm, resp, misc, notes, t, fft)

FFTData(C::SeisChannel,freqmin::Float64, freqmax::Float64,cc_len::Int,
        cc_step::Int, whitened::Bool,time_norm::Union{Bool,String},
        t::Array{Float64,1},fft::Array{<:Union{Complex{Float32},Complex{Float64}},2}
       ) = FFTData(C.id, Dates.format(u2d(C.t[1,2]*1e-6),"Y-mm-dd"), C.loc, C.fs, C.gain, freqmin, freqmax, cc_len,
                   cc_step, whitened, time_norm, C.resp, C.misc, C.notes, t, fft)

in(s::String, F::FFTData) = F.id==s

isempty(F::FFTData) = min(F.name=="",F.id == "",isempty(F.loc),
                          F.fs == zero(typeof(F.fs)),
                          F.gain == one(typeof(F.gain)),
                          F.freqmin == zero(typeof(F.freqmin)),
                          F.freqmax == zero(typeof(F.freqmax)),
                          F.cc_len == zero(typeof(F.cc_len)),
                          F.cc_step == zero(typeof(F.cc_step)),
                          F.whitened == false,
                          F.time_norm == false,
                          isempty(F.resp), isempty(F.misc),isempty(F.notes),
                          isempty(F.t),isempty(F.fft))

isequal(F::FFTData, U::FFTData) = minimum([hash(getfield(F,i))==hash(getfield(U,i)) for i in fftfields]::Array{Bool,1})
==(F::FFTData, U::FFTData) = isequal(F,U)::Bool

append!(F::FFTData, U::FFTData)  = (if F == U;
  setfield!(F, :t, vcat(getfield(F,:t), getfield(U,:t)));
  setfield!(F, :fft, hcat(getfield(F,:fft), getfield(U,:fft)))
  elseif isempty(F);
  [setfield!(F, i, getfield(U,i)) for i in fftfields];
  [setfield!(F, i, getfield(U,i)) for i in [:t,:fft]];
  end;
  return F)
+(F::FFTData, U::FFTData) = (T = deepcopy(F); return append!(T, U))

function sizeof(F::FFTData)
s = sum([sizeof(getfield(F,f)) for f in fftfields])
if !isempty(F.notes)
  s += sum([sizeof(i) for i in F.notes])
end
if !isempty(F.misc)
  s += sum([sizeof(i) for i in values(F.misc)])
end
return s
end


export rasterize_datasets, create_scenario_datasets, cleanup_datasets, makeprotected, savelandcover,
        createGDP, creategridaccess, getpopulation, getwindatlas

# cleanup options: :none, :limited, :all
function rasterize_datasets(; cleanup=:all)
    rasterize_GADM()
    rasterize_NUTS()
    rasterize_protected()
    downscale_landcover()
    savelandcover()
    upscale_topography()
    saveregions_global()
    rasterize_timezones()
    maketimezones()
    cleanup_datasets(cleanup=cleanup)
end

function create_scenario_datasets(scen, year)
    if !isfile(in_datafolder("population_$(scen)_$year.jld"))
        println("\nCreating population dataset for $scen $year...")
        downscale_population(scen, year)
    end
    if !isfile(in_datafolder("gdp_$(scen)_$year.jld"))
        println("\nCreating GDP dataset for $scen $year...")
        createGDP(scen, year)
    end
    if !isfile(in_datafolder("gridaccess_$(scen)_$year.jld"))
        println("\nCreating grid access dataset for $scen $year...")
        creategridaccess(scen, year)
    end
end

# cleanup options: :none, :limited, :all
function cleanup_datasets(; cleanup=:all)
    cleanup == :none && return
    for i = 0:2
        rm(in_datafolder("WDPA", "protected_raster$i.tif"), force=true)
        rm(in_datafolder("WDPA", "protectedfields$i.csv"), force=true)
        rm(in_datafolder("WDPA", "protected.jld$i"), force=true)
        rm(in_datafolder("WDPA", "WDPA-shapefile$i"), force=true, recursive=true)
    end
    rm(in_datafolder("landcover.tif"), force=true)
    rm(in_datafolder("topography.tif"), force=true)
    rm(in_datafolder("timezones.tif"), force=true)
    rm(in_datafolder("timezone_names.csv"), force=true)
    if cleanup == :all
        rm(in_datafolder("Landcover - USGS MODIS.tif"), force=true)
        rm(in_datafolder("ETOPO1_Ice_c_geotiff.tif"), force=true)
        rm(in_datafolder("gadm36"), force=true, recursive=true)
        rm(in_datafolder("nuts2016-level3"), force=true, recursive=true)
        rm(in_datafolder("WDPA"), force=true, recursive=true)
        rm(in_datafolder("timezones-with-oceans.shapefile"), force=true, recursive=true)
    end
end

function rasterize_GADM()
    println("\nRasterizing GADM shapefile for global administrative areas (2-10 minute run time)...")
    shapefile = in_datafolder("gadm36", "gadm36.shp")
    outfile = in_datafolder("gadm.tif")
    options = "-a UID -ot Int32 -tr 0.01 0.01 -te -180 -90 180 90 -co COMPRESS=LZW"
    # options = "-a UID -ot Int32 -tr 0.02 0.02 -te -180 -90 180 90 -co COMPRESS=LZW"
    @time rasterize(shapefile, outfile, split(options, ' '))

    println("Creating .csv file for regional index and name lookup...")
    sql = "select uid,name_0,name_1,name_2 from gadm36"
    # sql = "select uid,id_0,name_0,id_1,name_1,id_2,name_2 from gadm36"
    outfile = in_datafolder("gadmfields.csv")
    ogr2ogr_path() do ogr2ogr
        @time run(`$ogr2ogr -f CSV $outfile -sql $sql $shapefile`)
    end
    nothing
end

function rasterize_NUTS()
    println("\nRasterizing NUTS shapefile for European administrative areas...")
    name = "NUTS_RG_01M_2016_4326_LEVL_3"
    shapefile = in_datafolder("nuts2016-level3", "$name.shp")
    outfile = in_datafolder("nuts.tif")
    options = "-a ROWID -ot Int16 -tr 0.01 0.01 -te -180 -90 180 90 -co COMPRESS=LZW -dialect SQlite"
    sql = "select ROWID+1 AS ROWID,* from $name"
    @time rasterize(shapefile, outfile, split(options, ' '), sql=sql)

    println("Creating .csv file for regional index and name lookup...")
    outfile = in_datafolder("nutsfields.csv")
    sql = "select ROWID+1 AS ROWID,* from $name"
    ogr2ogr_path() do ogr2ogr
        @time run(`$ogr2ogr -f CSV $outfile -dialect SQlite -sql $sql $shapefile`)
    end
    nothing
end

function read_gadm()
    println("Reading GADM rasters...")
    gadmfields = readdlm(in_datafolder("gadmfields.csv"), ',', header=true)[1]
    imax = maximum(gadmfields[:,1])
    subregionnames = fill("", (imax,3))
    subregionnames[gadmfields[:,1],:] = string.(gadmfields[:,2:4])
    gadm = readraster(in_datafolder("gadm.tif"))
    return gadm, subregionnames
end

function read_nuts()
    println("Reading NUTS rasters...")
    nutsfields = readdlm(in_datafolder("nutsfields.csv"), ',', header=true)[1]
    imax = maximum(nutsfields[:,1])
    subregionnames = nutsfields[:,3]    # indexes of NUTS regions are in order 1:2016, let's use that
    nuts = readraster(in_datafolder("nuts.tif"))
    return nuts, subregionnames
end

function rasterize_protected()
    println("\nRasterizing three WDPA shapefiles for protected areas (total run time 10 minutes - 2 hours)...")

    for i = 0:2
        println("\nFile $(i+1)/3:")
        shapefile = in_datafolder("WDPA", "WDPA-shapefile$i", "WDPA-shapefile-polygons.shp")

        println("Rasterizing...")
        gdal_rasterize_path() do gdal_rasterize
            outfile = in_datafolder("WDPA", "protected_raster$i.tif")
            sql = "select FID from \"WDPA-shapefile-polygons\""
            options = "-a FID -a_nodata -1 -ot Int32 -tr 0.01 0.01 -te -180 -90 180 90 -co COMPRESS=LZW"
            @time run(`$gdal_rasterize $(split(options, ' ')) -sql $sql $shapefile $outfile`)
        end

        println("Creating .csv file for WDPA index and name lookup...")
        ogr2ogr_path() do ogr2ogr
            outfile = in_datafolder("WDPA", "protectedfields$i.csv")
            sql = "select FID,IUCN_CAT from \"WDPA-shapefile-polygons\""
            run(`$ogr2ogr -f CSV $outfile -sql $sql $shapefile`)
        end

        makeprotected(i)
    end

    println("\nMerging the three rasters...")
    protected = max.(
        JLD.load(in_datafolder("WDPA", "protected0.jld"), "protected"),
        JLD.load(in_datafolder("WDPA", "protected1.jld"), "protected"),
        JLD.load(in_datafolder("WDPA", "protected2.jld"), "protected")
    )
    JLD.save(in_datafolder("protected.jld"), "protected", protected, compress=true)
    println("Done.")

    nothing
end

function makeprotected(n)
    println("Reading rasters...")
    protectedfields = readdlm(in_datafolder("WDPA", "protectedfields$n.csv"), ',', header=true)[1]
    IUCNcodes = ["Ia", "Ib", "II", "III", "IV", "V", "VI", "Not Reported", "Not Applicable", "Not Assigned"]
    IUCNlookup = Dict(c => i for (i,c) in enumerate(IUCNcodes))
    protected0 = readraster(in_datafolder("WDPA", "protected_raster$n.tif"))

    println("Converting indexes to protected area types...")
    protected = similar(protected0, UInt8)
    # could replace loop with:  map!(p -> p == -1 ? 0 : IUCNlookup[protectedfields[p+1,2], protected, protected0)
    # alternatively             map!(p -> ifelse(p == -1, 0, IUCNlookup[protectedfields[p+1,2]), protected, protected0)
    @time for (i, p) in enumerate(protected0)
        protected[i] = (p == -1) ? 0 : IUCNlookup[protectedfields[p+1,2]]
    end
    println("Saving...")
    # JLD.save(in_datafolder("protected$n.jld"), "protected", protected, compress=true)
    JLD.save(in_datafolder("WDPA", "protected$n.jld"), "protected", protected)
end

function rasterize_timezones()
    println("Rasterizing shapefile of time zones...")
    shapefile = in_datafolder("timezones-with-oceans.shapefile", "dist", "combined-shapefile-with-oceans.shp")
    sql = "select FID+1 as FID from \"combined-shapefile-with-oceans\""
    outfile = in_datafolder("timezones.tif")
    options = "-a FID -a_nodata 0 -ot Int16 -tr 0.01 0.01 -te -180 -90 180 90 -co COMPRESS=LZW"
    gdal_rasterize_path() do gdal_rasterize
        @time run(`$gdal_rasterize $(split(options, ' ')) -sql $sql $shapefile $outfile`)
    end

    println("Creating .csv file for time zone index and name lookup...")
    sql = "select FID+1 as FID,tzid from \"combined-shapefile-with-oceans\""
    outfile = in_datafolder("timezone_names.csv")
    ogr2ogr_path() do ogr2ogr
        @time run(`$ogr2ogr -f CSV $outfile -sql $sql $shapefile`)
    end
    nothing
end

function maketimezones()
    println("Reading time zone raster file...")
    tznames = string.(readdlm(in_datafolder("timezone_names.csv"), ',', header=true)[1][:,2])
    timezones = readraster(in_datafolder("timezones.tif"))

    f0 = findall(timezones.==0)     # find any "no data" pixels (should only be one pixel in Canada near Halifax)
    for f in f0
        timezones[f] = maximum(timezones[f .+ CartesianIndices((-1:1, -1:1))])  # replace with largest neighbor
    end

    println("Saving time zones dataset...")
    JLD.save(in_datafolder("timezones.jld"), "timezones", timezones, "tznames", tznames, compress=true)
end

function loadtimezones(lonrange, latrange)
    jldopen(in_datafolder("timezones.jld"), "r") do file
        return read(file, "timezones")[lonrange, latrange], read(file, "tznames")
    end
end

function resample(infile::String, outfile::String, options::Vector{<:AbstractString})
    gdal_translate_path() do gdal_translate
        @time run(`$gdal_translate $options -co COMPRESS=LZW $infile $outfile`)
    end
end

function downscale_landcover()
    println("\nDownscaling landcover dataset (2-10 minutes)...")
    infile = in_datafolder("Landcover - USGS MODIS.tif")
    options = "-r mode -ot Byte -tr 0.01 0.01"
    resample(infile, in_datafolder("landcover.tif"), split(options, ' '))
    nothing
end

function savelandcover()
    println("Converting landcover dataset from TIFF to JLD...")
    landcover = readraster(in_datafolder("landcover.tif"))
    landtypes = [
        "Water", "Evergreen Needleleaf Forests", "Evergreen Broadleaf Forests", "Deciduous Needleleaf Forests", "Deciduous Broadleaf Forests",
        "Mixed Forests", "Closed Shrublands", "Open Shrublands", "Woody Savannas", "Savannas", "Grasslands", "Permanent Wetlands",
        "Croplands", "Urban", "Cropland/Natural", "Snow/Ice", "Barren"
    ]
    landcolors = 1/255 * [
        190 247 255; 0 100 0; 77 167 86; 123 204 6; 104 229 104;
        55 200 133; 216 118 118; 255 236 163; 182 231 140; 255 228 18; 255 192 107; 40 136 213;
        255 255 0; 255 0 0; 144 144 0; 255 218 209; 190 190 190;
    ]
    println("Saving landcover dataset...")
    JLD.save(in_datafolder("landcover.jld"), "landcover", landcover, "landtypes", landtypes,
                "landcolors", landcolors, compress=true)
end

function upscale_topography()
    println("\nUpscaling topography dataset...")
    infile = in_datafolder("ETOPO1_Ice_c_geotiff.tif")
    options = "-r cubicspline -tr 0.01 0.01"
    outfile = in_datafolder("topography.tif")
    resample(infile, outfile, split(options, ' '))
    println("Reading new topography raster...")
    topography = readraster(outfile)
    println("Saving topography dataset...")
    JLD.save(in_datafolder("topography.jld"), "topography", topography, compress=true)
end

# gettopography() = readraster("topography.tif")

function downscale_population(scen, year)
    scen = lowercase(scen)
    println("Reading population dataset...")
    dataset = Dataset(in_datafolder("SSP_1km", "$(scen)_total_$year.nc4"))
    pop = replace(dataset["Band1"][:,:], missing => Float32(0))

    lat = dataset["lat"][:]
    res = 0.5/60    # source resolution 0.5 arcminutes

    println("Padding and saving intermediate dataset...")
    skiptop = round(Int, (90-(lat[end]+res/2)) / res)
    skipbottom = round(Int, (lat[1]-res/2-(-90)) / res)
    nlons = size(pop,1)
    # the factor (.01/res)^2 is needed to conserve total population
    pop = [zeros(Float32,nlons,skiptop) reverse(pop, dims=2)*Float32((.01/res)^2) zeros(Float32,nlons,skipbottom)]
    temptiff = "$(tempname()).tif"
    temptiff2 = "$(tempname()).tif"
    saveTIFF(pop, temptiff)

    println("Downscaling population dataset...")
    options = "-r cubicspline -tr 0.01 0.01"
    resample(temptiff, temptiff2, split(options, ' '))
    newpop = readraster(temptiff2)

    println("Saving population dataset...")
    JLD.save(in_datafolder("population_$(scen)_$year.jld"), "population", newpop, compress=true)

    rm(temptiff, force=true)
    rm(temptiff2, force=true)
end

getpopulation(scen, year) = JLD.load(in_datafolder("population_$(scen)_$year.jld"), "population")

function createGDP(scen, year)
    scen = lowercase(scen)
    scennum = scen[end]
    println("Reading low resolution population and GDP datasets...")
    pop, extent = readraster(in_datafolder("global_population_and_gdp", "p$(scennum)_$year.tif"), :getextent) # million people
    gdp = readraster(in_datafolder("global_population_and_gdp", "g$(scennum)_$year.tif"))    # billion USD(2005), PPP

    # Convert to USD 2010 using US consumer price index (CPI-U). CPI-U 2005: 195.3, CPI-U 2010: 218.056
    # https://www.usinflationcalculator.com/inflation/consumer-price-index-and-annual-percent-changes-from-1913-to-2008/
    tempfile = tempname()
    gdp_per_capita = gdp./pop * 218.056/195.3 * 1000    # new unit: USD(2010)/person, PPP
    gdp_per_capita[pop.<=0] .= 0    # non land cells have pop & gdp set to -infinity, set to zero instead
    saveTIFF(gdp_per_capita, tempfile, extent)

    # println("Downscaling to high resolution and saving...")
    # options = "-r average -tr 0.01 0.01"
    # resample(tempfile, "gdp_per_capita_$(scen)_$year.tif", split(options, ' '))
    # rm(tempfile)

    @time gdphigh = downscale_lowres_gdp_per_capita(tempfile, scen, year)     # unit: USD(2010)/grid cell, PPP
    rm(tempfile, force=true)
    println("Saving high resolution GDP...")
    JLD.save(in_datafolder("gdp_$(scen)_$year.jld"), "gdp", gdphigh, compress=true)
end

function downscale_lowres_gdp_per_capita(tempfile, scen, year)
    println("Create high resolution GDP set using high resolution population and low resolution GDP per capita...")
    gpclow, extent = readraster(tempfile, :extend_to_full_globe)
    pop = getpopulation(scen, year)
    gdphigh = similar(pop, Float32)

    nrows, ncols = size(gpclow)
    sizemult = size(pop,1) ÷ nrows

    for c = 1:ncols
        cols = (c-1)*sizemult .+ (1:sizemult)
        for r = 1:nrows
            gpc = gpclow[r,c]
            rows = (r-1)*sizemult .+ (1:sizemult)
            gdphigh[rows,cols] = gpc * pop[rows,cols]
        end
    end
    return gdphigh
end

function creategridaccess(scen, year)
    println("Estimate high resolution grid access dataset by filtering gridded GDP...")
    gdp = JLD.load(in_datafolder("gdp_$(scen)_$year.jld"), "gdp")
    res = 360/size(gdp,1)

    disk = diskfilterkernel(1/6/res)                        # filter radius = 1/6 degrees
    gridaccess = gridsplit(gdp .> 100_000, x -> imfilter(x, disk), Float32)
    # gridaccess = Float32.(imfilter(gdp .> 100_000, disk))   # only "high" income cells included (100 kUSD/cell), cell size = 1x1 km
    println("\nCompressing...")
    selfmap!(x -> ifelse(x<1e-6, 0, x), gridaccess)         # force small values to zero to reduce dataset size
    println("Saving high resolution grid access...")
    JLD.save(in_datafolder("gridaccess_$(scen)_$year.jld"), "gridaccess", gridaccess, compress=true)

    # maybe better:
    # loop through countries, index all pixels into vector, sort by GDP, use electrification to assign grid access
end

function getwindatlas()
    # filename = in_datafolder("gwa3_250_wind-speed_100m.tif") # v3.0 (lon extent [-180.3, 180.3], strangely)
    # filename = in_datafolder("global_ws.tif") # v2.3 (lon extent [-180.3, 180.3], strangely)
    # filename = in_datafolder("Global Wind Atlas v1 - 100m wind speed.tif")   # v1.0
    # windatlas = readraster(filename, :extend_to_full_globe)[1]
    filename = in_datafolder("Global Wind Atlas v3 - 100m wind speed.tif")   # v3.0
    windatlas = readraster(filename)
    clamp!(windatlas, 0, 25)
end

# Convert the Global Wind Atlas 3.0 dataset from 250 m to 1 km resolution.
# This reduces file size from 13 GB to 1 GB. Interpolate using cubic splines.
# Also change its weird lon-lat extents to standard [-180,-90] - [180, 90].
# Lanczos algorithm is supposedly very good at retaining detail while
# reducing image size.
function convert_windatlas3()
    infile = in_datafolder("gwa3_250_wind-speed_100m.tif")
    gdalinfo_path() do gdalinfo
        run(`$gdalinfo $infile`)
    end
    println("\n")
    outfile = in_datafolder("Global Wind Atlas v3 - 100m wind speed.tif")
    options = split("-r lanczos -te -180 -90 180 90 -tr 0.01 0.01", ' ')
    gdalwarp_path() do gdalwarp
        @time run(`$gdalwarp $options -co COMPRESS=LZW $infile $outfile`)
    end
end


const evtypes = String[ "not_existing",
                        "not_reported",
                        "anthropogenic_event",
                        "collapse",
                        "cavity_collapse",
                        "mine_collapse",
                        "building_collapse",
                        "explosion",
                        "accidental_explosion",
                        "chemical_explosion",
                        "controlled_explosion",
                        "experimental_explosion",
                        "industrial_explosion",
                        "mining_explosion",
                        "quarry_blast",
                        "road_cut",
                        "blasting_levee",
                        "nuclear_explosion",
                        "induced_or_triggered_event",
                        "rock_burst",
                        "reservoir_loading",
                        "fluid_injection",
                        "fluid_extraction",
                        "crash",
                        "plane_crash",
                        "train_crash",
                        "boat_crash",
                        "other_event",
                        "atmospheric_event",
                        "sonic_boom",
                        "sonic_blast",
                        "acoustic_noise",
                        "thunder",
                        "avalanche",
                        "snow_avalanche",
                        "debris_avalanche",
                        "hydroacoustic_event",
                        "ice_quake",
                        "slide",
                        "landslide",
                        "rockslide",
                        "meteorite",
                        "volcanic_eruption"]

"""
    randSeisSrc()

Generate a SeisSrc structure filled with random values.
"""
function randSeisSrc(; mw::Float32=0.0f0)
  R = SeisSrc()

#    src:
#     st: dur 0.0, rise 0.0, decay 0.0

  m0 = 1.0e-7*(10^(1.5*((mw == zero(Float32) ? 10^(rand(Float64)-1.0) : Float64(mw))+10.7)))

  setfield!(R, :id, string(rand(1:2^62)))                           # :id
  setfield!(R, :m0, m0)                                             # :m0
  setfield!(R, :gap, rand(Float64)*180.0)                           # :gap
  pop_rand_dict!(getfield(R, :misc), rand(4:24))                    # :misc
  setfield!(R, :mt, m0.*(rand(Float64, 6).-0.5))                    # :mt
  setfield!(R, :dm, m0.*(rand(Float64, 6).-0.5))                    # :dm
  setfield!(R, :npol, rand(Int64(6):Int64(120)))                    # :npol
  [note!(R, randstring(rand(16:256))) for i = 1:rand(1:6)]          # :notes
  setfield!(R, :pax,
    vcat(360.0.*(rand(Float64, 2, 3).-0.5),
          (rand(-1:2:1)*m0).*[rand() -1.0*rand() rand()]))          # :pax
  setfield!(R, :planes, 360.0.*(rand(Float64, 3, 2).-0.5))          # :planes
  setfield!(R, :src, "randSeisSrc")                                 # :src
  # setfield!(R, :src, join([join([randstring(12), " ",
  #                           randstring(20)]) for i=1:5], ","
  #                         ))                                        # :src
  setfield!(R, :st, SourceTime( randstring(2^rand(6:8)),            # :st
                                rand(Float64),
                                rand(Float64),
                                rand(Float64)))
  return R
end

"""
    randSeisHdr()

Generate a SeisHdr structure filled with random values.
"""
function randSeisHdr()
  H = SeisHdr()

  # moment magnitude and m0 in SI units
  mw = Float32(log10(10.0^rand(0.0:0.1:7.0)))
  m0 = 1.0e-7*(10^(1.5*(Float64(mw)+10.7)))

  # a random earthquake location
  dmin = 10.0^(rand()+0.5)
  dmax = dmin + exp(mw)*10.0^rand()

  loc = EQLoc(
           (rand(0.0:1.0:89.0) + rand())*-1.0^rand(1:2),
           (rand(0.0:1.0:179.0)+rand())*-1.0^rand(1:2),
           min(10.0*randexp(Float64), 660.0),
           rand()*4.0,
           rand()*4.0,
           rand()*8.0,
           rand(),
           rand(),
           rand(),
           10.0*10.0^rand(),
           dmin,
           dmax,
           4 + round(Int64, rand()*exp(mw)),
           rand(0x00:0x10:0xf0),
           rand_datum(),
           "hypocenter",
           "",
           rand(["HYPOELLIPSE", "HypoDD", "Velest", "centroid"]))

  # setfield!(loc, :sig, "1σ")
  # event type
  rtyp = rand()
  typ = rtyp > 0.3 ? "earthquake" : rand(evtypes)

  # Generate random header
  setfield!(H, :id, string(rand(1:2^62)))                           # :id
  setfield!(H, :int, (mw < 0.0f0 ? 0x00 : floor(UInt8, mw),
    randstring(rand(2:4))))                                         # :int
  setfield!(H, :loc, loc)                                           # :loc
  setfield!(H, :mag, EQMag( val   = mw,                             # :mag
                            scale = rand(["M","m"]) * randstring(2),
                            nst   = rand(3:100),
                            gap   = 360.0*rand(Float64),
                            src   = randstring(24)
                            )
            )
  # setfield!(H, :mech, FocalMech(src = join([join([randstring(12),     # :mech
  #                                     " ",
  #                                     randstring(20)]) for i=1:5], ","),
  #                               np  = 360.0.*(rand(Float64, 3, 2).-0.5),
  #                               pax = vcat(360.0.*(rand(Float64, 2, 2).-0.5),
  #                                     (rand(-1:2:1)*m0).*[rand() -1.0*rand()]),
  #                               m0  = m0,
  #                               mt  = m0.*(rand(Float64, 6).-0.5),
  #                               dm  = m0.*(rand(Float64, 6).-0.5)
  #                               )
  #         ) # obviously not pure DC, jajaja "random"
  pop_rand_dict!(H.misc, rand(4:24))                                # :misc
  [note!(H, randstring(rand(16:256))) for i = 1:rand(1:6)]          # :notes
  setfield!(H, :ot, now())                                          # :ot
  setfield!(H, :src, "randSeisHdr")                                 # :src
  setfield!(H, :typ, typ)                                           # :typ

  return H
end

"""
    randPhaseCat()

Generate a random seismic phase catalog suitable for testing EventChannel,
EventTraceData, and SeisEvent objects.
"""
function randPhaseCat(n::Int64=0)
  phase_list = String["P", "PKIKKIKP", "PKIKKIKS", "PKIKPPKIKP", "PKPPKP",
  "PKiKP", "PP", "PS", "PcP", "S", "SKIKKIKP", "SKIKKIKS", "SKIKSSKIKS", "SKKS",
  "SKS", "SKiKP", "SP", "SS", "ScS", "pP", "pPKiKP", "pS", "pSKS", "sP",
  "sPKiKP", "sS", "sSKS"]
  pol_list = Char['U', 'D', '-', '+', '_', ' ']

  phase = Array{String,1}(undef, n == 0 ? rand(3:18) : n)
  for j = 1:length(phase)
    phase[j] = rand(phase_list)
  end
  unique!(phase)

  P = PhaseCat()
  for j in phase
    P[j] = SeisPha(rand(), rand(), rand(), rand(), rand(), rand(), rand(), rand(), rand(pol_list), Char(rand(0x30:0x39)))
  end
  return P
end

"""
    randSeisEvent([, c=0.2, s=0.6])

Generate a SeisEvent structure filled with random header and channel data.
* 100*c is the percentage of :data channels _after the first_ with irregularly-sampled data (fs = 0.0)
* 100*s is the percentage of :data channels _after the first_ with guaranteed seismic data.
"""
function randSeisEvent(N::Int64; c::Float64=0.0, s::Float64=1.0, nx::Int64=0)
  V = SeisEvent(hdr=randSeisHdr(),
                source=randSeisSrc(),
                data=convert(EventTraceData, randSeisData(N, c=c, s=s, nx=nx)))

  for i = 1:V.data.n
    setindex!(getfield(getfield(V, :data), :pha), randPhaseCat(), i)
    setindex!(getfield(getfield(V, :data), :az),   180*(rand()-0.5), i)
    setindex!(getfield(getfield(V, :data), :baz),  180*(rand()-0.5), i)
    setindex!(getfield(getfield(V, :data), :dist), 180*rand(),       i)
  end
  return V
end
randSeisEvent(; c::Float64=0.0, s::Float64=1.0, nx::Int64=0) = randSeisEvent(rand(8:24), c=c, s=s, nx=nx)