#####
##### Sources
#####

eq_tends(pv::AbstractPrognosticVariable, m::LandModel, tt::Source) =
    (m.source_dt[pv]...,)

#####
##### First order fluxes
#####

eq_tends(pv::PV, land::LandModel, tt::Flux{FirstOrder}) where {PV} = (
    eq_tends(pv, land.soil.heat, tt)...,
    eq_tends(pv, land.soil.water, tt)...,
    eq_tends(pv, land.surface, tt)...,
)

eq_tends(::PV, ::AbstractSoilComponentModel, ::Flux{FirstOrder}) where {PV} = ()
eq_tends(::PV, ::NoSurfaceFlowModel, ::Flux{FirstOrder}) where {PV} = ()

eq_tends(::SurfaceWaterHeight, ::OverlandFlowModel, ::Flux{FirstOrder}) =
    (VolumeAdvection(),)

#####
##### Second order fluxes
#####

# Empty by default

eq_tends(pv::PV, ::AbstractSoilComponentModel, ::Flux{SecondOrder}) where {PV} =
    ()
eq_tends(pv::PV, ::OverlandFlowModel, ::Flux{SecondOrder}) where {PV} = ()
eq_tends(pv::PV, ::NoSurfaceFlowModel, ::Flux{SecondOrder}) where {PV} = ()

eq_tends(pv::PV, land::LandModel, tt::Flux{SecondOrder}) where {PV} = (
    eq_tends(pv, land.soil.heat, tt)...,
    eq_tends(pv, land.soil.water, tt)...,
    eq_tends(pv, land.surface, tt)...,
)

eq_tends(
    pv::PV,
    land::LandModel,
    tt::Flux{SecondOrder},
) where {PV <: VolumetricInternalEnergy} =
    (eq_tends(pv, land.soil.heat, land.soil.water, tt)...,)

eq_tends(
    ::VolumetricInternalEnergy,
    ::SoilHeatModel,
    ::SoilWaterModel,
    ::Flux{SecondOrder},
) = (DiffHeatFlux(), DarcyDrivenHeatFlux())

eq_tends(
    ::VolumetricInternalEnergy,
    ::SoilHeatModel,
    ::PrescribedWaterModel,
    ::Flux{SecondOrder},
) = (DiffHeatFlux(),)

eq_tends(::VolumetricLiquidFraction, ::SoilWaterModel, ::Flux{SecondOrder}) =
    (DarcyFlux(),)


# Alternatively, we could use `InteractiveUtils.subtypes` to get a list of it.
threshold_methods = (
    Balanced,
    Entropy,
    Intermodes,
    MinimumError,
    MinimumIntermodes,
    Moments,
    Otsu,
    UnimodalRosin,
    Yen
)

"""
    SingleHistogramThreshold <: AbstractImageBinarizationAlgorithm
    SingleHistogramThreshold(alg::ThresholdAlgorithm; nbins=256)

    binarize([T,] img, f::ThresholdAlgorithm; nbins=256)
    binarize!([out,] img, f::ThresholdAlgorithm; nbins=256)

Binarizes the image `img` using the threshold found by given threshold finding algorithm `alg`.

# Output

Return the binarized image as an `Array{Gray{T}}` of size `size(img)`. If `T` is not specified, it
is inferred from `out` and `img`.

# Arguments

The function argument is described in more detail below.

##  `img::AbstractArray`

The image that needs to be binarized.  The image is automatically converted to `Gray` in order to
construct the requisite graylevel histogram.

## `alg::ThresholdAlgorithm`

`ThresholdAlgorithm` is an Abstract type defined in `ThresholdAlgorithm.jl`, it provides various
threshold finding algorithms:

$(mapreduce(x->"- [`"*string(x)*"`](@ref)", (x,y)->x*"\n"*y, threshold_methods))

For the more detailed explaination and the construction, please refer to each concrete algorithm.
For example, type `?Otsu` in REPL will give you more details on how to use `Otsu` methods.

## `nbins::Integer`

The number of discrete bins that used to build the histogram. A smaller `nbins` could possibly gives
a less noisy, or in other words, a smoother output. The default value is `256`.

# Examples

All the usage follows the same pattern, take `Otsu` as an example:

```julia
using TestImages, ImageBinarization

img = testimage("cameraman")
img_binary = binarize(img, Otsu())
```

It is less convenient, but still, you could also construct a `SingleHistogramThreshold` by yourself:

```julia
using TestImages, ImageBinarization

img = testimage("cameraman")
f = SingleHistogramThreshold(Otsu(), nbins=256)
img_binary = binarize(img, f)
```

"""
struct SingleHistogramThreshold{T} <: AbstractImageBinarizationAlgorithm
    alg::T
    nbins::Int
    function SingleHistogramThreshold(alg::T; nbins::Integer) where T <: ThresholdAlgorithm
        new{T}(alg, nbins)
    end
end

function binarize!(out::GenericGrayImage, img, f::ThresholdAlgorithm, args...; nbins::Integer=256, kwargs...)
    binarize!(out, img, SingleHistogramThreshold(f, nbins=nbins), args...; kwargs...)
end
function binarize!(img::GenericGrayImage, f::ThresholdAlgorithm, args...; nbins::Integer=256, kwargs...)
    binarize!(img, SingleHistogramThreshold(f, nbins=nbins), args...; kwargs...)
end
function binarize(::Type{T}, img, f::ThresholdAlgorithm, args...; nbins::Integer=256, kwargs...) where T
    binarize(T, img, SingleHistogramThreshold(f, nbins=nbins), args...; kwargs...)
end
function binarize(img, f::ThresholdAlgorithm, args...; nbins::Integer=256, kwargs...)
    binarize(ccolor(Gray, eltype(img)), img, SingleHistogramThreshold(f, nbins=nbins), args...; kwargs...)
end
function binarize(img::AbstractArray{T}, f::ThresholdAlgorithm, args...; kwargs...) where T <: Number
    # issue #46: Do not promote Number to Gray{<:Number}
    binarize(T, img, f, args...; kwargs...)
end

function (f::SingleHistogramThreshold)(out::GenericGrayImage, img::GenericGrayImage)
    edges, counts = build_histogram(img, f.nbins)
    t = find_threshold(f.alg, counts[1:end], edges)
    @. out = img > t # here we rely on implicit type conversion to `eltype(out)`
end

function (f::SingleHistogramThreshold)(out::GenericGrayImage, img::AbstractArray{<:Color3})
    # map `img` to grayspace while keeping the storage type
    # TODO: this mapping operation can be done lazily to reduce memory allocation
    GT = base_color_type(eltype(out)){eltype(eltype(img))}
    f(out, GT.(img))
end


export sync, sync!

function get_sync_t(s::Union{String,DateTime}, t::Array{Int64,1})
  isa(s, DateTime) && return floor(Int64, d2u(s)*sμ)
  if s == "first"
    return minimum(t)
  elseif s == "last"
    return maximum(t)
  else
    return floor(Int64, d2u(DateTime(s))*sμ)
  end
end

function get_sync_inds(t::AbstractArray{Int64,1}, Ω::Bool, t₀::Int64, t₁::Int64)
  if Ω == true
    return union(findall(t.<t₀), findall(t.>t₁))
  else
    return findall(t.<t₀)
  end
end

@doc """
sync!(S::GphysData)

Synchronize the start times of all data in S to begin at or after the last
start time in S.

sync!(S, [s=ST, t=ET, v=VV])

Synchronize all data in S to start at `ST` and terminate at `EN` with verbosity level VV.

For regularly-sampled channels, gaps between the specified and true times
are filled with the mean; this isn't possible with irregularly-sampled data.

#### Specifying start time
* s="last": (Default) sync to the last start time of any channel in `S`.
* s="first": sync to the first start time of any channel in `S`.
* A numeric value is treated as an epoch time (`?time` for details).
* A DateTime is treated as a DateTime. (see Dates.DateTime for details.)
* Any string other than "last" or "first" is parsed as a DateTime.

#### Specifying end time (t)
* t="none": (Default) end times are not synchronized.
* t="last": synchronize all channels to end at the last end time in `S`.
* t="first" synchronize to the first end time in `S`.
* numeric, datetime, and non-reserved strings are treated as for `-s`.

Related functions: time, Dates.DateTime, parsetimewin
""" sync!
function sync!(S::GphysData;
                s="last"::Union{String,DateTime},
                t="none"::Union{String,DateTime},
                v::Int64=KW.v,
                )

  # Dekete empty traces
  deleteat!(S, findall([isempty(S.x[i]) for i =1:S.n]))     # delete empty channels
  S.n == 0 && return nothing                                # pointless to continue

  do_end = t=="none" ? false : true

  # Do not edit order of operations -------------------------------------------
  start_times = zeros(Int64, S.n)
  if do_end
    end_times = zeros(Int64, S.n)
  end

  # non-timeseries data
  irr = falses(S.n)
  non_ts = findall(S.fs .== 0)
  irr[non_ts] .= true
  z = zero(Int64)
  for i = 1:S.n
    start_times[i] = S.t[i][1,2]
    if do_end
      end_times[i] = S.t[i][end,2] + (irr[i] ? z : start_times[i] + sum(S.t[i][2:end,2]) +
        round(Int64, (length(S.x[i])-1)/(μs*S.fs[i])))
        # ts data: start time in μs from epoch + sum of time gaps in μs + length of trace in μs
        # non-ts data: time of last sample
    end
  end
  # Do not edit order of operations -------------------------------------------

  # Start and end times
  t_start = get_sync_t(s, start_times)
  sstr = string(u2d(t_start*μs))

  if do_end
    t_end = get_sync_t(t, end_times)
    t_end > t_start || error("No time overlap with given start & end times!")
    tstr = string(u2d(t_end*μs))
    if v > 0
      @info(@sprintf("Synchronizing %.2f seconds of data\n", (t_end - t_start)*μs))
      if v > 1
        @info(string("t_start = ", u2d(t_start*μs)))
        @info(string("t_end = ", tstr))
    end
    elseif v > 0
      @info(string("Synchronizing to start at ", u2d(t_start*μs)))
    end
  else
    t_end = z
  end

  # Loop over non-timeseries data
  dflag = falses(S.n)
  for i in non_ts
    t = view(S.t[i], :, 2)
    Lt = length(t)

    j = get_sync_inds(t, do_end, t_start, t_end)
    Lj = length(j)
    if Lj ≥ Lt
      dflag[i] = true
      continue
    elseif do_end
      note!(S, i, @sprintf("sync!, s = %s, t = %s, removed %i samples (out of time range) from :x", sstr, tstr, Lj))
    else
      note!(S, i, @sprintf("sync! s = %s, t = none, removed %i samples (out of time range) from :x", sstr, Lj))
    end
    ti = collect(1:Lt)
    deleteat!(S.x[i], j)
    deleteat!(ti, j)
    S.t[i] = S.t[i][ti,:]
  end

  # Loop over timeseries data
  for i = 1:S.n
    if irr[i] == false
      sync_str = Array{String,1}(undef,0)
      desc_str = ""

      # truncate X to values within bounds
      t = t_expand(S.t[i], S.fs[i])
      j = get_sync_inds(t, do_end, t_start, t_end)
      deleteat!(S.x[i], j)
      deleteat!(t, j)

      #= length 0 traces _can_ happen with resampled :x where (lx*f_rat) < 0.5
      requries short series of high-frequency data resampled to too low fs =#
      if length(S.x[i]) == 0
        dflag[i] = true
        continue
      end

      if !isempty(j)
        push!(sync_str, string("deleted ", length(j), " samples from :x"))
      end

      S.t[i] = t_collapse(t, S.fs[i])

      # prepend points to time series data that begin late
      T = eltype(S.x[i])
      fμs = S.fs[i]*μs
      dtμ = round(Int64, 1/fμs)
      μ_x = T(mean(S.x[i]))

      if (start_times[i] - t_start) ≥ dtμ
        ni = div(start_times[i]-t_start, dtμ)
        prepend!(S.x[i], ones(T, ni).*μ_x)

        # corrected 2019-02-28
        S.t[i][1,2] = S.t[i][1,2] - ni*dtμ

        push!(sync_str, string("prepended ", ni, " samples to :x."))
      end

      # append points to time series data that end early
      if do_end
        end_times[i] = S.t[i][1,2] +
                        sum(S.t[i][2:end,2]) +
                        round(Int64, length(S.x[i])/fμs)
        if (t_end - end_times[i]) ≥ dtμ
          ni = div(t_end-end_times[i], dtμ)
          append!(S.x[i], ones(T,ni).*μ_x)

          push!(sync_str, string("appended ", ni, " samples to :x."))
        end

        # Correct for length aberration if necessary
        S.t[i][end,1] = length(S.x[i])

        desc_str = string("sync!, s = ", sstr, ", t = ", tstr)
      else
        desc_str = string("sync!, s = ", sstr, ", t = none")
      end

      if length(sync_str) > 0
        desc_str *= string(", ", join(sync_str, ";"))
      end
      note!(S, i, desc_str)
    end
  end
  del_flagged!(S, dflag, "length 0 after sync")
  return nothing
end

function sync!(C::SeisChannel;
                s::DateTime,
                t="none"::Union{String,DateTime},
                v::Int64=KW.v )
  S = SeisData(C)
  sync!(S, s=s, t=t, v=v)
  return nothing
end

@doc (@doc sync!)
function sync(S::GphysData;
                s="last"::Union{String,DateTime},
                t="none"::Union{String,DateTime},
                v::Int64=KW.v )

  T = deepcopy(S)
  sync!(T, s=s, t=t, v=v)
  return T
end

function sync(C::SeisChannel;
                s::DateTime,
                t="none"::Union{String,DateTime},
                v::Int64=KW.v )

  U = deepcopy(C)
  S = SeisData(U)
  sync!(S, s=s, t=t, v=v)
  return S[1]
end


using Documenter
using LearningHorse.NeuralNetwork
makedocs(;
    sitename="LearningHorse",
    pages = [
        "Home" => "index.md",
        "Manual" => [
            "Regression" => "Regression.md",
            "Classification" => "Classification.md",
            "Preprocessing" => "Preprocessing.md",
            "LossFunction" => "LossFunction.md",
            "NeuralNetwork" => "neuralnetwork.md",
        ],
        "日本語版マニュアル" => "Japanese.md"
    ]
)


const MD_TOGGLE = true

using MatrixDepot
using Test
using LinearAlgebra
using SparseArrays

import MatrixDepot: DataError

macro inc(a)
    :(@testset $a begin let n, p, i; include($a) end end)
end

include("clean.jl")
data_save = save_target(data_dir)
user_save = save_target(user_dir)

# that will download the index files if necessary and initialize internal data
MatrixDepot.init()
@testset "MatrixDepot generator tests" begin

    @inc("test_magic.jl")
    @inc("test_cauchy.jl")
    @inc("test_circul.jl")
    @inc("test_hadamard.jl")
    @inc("test_hilb.jl")
    @inc("test_dingdong.jl")
    @inc("test_frank.jl")
    @inc("test_invhilb.jl")
    @inc("test_forsythe.jl")
    @inc("test_grcar.jl")
    @inc("test_triw.jl")
    @inc("test_moler.jl")
    @inc("test_pascal.jl")
    @inc("test_kahan.jl")
    @inc("test_pei.jl")
    @inc("test_vand.jl")
    @inc("test_invol.jl")
    @inc("test_chebspec.jl")
    @inc("test_lotkin.jl")
    @inc("test_clement.jl")
    @inc("test_fiedler.jl")
    @inc("test_minij.jl")
    @inc("test_binomial.jl")
    @inc("test_tridiag.jl")
    @inc("test_lehmer.jl")
    @inc("test_parter.jl")
    @inc("test_chow.jl")
    @inc("test_randcorr.jl")
    @inc("test_poisson.jl")
    @inc("test_neumann.jl")
    @inc("test_rosser.jl")
    @inc("test_sampling.jl")
    @inc("test_wilkinson.jl")
    @inc("test_rando.jl")
    @inc("test_randsvd.jl")
    @inc("test_rohess.jl")
    @inc("test_kms.jl")
    @inc("test_wathen.jl")
    @inc("test_oscillate.jl")
    @inc("test_toeplitz.jl")
    @inc("test_hankel.jl")
    @inc("test_golub.jl")
    @inc("test_companion.jl")
    @inc("test_erdrey.jl")
    @inc("test_gilbert.jl")
    @inc("test_smallworld.jl")
end

@testset "MatrixDepot remote matrix tests" begin
    tests = [
            "regu",
            "include_generator",
            "download",
            "common",
            "number",
            "property",
            ]

    try
        @testset "$t" for t in tests
            tp = joinpath(@__DIR__(), "$(t).jl")
            println("running $(tp) ...")
            include(tp)
            println("finished $(tp)")
        end

        xdir = MatrixDepot.DATA_DIR
        xtmpdir = string(xdir, ".tmp")
        println("mv $xdir $xtmpdir")
        mv(xdir, xtmpdir, force=true)
        MatrixDepot.toggle_remote()
        MatrixDepot.init(ignoredb=true)
        MatrixDepot.update()
        @test mdopen(sp(1)) != nothing
        mv(xtmpdir, xdir, force=true)
    finally
        revert_target(user_save, user_dir)
        revert_target(data_save, data_dir)
    end
end

println("Success in all tests.")


#=

     All methods, functions, and utilities required to work seamlessly with `StructArrays.jl`

=#

using StructArrays

Base.getproperty(x::TwoFloat, s::Symbol) =
    (s == :hi ? getfield(x, 1) :
    (s == :lo ? getfield(x, 2) :
          error("($s) is not a field of `TwoFloat`.")))

Base.propertynames(b::TwoFloat) = (:hi, :lo)

# explicitly give the "schema" of the object to StructArrays

function StructArrays.staticschema(::Type{TwoFloat{T}}) where {T}}
    TwoFloat(::TwoFloat{(:hi, :lo), Base.tuple_type_cons(T, types)}
end

# generate an instance of TwoFloat type
function StructArrays.createinstance(::Type{TwoFloat{T}}, x, args...) where {T}
    TwoFloat(x, T(args)
end


export local_field, inertia_degree, absolute_inertia_degree, absolute_ramification_index,
        eisenstein_extension, unramified_extension

################################################################################
#
#  Show function
#
################################################################################

function show(io::IO, a::LocalField{S, EisensteinLocalField}) where S
  print(io, "Eisenstein extension with defining polynomial ", defining_polynomial(a))
  print(io, " over ", base_field(a))
end

function show(io::IO, a::LocalField{S, UnramifiedLocalField}) where S
  print(io, "Unramified extension with defining polynomial ", defining_polynomial(a))
  print(io, " over ", base_field(a))
end

function show(io::IO, a::LocalField{S, GenericLocalField}) where S
  print(io, "Extension with defining polynomial ", defining_polynomial(a))
  print(io, " over ", base_field(a))
end

################################################################################
#
#  Characteristic
#
################################################################################

characteristic(K::LocalField) = characteristic(base_field(K))
prime(K::LocalField) = prime(base_field(K))

################################################################################
#
#  Type derivation
#
################################################################################

base_field_type(K::LocalField{S, T}) where {S <: FieldElem, T <: LocalFieldParameter} = parent_type(S)
base_field_type(::Type{LocalField{S, T}}) where {S <: FieldElem, T <: LocalFieldParameter} = parent_type(S)

elem_type(K::LocalField{S, T}) where {S <: FieldElem, T <: LocalFieldParameter} = LocalFieldElem{S, T}
elem_type(::Type{LocalField{S, T}}) where {S <: FieldElem, T <: LocalFieldParameter} = LocalFieldElem{S, T}

dense_matrix_type(K::LocalField{S, T}) where {S <: FieldElem, T <: LocalFieldParameter} =  Generic.MatSpaceElem{LocalFieldElem{S, T}}
dense_matrix_type(::Type{LocalField{S, T}}) where {S <: FieldElem, T <: LocalFieldParameter} =  Generic.MatSpaceElem{LocalFieldElem{S, T}}

dense_poly_type(K::LocalField{S, T}) where {S <: FieldElem, T <: LocalFieldParameter} = Generic.Poly{LocalFieldElem{S, T}}
dense_poly_type(::Type{LocalField{S, T}}) where {S <: FieldElem, T <: LocalFieldParameter} = Generic.Poly{LocalFieldElem{S, T}}

################################################################################
#
#  Basic predicates
#
################################################################################

is_domain_type(::Type{S}) where S <: LocalField = true
is_exact_type(::Type{S}) where S <: LocalField = false
isfinite(K::LocalField) = isfinite(base_field(K))
is_infinite(K::LocalField) = is_infinite(base_field(K))

################################################################################
#
#  Generating polynomials properties
#
################################################################################

function is_eisenstein_polynomial(f::PolyElem{S}) where S <: Union{padic, qadic, LocalFieldElem}
  if !iszero(valuation(leading_coefficient(f)))
    return false
  end
  if !isone(absolute_ramification_index(base_ring(f))*valuation(constant_coefficient(f)))
    return false
  end
  for i = 1:degree(f)-1
    c = coeff(f, i)
    if !iszero(c) && valuation(c) <= 0
      return false
    end
  end
  return true
end

function is_eisenstein_polynomial(f::T, p::S) where {T <: Union{fmpq_poly, fmpz_poly}, S<: Union{fmpz, Int}}
  @assert is_prime(p)
  if !iszero(valuation(leading_coefficient(f), p))
    return false
  end
  if !isone(valuation(constant_coefficient(f), p))
    return false
  end
  for i = 1:degree(f)-1
    c = coeff(f, i)
    if !iszero(c) && valuation(c, p) <= 0
      return false
    end
  end
  return true
end

function is_eisenstein_polynomial(f::PolyElem{<:NumFieldElem}, p::NumFieldOrdIdl)
  @assert is_prime(p)
  if !iszero(valuation(leading_coefficient(f), p))
    return false
  end
  if !isone(valuation(constant_coefficient(f), p))
    return false
  end
  for i = 1:degree(f)-1
    c = coeff(f, i)
    if !iszero(c) && valuation(c, p) <= 0
      return false
    end
  end
  return true
end

function _generates_unramified_extension(f::PolyElem{S}) where S <: Union{padic, qadic, LocalFieldElem}
  K = base_ring(f)
  F, mF = ResidueField(K)
  g = map_coefficients(mF, f)
  return is_irreducible(g)
end

var(a::LocalField) = a.S

function gen(K::LocalField)
  g = gen(parent(defining_polynomial(K)))
  el = K(g)
  return setprecision!(el, precision(K))
end

################################################################################
#
#  Subfields
#
################################################################################

function prime_field(L::Union{FlintQadicField, LocalField})
  L = base_ring(defining_polynomial(L))
  while typeof(L) != FlintPadicField
    L = base_ring(defining_polynomial(L))
  end
  return L
end

function base_field(L::LocalField)
  return base_ring(L.defining_polynomial)
end

function absolute_base_field(L::LocalField)
  return absolute_base_field(base_field(L))
end

absolute_base_field(L::FlintPadicField) = L
absolute_base_field(L::FlintQadicField) = base_field(L)

################################################################################
#
#  Degree
#
################################################################################

function degree(K::LocalField)
  return degree(defining_polynomial(K))
end

function absolute_degree(::FlintPadicField)
  return 1
end

function absolute_degree(K::FlintQadicField)
  return degree(K)
end
function absolute_degree(K::LocalField)
  return degree(K)*absolute_degree(base_field(K))
end

################################################################################
#
#  Traces
#
################################################################################

function assure_traces(K::LocalField{S, T}) where {S <: FieldElem, T <: LocalFieldParameter}
  if isdefined(K, :traces_basis)
    return nothing
  end
  res = S[base_field(K)(degree(K))]
  append!(res, polynomial_to_power_sums(defining_polynomial(K), degree(K)-1))
  K.traces_basis = res
  return nothing
end

################################################################################
#
#  Ramification index
#
################################################################################

function ramification_index(K::FlintPadicField)
  return 1
end

function ramification_index(K::FlintQadicField)
  return 1
end

function ramification_index(K::LocalField{S, EisensteinLocalField}) where S <: FieldElem
  return degree(K)
end

function ramification_index(K::LocalField{S, UnramifiedLocalField}) where S <: FieldElem
  return 1
end

function ramification_index(K::LocalField{S, GenericLocalField}) where S <: FieldElem
  error("Not yet implemented")
end

absolute_ramification_index(K::PadicField) = 1
absolute_ramification_index(K::QadicField) = 1

function absolute_ramification_index(K::LocalField{S, T}) where {S <: FieldElem, T <: LocalFieldParameter}
  return ramification_index(K)*absolute_ramification_index(base_field(K))
end

function ramification_index(L::LocalField, K::LocalField)
  if base_field(L) === K
    return ramification_index(L)
  else
    return ramification_index(L)*ramification_index(base_field(L), K)
  end
end

################################################################################
#
#  Inertia degree
#
################################################################################

function inertia_degree(K::FlintPadicField)
  return 1
end

function inertia_degree(K::FlintQadicField)
  return degree(K)
end

function inertia_degree(K::LocalField{S, EisensteinLocalField}) where S
  return 1
end

function inertia_degree(K::LocalField{S, UnramifiedLocalField}) where S
  return degree(K)
end

function inertia_degree(L::LocalField, K::LocalField)
  if base_field(L) === K
    return inertia_degree(L)
  else
    return inertia_degree(L)*inertia_degree(base_field(L), K)
  end
end

function absolute_inertia_degree(L::LocalField)
  return inertia_degree(L)*absolute_inertia_degree(base_field(L))
end

absolute_inertia_degree(::PadicField) = 1
absolute_inertia_degree(K::QadicField) = degree(K)

################################################################################
#
#  Basis
#
################################################################################

function basis(K::Union{FlintQadicField, LocalField})
  return powers(gen(K), degree(K)-1)
end

function absolute_basis(K::LocalField)
  Bk = absolute_basis(base_field(K))
  BKr = basis(K)
  BK = Vector{elem_type(K)}()
  for i = 1:length(BKr)
    for j = 1:length(Bk)
      push!(BK, BKr[i]*K(Bk[j]))
    end
  end
  return BK
end

absolute_basis(K::FlintQadicField) = basis(K)
absolute_basis(K::FlintPadicField) = padic[one(K)]

################################################################################
#
#  Constructors
#
################################################################################

#=
function find_irreducible_polynomial(K, n::Int)
  Zx, x = PolynomialRing(FlintZZ, "x", cached = false)
  f = cyclotomic(ppio(degree(K), n)*n, x)
  lf = factor(f, K)
  return first(keys(lf[1]))
end

function unramified_extension(L::LocalField, n::Int, prec::Int, s::String = "z")
  K, mK = ResidueField(L)
  f = find_irreducible_polynomial(K, n)
  coeffs =
  return local
end
=#

function eisenstein_extension(f::Generic.Poly{S}, s::String = "a"; check::Bool = true, cached::Bool = true) where S
  return local_field(f, s, EisensteinLocalField, check = check, cached = cached)
end

function unramified_extension(f::Generic.Poly{S}, s::String = "a"; check::Bool = true, cached::Bool = true) where S
  return local_field(f, s, UnramifiedLocalField, check = check, cached = cached)
end

function local_field(f::Generic.Poly{S},::Type{T}; check::Bool = true, cached::Bool = true) where {S <: FieldElem, T <: LocalFieldParameter}
  return local_field(f, "a", T, check = check, cached = cached)
end

function local_field(f::Generic.Poly{S}, s::String, ::Type{EisensteinLocalField}; check::Bool = true, cached::Bool = true) where {S <: FieldElem}
  symb = Symbol(s)
  if check && !is_eisenstein_polynomial(f)
    error("Defining polynomial is not Eisenstein")
  end
  K = LocalField{S, EisensteinLocalField}(f, symb)
  return K, gen(K)
end

function local_field(f::Generic.Poly{S}, s::String, ::Type{UnramifiedLocalField}; check::Bool = true, cached::Bool = true) where {S <: FieldElem}
  symb = Symbol(s)
  if check && !_generates_unramified_extension(f)
    error("Defining polynomial is not irreducible over the residue field!")
  end
  K = LocalField{S, UnramifiedLocalField}(f, symb)
  return K, gen(K)
end

function local_field(f::Generic.Poly{S}, s::String, ::Type{T} = GenericLocalField; check::Bool = true, cached::Bool = true) where {S <: FieldElem, T <: LocalFieldParameter}
  symb = Symbol(s)
  if check && !is_irreducible(f)
    error("Defining polynomial is not irreducible")
  end
  K = LocalField{S, T}(f, symb)
  return K, gen(K)
end

function local_field(f::fmpq_poly, p::Int, precision::Int, s::String, ::Type{T} = GenericLocalField; check::Bool = true, cached::Bool = true) where T <: LocalFieldParameter
  @assert is_prime(p)
  K = PadicField(p, precision)
  fK = map_coefficients(K, f)
  return local_field(fK, s, T, cached = cached, check = check)
end

function defining_polynomial(K::LocalField)
  return K.defining_polynomial
end

function precision(K::LocalField)
  return precision(defining_polynomial(K))*ramification_index(K)
end

function setprecision!(K::LocalField, n::Int)
  K.defining_polynomial = setprecision(defining_polynomial(K), n)
  return nothing
end

################################################################################
#
#  Uniformizer
#
################################################################################

function uniformizer(K::LocalField{S, EisensteinLocalField}) where S <: FieldElem
  return gen(K)
end

function uniformizer(K::LocalField{S, UnramifiedLocalField}) where S <: FieldElem
  return K(uniformizer(base_field(K)))
end

function uniformizer(K::LocalField{S, GenericLocalField}) where S <: FieldElem
  error("Not yet implemented")
end

################################################################################
#
#  ResidueField
#
################################################################################

function ResidueField(K::LocalField{S, EisensteinLocalField}) where {S <: FieldElem}
  if isdefined(K, :residue_field_map)
    mp = K.residue_field_map
    return codomain(mp), mp
  end
  k = base_field(K)
  ks, mks = ResidueField(k)

  function proj(a::LocalFieldElem)
    @assert parent(a) === K
    for i = 1:degree(a.data)
      if valuation(coeff(a, i)) < 0
        error("The projection is not well defined!")
      end
    end
    return mks(coeff(a, 0))
  end

  function lift(a)
    @assert parent(a) === ks
    return setprecision(K(mks\(a)), precision(K))
  end
  mp = MapFromFunc(proj, lift, K, ks)

  K.residue_field_map = mp

  return ks, mp
end

 ########### Residue field of unramified local field ext ################
function ResidueField(K::LocalField{ S, UnramifiedLocalField}) where {S <: FieldElem}
   if isdefined(K, :residue_field_map)
     mp = K.residue_field_map
     return codomain(mp), mp
   end
   k = base_field(K)
   ks, mks = ResidueField(k)
   Fpt = PolynomialRing(ks, cached = false)[1]
   g = defining_polynomial(K)
   f = Fpt([ks(mks(coeff(g, i))) for i=0:degree(K)])
   kk = FiniteField(f)[1]
   bas = basis(K)
   u = gen(kk)
   function proj(a:: Hecke.LocalFieldElem)
     col = typeof(kk(1))[]
     for i = 0:degree(K)-1
       push!(col, mks(coeff(a,i)) * u^i )
     end
     return sum(col)
   end
   function lift(b:: Hecke.RelFinFieldElem)
     col = typeof(K(1))[]
     for i = 0:degree(kk)-1
       push!(col, K(mks\(coeff(b,i))) * bas[i+1] )
     end
     return sum(col)
   end
   mp = MapFromFunc(proj, lift, K, kk)
   K.residue_field_map = mp
  return kk, mp
end

 ################### unramified extension over local field L of a given degree n ####################

 function unramified_extension(L::Union{FlintPadicField, FlintQadicField, LocalField}, n::Int)
   R, mR = ResidueField(L)
   f = polynomial(R, push!([rand(R) for i = 0:n-1], one(R)))
   while !is_irreducible(f)
     f = polynomial(R, push!([rand(R) for i = 0:n-1], one(R)))
   end
   f_L = polynomial(L, [mR\(coeff(f, i)) for i = 0:degree(f)])
   return unramified_extension(f_L)
 end


using PkgMining
using DataFrames
using Test

@testset "basic" begin
    let df = pkg_mining(), df_owners = combine(groupby(df, :owner), nrow => :pkgs_owned)
        @test nrow(df) > 1
        @test nrow(df_owners) > 1
        @test sum(df_owners.pkgs_owned) == nrow(df)
    end
end


data = CSV.read(abspath(joinpath(@__DIR__, "..", "data", "word_freq_probs.csv")),DataFrame)
const words, freqs, depth_score, safety_score  = data.word, data.frequency, data.depth_score, data.safety_score
const word_freq_dict = Dict(zip(words,freqs))


#module A6

#Use this to print
#=
weave(Pkg.dir("$(pwd())\\A6.jl"),
  informat="script",
  out_path = "$(pwd())\\A6.html",
  doctype = "md2html")
=#
if pwd() ∉ LOAD_PATH
  push!(LOAD_PATH,pwd())
end

using DataFrames, Distributions, StatsBase, GZip, JLD,
  Gadfly, CTModCopy, JuMP, NLopt, HCubature, StaticArrays, CTNumerical

const WORKERS = 8 #seems like logical cores * (3/4) is the magic value
const CLEAN_WORKERS = true

if nworkers() < WORKERS || (nworkers()==1 && WORKERS == 1)
  addprocs(nworkers()==1?WORKERS-nworkers()+1:WORKERS-nworkers())
end

@everywhere using DataFrames, Distributions, StatsBase, GZip, JLD,
  Gadfly, CTModCopy, JuMP, NLopt, HCubature, StaticArrays, CTNumerical

@everywhere if !isdefined(:constDef)
  const constDef = true
  const DATA_PATH = pwd() * "\\data" #path where we store the data
  const OUTPUT_PATH = pwd() * "\\output"
  const ALT_DATE_FORMAT_STR = "yyyymmdd"
  const DATE_FORMAT_STR = "m/d/y"
  const SP500_NAME = "GSPC"
  const TBILL_NAME = "t-bill"
  const TERM_NAME = "termStructure"
  const FOOTER_NAME = "footer.tex" #these are just headers for the tex tables
  const HEADER_NAME = "header.tex"
  const OPTION_STD_NAME = "VIXoptionsStd"
  const OPTION_NAME = "VIXoptions"

  const PlotContainer = Union{Plot,Gadfly.Compose.Context}
  const CSRegSpec = Tuple{Symbol, Symbol, Int}
  const CPRegSpec = Tuple{Symbol, Int}

  const MIN_VALS_FOR_EST = 10
  const Ι = Complex(0.0,1.0)
  const STANDARD_DAYS = [30, 60, 91, 122]
end

#pre-process the s&p500 data
function preProcessSP500()::Void
  dateFormat::String = DATE_FORMAT_STR
  sp500DF::DataFrame = readtable("$DATA_PATH\\$SP500_NAME.csv")

  #fix the dates
  rename!(sp500DF, :Date, :DATE)
  sp500DF[:,:DATE] = ((s::String)->Date(s, dateFormat)::Date).(sp500DF[:,:DATE])::DataVector{Date}
  sort!(sp500DF, cols = [:DATE])
  sp500DF[:,:returns] = similar(sp500DF[:,:Adj_Close])
  sp500DF[2:end,:returns] = log.(sp500DF[2:end,:Adj_Close] ./  sp500DF[1:(end-1),:Adj_Close])

  sp500DF[:,:dayOfQuarter] = zeros(Int,size(sp500DF,1))
  for i::Int ∈ 2:size(sp500DF,1)
    if Dates.firstdayofquarter(sp500DF[i,:DATE]) ≠ Dates.firstdayofquarter(sp500DF[i-1,:DATE])
      sp500DF[i,:dayOfQuarter] = 1
    else
      sp500DF[i,:dayOfQuarter] = sp500DF[i-1,:dayOfQuarter] + 1
    end
  end
  #showcols(sp500DF)

  stream::IOStream = open("$DATA_PATH\\$SP500_NAME.jls", "w")
  serialize(stream, sp500DF)
  close(stream)

  return nothing
end

#a simple routine for making an easy to read dataframe binary
function preProcessTBills()::Void
  dateFormat::String = DATE_FORMAT_STR
  tBillDF::DataFrame = readtable("$DATA_PATH\\$TBILL_NAME.csv")

  #fix the dates
  tBillDF[:,:DATE] = ((s::String)->Date(s, dateFormat)::Date).(tBillDF[:,:DATE])::DataVector{Date}
  sort!(tBillDF, cols = [:DATE])
  tBillDF[:,:TB3MS] .*= .01
  tBillDF[:,:rates]= tBillDF[:,:TB3MS] ./12.0 #(1.0+tBillDF[:,:TB3MS]).^(1.0/12.0) .- 1.0

  stream::IOStream = open("$DATA_PATH\\$TBILL_NAME.jls", "w")
  serialize(stream, tBillDF)
  close(stream)

  return nothing
end

#pre-process standard options data
function preProcessVIXOptionsStd(;verbose::Bool = false)::Void
  dateFormat::String = ALT_DATE_FORMAT_STR
  stdDF::DataFrame = readtable("$DATA_PATH\\$OPTION_STD_NAME.csv")

  stdDays::SVector{length(STANDARD_DAYS), Int} = SVector{length(STANDARD_DAYS), Int}(STANDARD_DAYS)

  #adjust date type
  stdDF[:, :date] = ((i::Int)->Date("$i", dateFormat)::Date).(stdDF[:,:date])::DataVector{Date}
  rename!(stdDF, :date, :DATE)

  stdDF = stdDF[completecases(stdDF[:,[:DATE, :days, :forward_price, :delta, :cp_flag]]),:]

  if verbose
    println("total record")
    showcols(stdDF)

    for d::Int ∈ unique(stdDF[:,:days])
      println("records for $d")
      showcols(stdDF[stdDF[:,:days].==d,:])
    end
  end

  maximumTau::Int = maximum(STANDARD_DAYS)
  stdDF = stdDF[stdDF[:,:days].≤maximumTau,:]

  #make this of type char
  delete!(stdDF, [:secid, :ticker, :index_flag])
  stdDF[:, :cp_flag] = ((s::String)->s[1]).(stdDF[:,:cp_flag])::DataVector{Char}
    sort!(stdDF, cols = [:DATE, :cp_flag, :days])

  #showcols(stdDF)

  stream::IOStream = open("$DATA_PATH\\$OPTION_STD_NAME.jls", "w")
  serialize(stream, stdDF)
  close(stream)

  return nothing

end

#pre-process standard options data
function preProcessVIXOptions()::Void
  #constants that will be used later
  dateFormat::String = ALT_DATE_FORMAT_STR
  optionDF::DataFrame = readtable("$DATA_PATH\\$OPTION_NAME.csv")

  #adjust date type
  optionDF[:, :date] = ((i::Int)->Date("$i", dateFormat)::Date).(optionDF[:,:date])::DataVector{Date}
  rename!(optionDF, :date, :DATE)

  optionDF[:, :exdate] = ((i::Int)->Date("$i", dateFormat)::Date).(optionDF[:,:exdate])::DataVector{Date}
  rename!(optionDF, :exdate, :exDATE)

  maximumTau::Int = maximum(STANDARD_DAYS)
  optionDF[:,:days] = (Dates.value).(optionDF[:, :exDATE])  .- (Dates.value).(optionDF[:, :DATE])
  optionDF = optionDF[(!isna).(optionDF[:,:days]), :]
  optionDF = optionDF[optionDF[:,:days] .≤ maximumTau, :]
  optionDF[:, :τ] = optionDF[:,:days] ./ 365.0

  #sort and re-scale
  optionDF[:, :strike_price] = optionDF[:, :strike_price] ./ 1000.0
  sort!(optionDF, cols = [:DATE, :exDATE, :cp_flag, :strike_price])

  optionDF[:, :price] = (optionDF[:, :best_offer] .+ optionDF[:, :best_bid]) ./ 2.0

  #filter out bad data
  optionDF[:, :cp_flag] = ((s::String)->s[1]).(optionDF[:,:cp_flag])::DataVector{Char}
  optionDF = optionDF[completecases(optionDF[:,[:price, :delta, :cp_flag]]),:]
  delete!(optionDF, [:best_offer, :best_bid, :secid, :ticker, :index_flag, :issuer, :exercise_style])

  #showcols(optionDF)

  stream::IOStream = open("$DATA_PATH\\$OPTION_NAME.jls", "w")
  serialize(stream, optionDF)
  close(stream)

  return nothing

end

#this gets a mapping of a given day to a standard day, and the weight for interpolation
function getInterpolationDayIndex()::Tuple{Vector{Int}, Vector{Float64}}
  standardDays::Vector{Int} = STANDARD_DAYS
  maxStandardDays = maximum(standardDays)

  #allocate the return containers, which include the mapping and the weights
  standardDayIndex::Vector{Int} = Vector{Int}(maximum(standardDays))
  weights::Vector{Float64} = Vector{Float64}(maxStandardDays)

  #iterate through all of the standardized days and map them to a standard day
  standardCtr::Int = 1
  for i::Int = 1:maxStandardDays
    if i>standardDays[standardCtr]
      standardCtr += 1
    end
    standardDayIndex[i] = standardCtr

    #calculate the weight
    weights[i] = standardCtr==1? 0.0 :
      (i-standardDays[standardCtr-1]) / (standardDays[standardCtr] - standardDays[standardCtr-1])

  end

  return (standardDayIndex, weights)
end

#merge the optionDF and stdDF
function mergeOptionDFs!(optionDF::DataFrame, stdDF::DataFrame)::DataFrame

  #calculate information about the standard time periods
  standardDays::Vector{Int} = STANDARD_DAYS
  minStandardDays = minimum(standardDays)
  maxStandardDays = maximum(standardDays)
  standardDayIndex::Vector{Int}, weights::Vector{Float64} = getInterpolationDayIndex();

  #index up the standard maturities (will help with merge)
  Dateτ::DataVector{Tuple{Date, Int, Char}} =
    ((d::Date, f::Int, c::Char)->(d,f,c)).(stdDF[:,:DATE],stdDF[:,:days], stdDF[:,:cp_flag])

  stdDict::Dict = Dict(Dateτ[i]=>i for i::Int ∈ 1:size(stdDF,1))
  FStd::DataVector{Float64} =  stdDF[:, :forward_price]

  #for each date and ex date pair
  N::Int =size(optionDF,1)

  optionDF[:,:F] = DataVector{Float64}(Vector{Float64}(N))
  optionDF[:,:F] .= NA
  optionDF[:,:invalid] = false

  #interpolate the forward price
  by(optionDF, :DATE) do dateDF::SubDataFrame
    by(dateDF, :exDATE) do exDateDF::SubDataFrame
      by(exDateDF, :cp_flag) do subDF::SubDataFrame

        period::Int = min(subDF[1,:days], maxStandardDays)
        current::Date = subDF[1,:DATE]
        cp::Char = subDF[1,:cp_flag]

        #now calculate the interpolated value
        if !haskey(stdDict, (current, standardDays[standardDayIndex[period]], cp))
          subDF[:,:invalid] = true
        elseif period ∈ standardDays #if no need for interpolation
          subDF[:,:F] = FStd[stdDict[(current, period, cp)]]
        elseif standardDayIndex[period] == 1
          subDF[:,:F] = FStd[stdDict[(current, standardDays[1], cp)]]
        else
          #take the weighted average of the nearest values
          subDF[:,:F] =
            FStd[stdDict[(current, standardDays[standardDayIndex[period]], cp)]] * weights[period] +
            FStd[stdDict[(current, standardDays[standardDayIndex[period]-1], cp)]] * (1.0-weights[period])
        end
      end
    end
  end

  optionDF = optionDF[(!isna).(optionDF[:,:F]),:]
  optionDF = optionDF[!optionDF[:,:invalid], :]
  delete!(optionDF, :invalid)
  N = size(optionDF,1)

  optionDF[:, :M] = optionDF[:, :F] ./ optionDF[:, :strike_price]

  #now get the high/low metric for the VIX

  medianST::Float64 = median(stdDF[stdDF[:,:days].==minStandardDays,:forward_price])
  medianLT::Float64 = median(stdDF[stdDF[:,:days].==maxStandardDays,:forward_price])

  optionDF[:,:LT_VIX] = Vector{Symbol}(N)
  optionDF[:,:LT_VIX] = NA

  optionDF[:,:ST_VIX] = Vector{Symbol}(N)
  optionDF[:,:ST_VIX] = NA

  by(optionDF, :DATE) do dateDF::SubDataFrame
    by(dateDF, :cp_flag) do subDF::SubDataFrame
      #appropriate standard indices

      #create a view for calculating if the VIX is high or low
      stdSubDF =
        view(stdDF, (stdDF[:,:DATE] .== subDF[1,:DATE]) .& (stdDF[:,:cp_flag] .== subDF[1,:cp_flag]))

      #compute if high or low
      maxDays::Int = maximum(stdSubDF[:,:days])
      minDays::Int = minimum(stdSubDF[:,:days])
      LTVIX::Float64 = (stdSubDF[stdSubDF[:,:days].==maxDays,:forward_price])[1]
      STVIX::Float64 = (stdSubDF[stdSubDF[:,:days].==minDays,:forward_price])[1]

      #update the dataframe
      subDF[:,:LT_VIX] = LTVIX>medianLT?(:H):(:L)
      subDF[:,:ST_VIX] = STVIX>medianLT?(:H):(:L)

    end
  end

  return optionDF
end

#this function merges in the risk-free rate
function mergeOptionDFρ!(optionDF::DataFrame, tBillDF::DataFrame)::DataFrame

  rates::Vector{Float64} = tBillDF[:,:TB3MS]
  rateDates::DataVector{Date} = tBillDF[:,:DATE]
  rateDict::Dict = Dict(rateDates[i]=>rates[i] for i::Int ∈ 1:length(rates))

  #allocate the interest rate column
  optionDF[:, :ρ] = DataVector{Float64}(Vector{Float64}(size(optionDF,1)))
  optionDF[:, :ρ] .= NA
  #for each date, look up the interest rate
  by(optionDF, :DATE) do subDF
    dt::Date = Dates.firstdayofmonth(subDF[1, :DATE])
    if haskey(rateDict, dt)
      subDF[:, :ρ] = rateDict[dt]
    end
  end

  #get rid of invalid interest rate data
  optionDF = optionDF[(!isna).(optionDF[:,:ρ]), :]
  optionDF[:,:ρ] = (log).(1.0 .+ optionDF[:,:ρ])

  return optionDF

end

#convert all puts to calls
function conformToCalls!(optionDF::DataFrame)::DataFrame
  putDF::SubDataFrame = view(optionDF, optionDF[:,:cp_flag].=='P')

  #use put call parity to convert all puts to calls
  putDF[:, :price] += ((exp).(-1.0 .* putDF[:,:τ] .* putDF[:,:ρ])) .*
    (putDF[:,:F] .- putDF[:,:strike_price])

  delete!(optionDF, :cp_flag)

  return optionDF
end

#black model with no dividends
@everywhere function getImpliedBlackσ(price::Float64, F::Float64, τ::Float64, X::Float64,
  r::Float64, startσ::Float64 = 0.15)::Float64
  #Algorithms: :LD_LBFGS, :LN_COBYLA, :LN_BOBYQA
  #  :LD_SLSQP, :LD_TNEWTON, :LD_VAR2/:LD_VAR1,
  #  :LD_MMA, :GN_ESCH, :GN_DIRECT, GN_DIRECT_L, :GD_STOGO
  # GN_ISRES, :LD_LBFGS, :LN_SBPLX,
  #GN_CRS2_LM
  #:GN_CRS2_LM***, :GN_ISRES*** (slow), :LD_MMA**, LN_SBPLX**, LD_TNEWTON** best:62,848

  B::Float64 = exp(-1.0*r*τ)

  function obj(σ::Real)::Real
    d1 = (log(F/X) + τ*(σ^2/2.0))/ (σ*τ^0.5)

    calcPrice::Real = (F*cdf(Normal(),d1) - X*cdf(Normal(),d1-σ*τ^0.5))*B
    return (price-calcPrice)*(price-calcPrice)
  end

  #println("test obj: $(obj(0.3))")

  #works well:LN_NELDERMEAD , LN_SBPLX, LD_SLSQP, LN_BOBYQA
  mod = Model(solver=NLoptSolver(algorithm=:LD_SLSQP, maxtime=1))
  JuMP.register(mod, :obj, 1, obj, autodiff=true)

  #constrain vol to 0 and 1000%
  @variable(mod, 3.0 >= σ >= 10.^-6.0, start = startσ)

  #set the objective function
  @NLobjective(mod, Min, obj(σ))

  σBest::Float64  = -99.0
  objBest::Float64 = -9999.0

  try
    output = solve(mod)
    σBest  = getvalue(σ)
  catch err
    σBest = -99.0
  end

  if σBest ≤   2.0*10.0 ^ -6.0
    σBest = -99.0
  end

  return σBest
end

#calcualte the IV for all options in the dataframe
function calculateIVs!(optionDF)::DataFrame
  N::Int = size(optionDF,1)

  Cs::Array{Float64} = optionDF[:, :price]
  Fs::Array{Float64} = optionDF[:, :F]
  startσs::Array{Float64} = optionDF[:, :impl_volatility]
  τs::Array{Float64} = optionDF[:, :τ]
  Xs::Array{Float64} = optionDF[:, :strike_price]
  ρs::Array{Float64} = optionDF[:, :ρ]

  #pre-allocate the answer space
  σs::Array{Float64} = Vector{Float64}(N)
  #  σs .= NA

  #=@sync @parallel for i  ∈  1:N
    σs[i] = getImpliedBlackσ(Cs[i], Fs[i], τs[i], Xs[i], ρs[i], startσs[i])
  end=#

  σs .= pmap(getImpliedBlackσ, Cs, Fs, τs, Xs, ρs, startσs)

  optionDF[:,:σ] = DataVector{Float64}(Vector{Float64}(N))
  optionDF[:,:σ] .= ((f::Float64)->f<0.0?NA:f).(σs)
  println("Solution failures: $(length(optionDF[isna.(optionDF[:,:σ]),:σ]))
    out of $(length(optionDF[:,:σ]))")

  return optionDF
end

#this encapsulates the processing of the option DF
function processOptionData!(optionDF::DataFrame, stdDF::DataFrame, tBillDF::DataFrame)
  optionDF = mergeOptionDFs!(optionDF, stdDF) #get the merged data
  optionDF = mergeOptionDFρ!(optionDF, tBillDF) #merge in interest rates
  optionDF = conformToCalls!(optionDF) #convert puts to calls

  println("beginning implied volatility calculation")
  optionDF =  calculateIVs!(optionDF) #calculate the implied volatilities

  #write this out to a file
  stream::IOStream = open("$DATA_PATH\\$OPTION_NAME-merged.jls", "w")
  serialize(stream, optionDF)
  close(stream)
end

#makes the volatility smile graph
function makeSmileGraphs(optionDF::DataFrame; label::Symbol = :full,
    binScaleM::Float64 = 25.0, binScaleDays::Float64 = 0.1)::Void

  ####first smile plot
  plotDF::DataFrame = deepcopy(optionDF)
  delete!(plotDF, [:DATE, :exDATE, :strike_price, :F, :ρ, :impl_volatility, :τ, :price, :ST_VIX, :LT_VIX])
  completecases!(plotDF)

  plotDFNoBin::DataFrame = deepcopy(plotDF)

  plotDF[:,:M] = ((f::Float64)->round(f*binScaleM)/binScaleM).(plotDF[:,:M])
  plotDF[:,:days] = ((i::Int)->Int(round(i*binScaleDays)/binScaleDays)).(plotDF[:,:days])
  plotDF=aggregate(plotDF, [:days, :M], mean)

  #get the graphs
  Gadfly.push_theme(:default)
  plots::Vector{PlotContainer} = Vector{PlotContainer}()
  plotNames::Vector{String} = Vector{String}()

  push!(plotNames, "multiSmile")
  push!(plots, plot(plotDF, x=:M, y=:σ_mean, color=:days,
    Geom.line,
    Guide.ylabel("σ"), Guide.xlabel("F/X"),
    Coord.Cartesian(ymin=0.0,ymax=2.5, xmin=0.5, xmax=2.0),
    Guide.title("Volatility Smile  by Time and Moneyness")))

  push!(plotNames, "multiSmileDot")
  push!(plots, plot(plotDF, x=:M, y=:σ_mean, color=:days, shape=:days,
    Geom.point,
    Guide.ylabel("σ"), Guide.xlabel("F/X"),
    Coord.Cartesian(ymin=0.0,ymax=2.5, xmin=0.5, xmax=2.0),
    Guide.title("Volatility Smile  by Time and Moneyness")))

  for i ∈ 1:length(plots)
    draw(SVG("$OUTPUT_PATH\\$(plotNames[i])_$label.svg", 4inch, 4inch),plots[i])
  end

  return nothing
end

function getDF(pathStr::String, refreshData::Bool, preProcessFunc::Function)::DataFrame

  #re-process the data if needed or desired
  if !isfile(pathStr) || refreshData
    preProcessFunc()
  end

  #read the binary file
  stream::IOStream = open(pathStr)
  DF::DataFrame = deserialize(stream)
  close(stream)

  return DF
end

function makeAllSmileGraphs(optionDF::DataFrame)
  makeSmileGraphs(optionDF, label = :full_smooth, binScaleM = 10.0, binScaleDays=0.05)
  makeSmileGraphs(optionDF, label = :full_fine, binScaleM = 1000.0, binScaleDays=0.025)

  makeSmileGraphs(optionDF[(optionDF[:,:ST_VIX] .== :H) .& (optionDF[:,:LT_VIX] .== :H), :],
    label = :HH, binScaleM = 10.0, binScaleDays=0.05)
  makeSmileGraphs(optionDF[(optionDF[:,:ST_VIX] .== :L) .& (optionDF[:,:LT_VIX] .== :H), :],
    label = :LH, binScaleM = 10.0, binScaleDays=0.05)
  makeSmileGraphs(optionDF[(optionDF[:,:ST_VIX] .== :H) .& (optionDF[:,:LT_VIX] .== :L), :],
    label = :HL, binScaleM = 10.0, binScaleDays=0.05)
  makeSmileGraphs(optionDF[(optionDF[:,:ST_VIX] .== :L) .& (optionDF[:,:LT_VIX] .== :L), :],
    label = :LL, binScaleM = 10.0, binScaleDays=0.05)
end

#script for getting and processing the data
function A6Script(;refreshData::Bool=true, reprocessOptions=true)::Void

  #read the binary file
  tBillDF::DataFrame = getDF("$DATA_PATH\\$TBILL_NAME.jls", refreshData, preProcessTBills)
  sp500DF::DataFrame = getDF("$DATA_PATH\\$SP500_NAME.jls", refreshData, preProcessSP500)
  stdDF::DataFrame = getDF("$DATA_PATH\\$OPTION_STD_NAME.jls", refreshData, preProcessVIXOptionsStd)
  optionDF::DataFrame = getDF("$DATA_PATH\\$OPTION_NAME.jls", refreshData, preProcessVIXOptions)

  #we need to create an anonymous function to use the getDF interface
  #mergeOptionDFs!(deepcopy(optionDF),deepcopy(stdDF))
  optionDF = getDF("$DATA_PATH\\$OPTION_NAME-merged.jls", reprocessOptions,
    ()->processOptionData!(optionDF, stdDF,tBillDF))

  makeAllSmileGraphs(optionDF)

  #print the summary stats
  #preProcessVIXOptionsStd(verbose = true)

  #write out the processed option file for testing purposes
  writetable("$OUTPUT_PATH\\$OPTION_NAME.csv", optionDF)
  return nothing
end

function cleanworkers()
  myProcs = procs()
  if CLEAN_WORKERS
    for i in 2:length(myProcs)
      rmprocs(myProcs[i])
    end
  end
end

@time begin
  A6Script(refreshData=false, reprocessOptions=true)
  cleanworkers()
end

#end


# helper functions

"""
  Creates and preallocates an explicit Petsc matrix to be used as the Jacobian.
  Matrices created by this routine require the arguments to MatSetValues to
  be column-major.

  Petsc must already be initialized and have its options set.

  **Inputs**

   * mesh
   * sbp
   * eqn
   * opts:

"""
function createPetscMat(mesh::AbstractMesh, sbp::AbstractOperator,
                        eqn::AbstractSolutionData, opts)

#  const mattype = PETSc2.MATMPIAIJ # should this be BAIJ?
#  const mattype = PETSc2.MATMPIBAIJ # should this be BAIJ?
  const mattype = PETSc2.MATAIJ
#  const mattype = PETSc2.MATBAIJ  # was this
  numDofPerNode = mesh.numDofPerNode

  comm = eqn.comm
  myrank = mesh.myrank
  obj_size = PetscInt(mesh.numDof)  # length of vectors, side length of matrices

  # create the matrix
  A = PetscMat(comm)
  SetFromOptions(A)
  MatSetType(A, mattype)
  MatSetSizes(A, obj_size, obj_size, PETSC_DECIDE, PETSC_DECIDE)
  if mesh.isDG
    MatSetOption(A, PETSc2.MAT_IGNORE_OFF_PROC_ENTRIES, PETSC_TRUE)
  end

#  if mattype == PETSc2.MATMPIAIJ
#    bs = 1
#  else
    bs = PetscInt(mesh.numDofPerNode)  # block size
#  end

  @assert mesh.numDof % bs == 0
  nblocks = div(mesh.numDof, bs)
  dnnz = zeros(PetscInt, nblocks)  # diagonal non zeros per row
  onnz = zeros(PetscInt, nblocks)

  # calculate number of non zeros per row for A
  if bs == 1
    for i=1:mesh.numDof
      dnnz[i] = mesh.sparsity_counts[1, i]
      onnz[i] = mesh.sparsity_counts[2, i]
    end
  else

    disctype = getSparsityPattern(mesh, sbp, eqn, opts)
    face_type = getFaceType(mesh.sbpface)
    _dnnz, _onnz = getBlockSparsityCounts(mesh, mesh.sbpface, disctype, face_type)
    dnnz = convert(Vector{PetscInt}, _dnnz)
    onnz = convert(Vector{PetscInt}, _onnz)
  end  # end if bs == 1

  # preallocate A
  @mpi_master println(BSTDOUT, "preallocating A")

#  if mattype == PETSc2.MATMPIAIJ
#    MatMPIAIJSetPreallocation(A, PetscInt(0),  dnnz, PetscInt(0), onnz)
#  else
    MatXAIJSetPreallocation(A, bs, dnnz, onnz, PetscIntNullArray, PetscIntNullArray)
#  end
  MatZeroEntries(A)

  # Petsc objects if this comes before preallocation
  MatSetOption(A, PETSc2.MAT_ROW_ORIENTED, PETSC_FALSE)

  if opts["addShockCapturing"]
    writeSparsityPattern(mesh, sbp, eqn, opts, A, disctype)
  end

  return A
end

"""
  Create a Petsc shell matrix

  **Inputs**
"""
function createPetscMatShell(mesh::AbstractMesh, sbp::AbstractOperator,
                             eqn::AbstractSolutionData, opts)

  obj_size = PetscInt(mesh.numDof)  # length of vectors, side length of matrices
  comm = eqn.comm

  ctx_ptr = C_NULL # this gets set later
  A = MatCreateShell(comm, obj_size, obj_size, PETSC_DECIDE, PETSC_DECIDE, ctx_ptr)
  SetFromOptions(A)  # necessary?
  SetUp(A)

  return A
end

"""
  Create a parallel Petsc vector of size mesh.numdof

  **Inputs**

   * mesh
   * sbp
   * eqn
   * opts
"""
function createPetscVec(mesh::AbstractMesh, sbp::AbstractOperator,
                        eqn::AbstractSolutionData, opts)

  @assert PetscInitialized()

  comm = eqn.comm
  const vectype = VECMPI
  obj_size = PetscInt(mesh.numDof)  # length of vectors, side length of matrices
  b = PetscVec(comm)
  VecSetType(b, vectype)
  VecSetSizes(b, obj_size, PETSC_DECIDE)

  return b
end

"""
  Create the Petsc PC object.

  Petsc must already be initialized and have its options set.

  **Inputs**

   * mesh
   * sbp
   * eqn
   * opts

"""
function createPetscPC(mesh::AbstractMesh, sbp::AbstractOperator,
                        eqn::AbstractSolutionData, opts)

  @assert PetscInitialized()
  comm = eqn.comm

  pc = PC(comm)
  SetFromOptions(pc)

  return pc
end

"""
  Create the KSP object

  Petsc must already be initialized and have its options set.

  **Inputs**

   * pc: a fully initailized Petsc preconditioner, either [`PetscMatPC`](@ref)
         or [`PetscMatFreePC`](@ref)
   * lo: a fully initialized Petsc linear operator, either [`PetscMatLO`](@ref)
         or [`PetscMatFreeLO`](@ref)
   * comm: the MPI communicator used to defined the pc and lo

"""
function createKSP(pc::Union{PetscMatPC, PetscMatFreePC},
                   lo::Union{PetscMatLO, PetscMatFreeLO}, comm::MPI.Comm)

  ksp = KSP(comm)
  SetFromOptions(ksp)
  KSPSetPC(ksp, pc.pc)

  if typeof(pc) <: PetscMatFreePC
    Ap = lo.A  # Ap is never used, so set it to whatever
  else  # matrix-explicit PC
    Ap = pc.A
  end

  SetOperators(ksp, lo.A, Ap)

  return ksp
end

"""
  Get the integer representing the sbpface type
"""
function getFaceType(sbpface::AbstractFace)
  if typeof(sbpface) <: DenseFace
    face_type = 1
  elseif typeof(sbpface) <: SparseFace
    face_type = 2
  else
    error("unrecogized sbpface type $(typeof(sbpface))")
  end

  return face_type
end

"""
  This function writes all zeros to the matrix.  This should only be used
  immediately after the matrix is created, to write the row and column
  indices to the matrix.  Otherwise use `MatZeroEntries`

  For most cases this is unnecessary, but for shock capturing it is required,
  because the physics module will only write the diffusion term entries to
  the elements of the matrix that have a shock in them.  On the first assemply,
  Petsc will remove any non-written to entries from the matrix structure.
  Thus if the shock moves position after the first Jacobian assembly, there
  won't be enough space in the matrix.  This function solves the problem by
  writing to all the entries (although this result in the Jacobian using more
  memory than is strictly necessary.

  **Inputs**

   * mesh
   * sbp
   * eqn
   * opts
   * A: the Petsc matrix
   * disc_type: the sparsity pattern enum
"""
function writeSparsityPattern(mesh, sbp, eqn, opts, A::PetscMat, disc_type)

  assem = _AssembleElementData(A, mesh, sbp, eqn, opts)

  jac = zeros(mesh.numDofPerNode, mesh.numDofPerNode, mesh.numNodesPerElement,
              mesh.numNodesPerElement)

  # volume terms
  for i=1:mesh.numEl
    assembleElement(assem, mesh, i, jac)
  end

  # interface terms
  for i=1:mesh.numInterfaces
    iface_i = mesh.interfaces[i]
    if disc_type == INVISCID
      assembleInterface(assem, mesh.sbpface, mesh, iface_i, jac, jac, jac, jac)
    elseif disc_type == VISCOUSTIGHT
      assembleInterfaceVisc(assem, mesh.sbpface, mesh, iface_i, jac, jac, jac, jac)
    elseif disc_type == COLORING
      assembleInterfaceFull(assem, mesh, iface_i, jac, jac, jac, jac)
    end  # else disc_type == VISCOUS: don't currently have assembly functions
         # for this, and I'm not sure the sparsity pattern is correct either.
  end

  # shared face terms
  for peer=1:mesh.npeers
    ifaces = mesh.shared_interfaces[peer]
    for i=1:length(ifaces)
      iface_i = ifaces[i]

      if disc_type == INVISCID
        assembleSharedFace(assem, mesh.sbpface, mesh, iface_i, jac, jac)
      elseif disc_type == VISCOUSTIGHT
        assembleSharedFaceVisc(assem, mesh.sbpface, mesh, iface_i, jac, jac)
      elseif disc_type == COLORING
        assembleSharedFaceFull(assem, mesh, iface_i, jac, jac)
      end
    end
  end

  assembly_begin(A, MAT_FINAL_ASSEMBLY)
  assembly_end(A, MAT_FINAL_ASSEMBLY)
  return nothing
end


##############################################################################
##
## table()
##
##############################################################################

function table{T}(d::AbstractDataVector{T})
    counts = Dict{Union(T, NAtype), Int}(0)
    for i = 1:length(d)
        if has(counts, d[i])
            counts[d[i]] += 1
        else
            counts[d[i]] = 1
        end
    end
    return counts
end

##############################################################################
##
## paste()
##
##############################################################################

const letters = convert(Vector{ASCIIString}, split("abcdefghijklmnopqrstuvwxyz", ""))
const LETTERS = convert(Vector{ASCIIString}, split("ABCDEFGHIJKLMNOPQRSTUVWXYZ", ""))

# Like string(s), but preserves Vector{String} and converts
# Vector{Any} to Vector{String}.
_vstring{T <: String}(s::T) = s
_vstring{T <: String}(s::AbstractVector{T}) = s
_vstring(s::AbstractVector) = String[_vstring(x) for x in s]
_vstring(s::Any) = string(s)

function paste(s...)
    s = map(vcat * _vstring, {s...})
    sa = {s...}
    N = max(length, sa)
    res = fill("", N)
    for i in 1:length(sa)
        Ni = length(sa[i])
        k = 1
        for j = 1:N
            res[j] = string(res[j], sa[i][k])
            if k == Ni   # This recycles array elements.
                k = 1
            else
                k += 1
            end
        end
    end
    res
end

function paste_columns(d::AbstractDataFrame, sep)
    res = fill("", nrow(d))
    for j in 1:ncol(d)
        for i in 1:nrow(d)
            res[i] *= string(d[i,j])
            if j != ncol(d)
                res[i] *= sep
            end
        end
    end
    res
end
paste_columns(d::AbstractDataFrame) = paste_columns(d, "_")

##############################################################################
##
## cut()
##
##############################################################################

function cut{S, T}(x::Vector{S}, breaks::Vector{T})
    if !issorted(breaks)
        sort!(breaks)
    end
    min_x, max_x = min(x), max(x)
    if breaks[1] > min_x
        unshift!(breaks, min_x)
    end
    if breaks[end] < max_x
        push!(breaks, max_x)
    end
    refs = fill(POOLED_DATA_VEC_REF_CONVERTER(0), length(x))
    for i in 1:length(x)
        if x[i] == min_x
            refs[i] = 1
        else
            refs[i] = searchsortedfirst(breaks, x[i]) - 1
        end
    end
    n = length(breaks)
    from = map(x -> sprint(showcompact, x), breaks[1:(n - 1)])
    to = map(x -> sprint(showcompact, x), breaks[2:n])
    pool = Array(ASCIIString, n - 1)
    if breaks[1] == min_x
        pool[1] = string("[", from[1], ",", to[1], "]")
    else
        pool[1] = string("(", from[1], ",", to[1], "]")
    end
    for i in 2:(n - 1)
        pool[i] = string("(", from[i], ",", to[i], "]")
    end
    PooledDataArray(refs, pool)
end
cut(x::Vector, ngroups::Int) = cut(x, quantile(x, [1 : ngroups - 1] / ngroups))


#= src/lapack.jl
=#

module _LAPACK
import LinearAlgebra.BLAS: @blasfunc
using LinearAlgebra: BlasFloat, BlasInt, chkstride1
import Base: Nothing
const liblapack = Base.liblapack_name

for (geqrf, gelqf, orgqr, orglq, elty) in
      ((:sgeqrf_, :sgelqf_, :sorgqr_, :sorglq_, :Float32   ),
       (:dgeqrf_, :dgelqf_, :dorgqr_, :dorglq_, :Float64   ),
       (:cgeqrf_, :cgelqf_, :cungqr_, :cunglq_, :ComplexF32 ),
       (:zgeqrf_, :zgelqf_, :zungqr_, :zunglq_, :ComplexF64))
  @eval begin
    function geqrf!(
        A::AbstractMatrix{$elty}, tau::Vector{$elty}, work::Vector{$elty})
      chkstride1(A)
      m, n  = size(A)
      k     = min(m, n)
      tau   = length(tau) < k ? Array{$elty}(undef, k) : tau
      lwork = BlasInt(-1)
      info  = Ref{BlasInt}()
      for i = 1:2
        ccall((@blasfunc($geqrf), liblapack), Nothing,
              (Ref{BlasInt}, Ref{BlasInt}, Ptr{$elty}, Ref{BlasInt},
               Ptr{$elty}, Ptr{$elty}, Ref{BlasInt}, Ptr{BlasInt}),
              m, n, A, max(1,stride(A,2)),
              tau, work, lwork, info)
        if i == 1
          lwork = BlasInt(real(work[1]))
          work  = length(work) < lwork ? Array{$elty}(undef, lwork) : work
          lwork = length(work)
        end
      end
      A, tau, work
    end

    function gelqf!(
        A::AbstractMatrix{$elty}, tau::Vector{$elty}, work::Vector{$elty})
      chkstride1(A)
      m, n  = size(A)
      k     = min(m, n)
      tau   = length(tau) < k ? Array{$elty}(undef, k) : tau
      lwork = BlasInt(-1)
      info  = Ref{BlasInt}()
      for i = 1:2
        ccall((@blasfunc($gelqf), liblapack), Nothing,
              (Ref{BlasInt}, Ref{BlasInt}, Ptr{$elty}, Ref{BlasInt},
               Ptr{$elty}, Ptr{$elty}, Ref{BlasInt}, Ptr{BlasInt}),
              m, n, A, max(1,stride(A,2)),
              tau, work, lwork, info)
        if i == 1
            lwork = BlasInt(real(work[1]))
            work  = length(work) < lwork ? Array{$elty}(undef, lwork) : work
            lwork = length(work)
        end
      end
      A, tau, work
    end

    function orglq!(
        A::AbstractMatrix{$elty}, tau::Vector{$elty}, k::Integer,
        work::Vector{$elty})
      chkstride1(A)
      n = size(A, 2)
      m = min(n, size(A, 1))
      0 <= k <= min(m, length(tau)) || throw(DimensionMismatch)
      lwork = BlasInt(-1)
      info  = Ref{BlasInt}()
      for i = 1:2
          ccall((@blasfunc($orglq), liblapack), Nothing,
                (Ref{BlasInt}, Ref{BlasInt}, Ref{BlasInt}, Ptr{$elty},
                 Ref{BlasInt}, Ptr{$elty}, Ptr{$elty}, Ref{BlasInt},
                 Ptr{BlasInt}),
                m, n, k, A,
                max(1,stride(A,2)), tau, work, lwork,
                info)
          if i == 1
              lwork = BlasInt(real(work[1]))
              work  = length(work) < lwork ? Array{$elty}(undef, lwork) : work
              lwork = length(work)
          end
      end
      A, tau, work
    end

    function orgqr!(
        A::AbstractMatrix{$elty}, tau::Vector{$elty}, k::Integer,
        work::Vector{$elty})
      chkstride1(A)
      m = size(A, 1)
      n = min(m, size(A,2))
      0 <= k <= min(n, length(tau)) || throw(DimensionMismatch)
      lwork = BlasInt(-1)
      info  = Ref{BlasInt}()
      for i = 1:2
        ccall((@blasfunc($orgqr), liblapack), Nothing,
              (Ref{BlasInt}, Ref{BlasInt}, Ref{BlasInt}, Ptr{$elty},
               Ref{BlasInt}, Ptr{$elty}, Ptr{$elty}, Ref{BlasInt},
               Ptr{BlasInt}),
              m, n, k, A,
              max(1,stride(A,2)), tau, work, lwork,
              info)
        if i == 1
          lwork = BlasInt(real(work[1]))
          work  = length(work) < lwork ? Array{$elty}(undef, lwork) : work
          lwork = length(work)
        end
      end
      A, tau, work
    end
  end
end

for (laqps, elty, relty) in ((:slaqps_, :Float32,    :Float32),
                             (:dlaqps_, :Float64,    :Float64),
                             (:claqps_, :ComplexF32,  :Float32),
                             (:zlaqps_, :ComplexF64, :Float64))
  @eval begin
    function laqps!(
        offset::BlasInt, nb::BlasInt, kb::Ref{BlasInt},
        A::AbstractMatrix{$elty},
        jpvt::AbstractVector{BlasInt}, tau::AbstractVector{$elty},
        vn1::AbstractVector{$relty}, vn2::AbstractVector{$relty},
        auxv::AbstractVector{$elty}, F::AbstractVector{$elty})
      m, n = size(A)
      ccall(
        (@blasfunc($laqps), liblapack), Nothing,
        (Ref{BlasInt}, Ref{BlasInt}, Ref{BlasInt}, Ref{BlasInt}, Ptr{BlasInt},
         Ptr{$elty}, Ref{BlasInt}, Ptr{BlasInt}, Ptr{$elty},
         Ptr{$relty}, Ptr{$relty}, Ptr{$elty}, Ptr{$elty}, Ref{BlasInt}),
        m, n, offset, nb, kb,
        A, max(1,stride(A,2)), jpvt, tau,
        vn1, vn2, auxv, F, max(1,n))
    end
  end
end

end  # module


"""
    OutNeighborVector <: AbstractVector

A structure for iterating over the out neighbors in a graph for a certain vertex.
"""
struct OutNeighborVector{V, G <: LG.AbstractGraph{V}} <: AbstractVector{V}
    graph::G
    vertex::V
end

"""
    SimpleEdgeVector <: AbstractVector

A structure for iterating over the edges of a graph
"""
struct SimpleEdgeVector{V, G <: LG.AbstractGraph{V}} <: AbstractVector{LG.Edge{V}}
    graph::G
end


##### directional funcs

export orthog, orthogs

orthogs(::Val{1}) = (2,3)
orthogs(::Val{2}) = (3,1)
orthogs(::Val{3}) = (1,2)
orthogs(i::Int) = orthogs(Val(i))

orthog(::Val{3}, ::Val{2}) = 1
orthog(::Val{2}, ::Val{3}) = 1
orthog(::Val{3}, ::Val{1}) = 2
orthog(::Val{1}, ::Val{3}) = 2
orthog(::Val{1}, ::Val{2}) = 3
orthog(::Val{2}, ::Val{1}) = 3
orthog(i::Int, j::Int) = orthog(Val(i), Val(j))


using StateSpaceReconstruction

@recipe function f(s::T) where {T <: AbstractSimplex}
    seriestype := :path
    label --> ""
    lc --> :black
    splitaxes(connectvertices(s))
end

@recipe function f(simplices::Vararg{T,N}) where {T <: AbstractSimplex, N}
    seriestype := :path
    legend --> false

    for simplex in simplices
        @series begin
            label --> ""
            lc --> :black
            lw --> 0.8
            splitaxes(connectvertices(simplex))
        end
    end

end

@recipe function f(simplices::AbstractVector{T}) where {T <: AbstractSimplex}
    seriestype := :path
    legend --> false

    for simplex in simplices
        @series begin
            label --> ""
            lc --> :black
            lw --> 0.8
            splitaxes(connectvertices(simplex))
        end
    end

end

export f


export DCDPTmpVars

""""
    ensure_diagonal(x)

Create (possibly sparse) matrix which ensures diagonal has all entries (even if zero).
"""
function ensure_diagonal(x)
    n = checksquare(x)
    A = x + SparseMatrixCSC(I, n, n)
    Arows = rowvals(A)
    Avals = nonzeros(A)
    for j = OneTo(n)
       for i in nzrange(A, j)
          j == Arows[i]  &&  (Avals[i] -= 1)
       end
    end
    check_rowvals_equal(A,x) || throw(error("rowvals are not the same..."))
    return A
end

check_rowvals_equal(A::SparseMatrixCSC, X::SparseMatrixCSC) = rowvals(A) == rowvals(X)
check_rowvals_equal(A, X) = true

"Temp vars for dynamic model"
struct DCDPTmpVars{
    T<:Real, SM<:AbstractMatrix{T}, AA3<:AbstractArray3{T}, AA3b<:AbstractArray3{T}, AA4<:AbstractArray4{T}
    } <: AbstractTmpVars
    ubVfull::AA3
    dubVfull::AA4
    dubVfullperm::AA4
    q::AA3b
    lse::Matrix{T}
    tmp::Matrix{T}
    tmp_cart::Matrix{CartesianIndex{3}}
    Πψtmp::Matrix{T}
    IminusTEVp::SM
    tmpEVj::Vector{T}

    ΠsumdubVj::Matrix{T} #  = view(lse(t), :, 1:nk) # Array{T}(nz,nθ)
    dev0tmpj::Matrix{T}  #  = view(tmp(t), :, 1:nk) # Array{T}(nz,nθ)

    function DCDPTmpVars(ubVfull::AA3, dubVfull::AA4, dubVfullperm::AA4, q::AA3b, lse, tmp, tmp_cart, Πψtmp, IminusTEVp::SM, tmpEVj, ΠsumdubVj, dev0tmpj) where {AA3, AA3b, AA4, SM}
        (nθt, nz, nψ, nd) = size(dubVfull)
        (     nz, nψ, nθt, nd) == size(dubVfullperm) || throw(DimensionMismatch())
        (nz, nψ, nd) == size(ubVfull) == size(q) || throw(DimensionMismatch())
        (nz,nψ) == size(lse) == size(tmp) == size(tmp_cart) || throw(DimensionMismatch())
        (nz,nθt) == size(dev0tmpj) == size(ΠsumdubVj) || throw(DimensionMismatch())
        nψ == checksquare(Πψtmp) || throw(DimensionMismatch())
        nz == checksquare(IminusTEVp) == length(tmpEVj)|| throw(DimensionMismatch())
        T = eltype(ubVfull)
        return new{T,SM,AA3,AA3b,AA4}(ubVfull, dubVfull, dubVfullperm, q, lse, tmp, tmp_cart, Πψtmp, IminusTEVp, tmpEVj, ΠsumdubVj, dev0tmpj)
    end
end

const dcdp_tmpvars = DCDPTmpVars
const DCDPTmpVarsArray{T,SM} = DCDPTmpVars{T,SM,Array{T,3},Array{T,3},Array{T,4}}
const DCDPTmpVarsView = DCDPTmpVars{T,SM,AA3} where {T,SM,AA3<:SubArray}

@getFieldFunction DCDPTmpVars ubVfull dubVfull dubVfullperm q lse tmp tmp_cart
@getFieldFunction DCDPTmpVars Πψtmp IminusTEVp tmpEVj ΠsumdubVj dev0tmpj

size(x::DCDPTmpVars) = size(dubVfullperm(x))

ubV(     x::DCDPTmpVars{T,SM,AA3}) where {T,SM,AA3<:SubArray} = ubVfull(x)
dubV(    x::DCDPTmpVars{T,SM,AA3}) where {T,SM,AA3<:SubArray} = dubVfull(x)
dubVperm(x::DCDPTmpVars{T,SM,AA3}) where {T,SM,AA3<:SubArray} = dubVfullperm(x)

function fill!(t::DCDPTmpVars, x)
    fill!(ubVfull(     t), x)
    fill!(dubVfull(    t), x)
    fill!(dubVfullperm(t), x)
    fill!(q(           t), x)
    fill!(lse(         t), x)
    fill!(tmp(         t), x)
    # fill!(tmp_cart(    t), x)
    fill!(Πψtmp(       t), x)
    fill!(tmpEVj(t), x)
end

function DCDPTmpVars(nθt, nz, nψ, nd, ztransition::AbstractMatrix{T}) where {T<:Real}
    dubVfull = Array{T,4}(undef, nθt, nz, nψ,      nd)
    dubVperm = Array{T,4}(undef,      nz, nψ, nθt, nd)
    ubVfull  = Array{T,3}(undef,      nz, nψ,      nd)
    q = similar(ubVfull)
    lse = Matrix{T}(undef, nz, nψ)
    tmp = similar(lse)
    tmp_cart = similar(lse, CartesianIndex{3})
    Πψtmp = Matrix{T}(undef, nψ, nψ)
    IminusTEVp = ensure_diagonal(ztransition)
    tmpEVj = Vector{T}(undef, nz)
    ΠsumdubVj = Matrix{T}(undef, nz, nθt)
    dev0tmpj = similar(ΠsumdubVj)

    return DCDPTmpVars(
        ubVfull, dubVfull, dubVperm, q, lse, tmp, tmp_cart,
        Πψtmp, IminusTEVp, tmpEVj, ΠsumdubVj, dev0tmpj
    )
end

function view(t::DCDPTmpVars, idxd::AbstractVector)
    first(idxd) == 1 || throw(DomainError())
    last(idxd) <= size(ubVfull(t),3) || throw(DomainError())
    @views ubV  = view(ubVfull(t), :,:,  idxd)
    @views dubV = view(dubVfull(t),:,:,:,idxd)
    @views dubvperm = view(dubVfullperm(t),:,:,:,idxd)
    @views qq   = view(q(t), :,:,idxd)
    return dcdp_tmpvars(ubV, dubV, dubvperm, qq, lse(t), tmp(t), tmp_cart(t),
        Πψtmp(t), IminusTEVp(t), tmpEVj(t), ΠsumdubVj(t), dev0tmpj(t))
end

# --------------------------------------------------------
# Fill reward matrices
# --------------------------------------------------------

function update_static_payoffs!(tmpv::DCDPTmpVars, ddm::AbstractDrillModel, θ::AbstractVector, sidx::Integer, ichars::Tuple, dograd::Bool)
    ψspace = psispace(ddm)
    zs = product(zspace(ddm)...)
    zψpdct = product(zs, ψspace)
    nk = _nparm(reward(ddm))
    nc = num_choices(statespace(ddm), sidx)

    sztup = (length(zs), length(ψspace), nk, nc)
    size(tmpv) == sztup || throw(DimensionMismatch("size tmvp = $(size(tmpv)) vs $sztup"))

    dubv  = reshape(dubVfull(tmpv),    nk,  :, nc)
    dubvp = reshape(dubVfullperm(tmpv), :, nk, nc)
    ubv   = reshape( ubVfull(tmpv),     :, nc)

    for dp1 in dp1space(statespace(ddm), sidx)
        @inbounds for (i, (z,ψ)) in enumerate(zψpdct)
            obs = ObservationDrill(ddm, ichars, z, dp1-1, sidx)
            grad = uview(dubv, :, i, dp1)
            ubv[i,dp1] = flow!(grad, obs, θ, ψ, dograd)
        end
    end
    dograd && permutedims!(dubvp, dubv, [2,1,3])
end

@deprecate flow!(t::DCDPTmpVars, ddm::AbstractDrillModel, θ::AbstractVector, sidx::Integer, ichars::Tuple, dograd::Bool) update_static_payoffs!(t,ddm,θ,sidx,ichars,dograd)


import Gmsh: gmsh

function generate_mesh(x0,radius; Lx=80e3, Ly=30e3, H_bdt = 10e3, lowres=0.02, highres=0.001, file)

    @assert length(x0) == length(radius)

    gmsh.initialize()
    gmsh.option.setNumber("General.Terminal", 1)
    gmsh.model.add("model")

    xmin, xmax = 0.0, Lx
    ymin, ymax = 0.0, Ly
    L_bdt = Ly-H_bdt

    p1 = gmsh.model.geo.addPoint(xmin, ymin, 0, lowres)
    p2 = gmsh.model.geo.addPoint(xmax, ymin,  0, lowres)
    p3 = gmsh.model.geo.addPoint(xmax, ymax, 0, lowres)
    p4 = gmsh.model.geo.addPoint(xmin, ymax, 0, lowres)
    pbdt1 = gmsh.model.geo.addPoint(xmin, L_bdt, 0, lowres)
    pbdt2 = gmsh.model.geo.addPoint(xmax, L_bdt, 0, lowres)
    l_bot = gmsh.model.geo.addLine(p1, p2)
    lr = gmsh.model.geo.addLine(p2, p3)
    lr_bot = gmsh.model.geo.addLine(p2, pbdt2)
    lr_top = gmsh.model.geo.addLine(pbdt2,p3)
    l_top = gmsh.model.geo.addLine(p3, p4)
    ll = gmsh.model.geo.addLine(p4, p1)
    ll_top = gmsh.model.geo.addLine(p4, pbdt1)
    ll_bot = gmsh.model.geo.addLine(pbdt1,p1)
    l_bdt1 = gmsh.model.geo.addLine(pbdt1, pbdt2)
    l_bdt2 = gmsh.model.geo.addLine(pbdt2, pbdt1)
    cl_lcrust = gmsh.model.geo.addCurveLoop([l_bot, lr_bot, l_bdt2, ll_bot])
    cl_ucrust = gmsh.model.geo.addCurveLoop([l_bdt1, lr_top, l_top, ll_top])
    #cl_domain = gmsh.model.geo.addCurveLoop([l_bot,lr,l_top,ll])
    # Add two points that will be used as fixed in ux :
    # ptop = gmsh.model.geo.addPoint(0.5, 2, 0, lowres)
    # pbot = gmsh.model.geo.addPoint(0.5, 0, 0, lowres)
    # embed these points in the surface to force the mesh to define nodes at their location.
    # pbot = gmsh.model.geo.addPoint(xmax/2, ymin, 0, lowres)
    # ptop = gmsh.model.geo.addPoint(xmax/2, ymax, 0, lowres)

    #pmid = gmsh.model.geo.addPoint(xmax/2, ymax/2, 0, highres)
    #gmsh.finalize()

    #circ = gmsh.model.occ.addCircle(xmax/2, ymax/2, 0.0, xmax/20)
    T = typeof(p1)
    curved_loops = T[]
    curved_loops_phys = T[]
    surfaces = T[]
    surfaces_phys = T[]
    for i in eachindex(radius)
        x,y = x0[i]
        r = radius[i]
        pmid = gmsh.model.geo.addPoint(x, y, 0, 2highres)
        pc1 = gmsh.model.geo.addPoint(x, y-r, 0, 2highres)
        pc2 = gmsh.model.geo.addPoint(x+r, y, 0, 2highres)
        pc3 = gmsh.model.geo.addPoint(x, y+r, 0, 2highres)
        pc4 = gmsh.model.geo.addPoint(x-r, y, 0, 2highres)
        arc1 = gmsh.model.geo.addCircleArc(pc1,pmid,pc2)
        arc2 = gmsh.model.geo.addCircleArc(pc2,pmid,pc3)
        arc3 = gmsh.model.geo.addCircleArc(pc3,pmid,pc4)
        arc4 = gmsh.model.geo.addCircleArc(pc4,pmid,pc1)
        cl_seed = gmsh.model.geo.addCurveLoop([arc1,arc2,arc3,arc4])
        seed = gmsh.model.geo.addPlaneSurface([cl_seed])

        cl_seed_phys = gmsh.model.addPhysicalGroup(1, [cl_seed])
        seed_phys = gmsh.model.addPhysicalGroup(2, [seed])

        push!(curved_loops, cl_seed)
        push!(surfaces, seed)
        push!(curved_loops_phys, cl_seed_phys)
        push!(surfaces_phys, seed_phys)
    end
    # domain excluding seeds
    upper_crust = gmsh.model.geo.addPlaneSurface([cl_ucrust,curved_loops...])
    upper_crust_phys = gmsh.model.addPhysicalGroup(2, [upper_crust])

    lower_crust = gmsh.model.geo.addPlaneSurface([cl_lcrust])
    lower_crust_phys = gmsh.model.addPhysicalGroup(2, [lower_crust])

    # domain = gmsh.model.geo.addPlaneSurface([cl_domain,curved_loops...])
    # domain_phys = gmsh.model.addPhysicalGroup(2, [domain])

    # gmsh.model.geo.synchronize()
    # gmsh.model.mesh.embed(0, [l_bdt], 2, domain)
    # gmsh.model.mesh.embed(0, [ptop], 1, l3)
    #gmsh.model.mesh.embed(2, [s2], 2, s1)

    bottom = gmsh.model.addPhysicalGroup(1, [l_bot])
    right = gmsh.model.addPhysicalGroup(1, [lr_bot,lr_top])
    top = gmsh.model.addPhysicalGroup(1, [l_top])
    left = gmsh.model.addPhysicalGroup(1, [ll_bot,ll_top])
    #l_bdt = gmsh.model.addPhysicalGroup(1, [l_bdt1])

    #gmsh.model.setPhysicalName(0, clamped, "clamped")
    gmsh.model.setPhysicalName(1, bottom, "bottom")
    gmsh.model.setPhysicalName(1, right, "right")
    gmsh.model.setPhysicalName(1, top, "top")
    gmsh.model.setPhysicalName(1, left, "left")
    #gmsh.model.setPhysicalName(1, l_bdt, "bdt")
    gmsh.model.setPhysicalName(2, lower_crust_phys, "lower_crust")
    gmsh.model.setPhysicalName(2, upper_crust_phys, "upper_crust")
    #gmsh.model.setPhysicalName(2, domain_phys, "domain")
    for i in eachindex(surfaces_phys)
        gmsh.model.setPhysicalName(1, curved_loops_phys[i], "circle_seed_$i")
        gmsh.model.setPhysicalName(2, surfaces_phys[i], "seed_$i")
    end
    gmsh.model.geo.synchronize()
    gmsh.model.mesh.generate(2)
    gmsh.write(file)
    gmsh.finalize()

    return nothing
end

function generate_mesh2(x0,radius; Lx=80e3, Ly=30e3, H_bdt = 10e3, lowres=0.02, highres=0.001, file)

    @assert length(x0) == length(radius)

    gmsh.initialize()
    gmsh.option.setNumber("General.Terminal", 1)
    gmsh.model.add("model")

    xmin, xmax = 0.0, Lx
    ymin, ymax = 0.0, Ly
    L_bdt = Ly-H_bdt

    p1 = gmsh.model.geo.addPoint(xmin, ymin, 0, lowres)
    p2 = gmsh.model.geo.addPoint(xmax, ymin,  0, lowres)
    p3 = gmsh.model.geo.addPoint(xmax, ymax, 0, lowres)
    p4 = gmsh.model.geo.addPoint(xmin, ymax, 0, lowres)
    pbdt1 = gmsh.model.geo.addPoint(xmin, L_bdt, 0, lowres)
    pbdt2 = gmsh.model.geo.addPoint(xmax, L_bdt, 0, lowres)
    l_bot = gmsh.model.geo.addLine(p2,p1)
    lr = gmsh.model.geo.addLine(p2, p3)
    lr_bot = gmsh.model.geo.addLine(pbdt2,p2)
    lr_top = gmsh.model.geo.addLine(pbdt2,p3)
    l_top = gmsh.model.geo.addLine(p3, p4)
    ll = gmsh.model.geo.addLine(p4, p1)
    ll_top = gmsh.model.geo.addLine(p4, pbdt1)
    ll_bot = gmsh.model.geo.addLine(p1, pbdt1)
    l_bdt1 = gmsh.model.geo.addLine(pbdt1, pbdt2)
    l_bdt2 = gmsh.model.geo.addLine(pbdt2, pbdt1)
    # cl_lcrust = gmsh.model.geo.addCurveLoop([l_bdt1, lr_bot, l_bot, ll_bot])
    # cl_ucrust = gmsh.model.geo.addCurveLoop([l_bdt1, lr_top, l_top, ll_top])
    cl_domain = gmsh.model.geo.addCurveLoop([l_bot,lr,l_top,ll])
    # Add two points that will be used as fixed in ux :
    # ptop = gmsh.model.geo.addPoint(0.5, 2, 0, lowres)
    # pbot = gmsh.model.geo.addPoint(0.5, 0, 0, lowres)
    # embed these points in the surface to force the mesh to define nodes at their location.
    # pbot = gmsh.model.geo.addPoint(xmax/2, ymin, 0, lowres)
    # ptop = gmsh.model.geo.addPoint(xmax/2, ymax, 0, lowres)

    #pmid = gmsh.model.geo.addPoint(xmax/2, ymax/2, 0, highres)
    #gmsh.finalize()

    #circ = gmsh.model.occ.addCircle(xmax/2, ymax/2, 0.0, xmax/20)
    T = typeof(p1)
    curved_loops = T[]
    curved_loops_phys = T[]
    surfaces = T[]
    surfaces_phys = T[]
    for i in eachindex(radius)
        x,y = x0[i]
        r = radius[i]
        pmid = gmsh.model.geo.addPoint(x, y, 0, 2highres)
        pc1 = gmsh.model.geo.addPoint(x, y-r, 0, 2highres)
        pc2 = gmsh.model.geo.addPoint(x+r, y, 0, 2highres)
        pc3 = gmsh.model.geo.addPoint(x, y+r, 0, 2highres)
        pc4 = gmsh.model.geo.addPoint(x-r, y, 0, 2highres)
        arc1 = gmsh.model.geo.addCircleArc(pc1,pmid,pc2)
        arc2 = gmsh.model.geo.addCircleArc(pc2,pmid,pc3)
        arc3 = gmsh.model.geo.addCircleArc(pc3,pmid,pc4)
        arc4 = gmsh.model.geo.addCircleArc(pc4,pmid,pc1)
        cl_seed = gmsh.model.geo.addCurveLoop([arc1,arc2,arc3,arc4])
        seed = gmsh.model.geo.addPlaneSurface([cl_seed])

        cl_seed_phys = gmsh.model.addPhysicalGroup(1, [cl_seed])
        seed_phys = gmsh.model.addPhysicalGroup(2, [seed])

        push!(curved_loops, cl_seed)
        push!(surfaces, seed)
        push!(curved_loops_phys, cl_seed_phys)
        push!(surfaces_phys, seed_phys)
    end
    # domain excluding seeds
    upper_crust = gmsh.model.geo.addPlaneSurface([cl_ucrust,curved_loops...])
    upper_crust_phys = gmsh.model.addPhysicalGroup(2, [upper_crust])

    lower_crust = gmsh.model.geo.addPlaneSurface([cl_lcrust])
    lower_crust_phys = gmsh.model.addPhysicalGroup(2, [lower_crust])

    # domain = gmsh.model.geo.addPlaneSurface([cl_domain,curved_loops...])
    # domain_phys = gmsh.model.addPhysicalGroup(2, [domain])

    # gmsh.model.geo.synchronize()
    # gmsh.model.mesh.embed(0, [l_bdt], 2, domain)
    # gmsh.model.mesh.embed(0, [ptop], 1, l3)
    #gmsh.model.mesh.embed(2, [s2], 2, s1)

    bottom = gmsh.model.addPhysicalGroup(1, [l_bot])
    right = gmsh.model.addPhysicalGroup(1, [lr_bot,lr_top])
    top = gmsh.model.addPhysicalGroup(1, [l_top])
    left = gmsh.model.addPhysicalGroup(1, [ll_bot,ll_top])
    #l_bdt = gmsh.model.addPhysicalGroup(1, [l_bdt1])

    #gmsh.model.setPhysicalName(0, clamped, "clamped")
    gmsh.model.setPhysicalName(1, bottom, "bottom")
    gmsh.model.setPhysicalName(1, right, "right")
    gmsh.model.setPhysicalName(1, top, "top")
    gmsh.model.setPhysicalName(1, left, "left")
    #gmsh.model.setPhysicalName(1, l_bdt, "bdt")
    gmsh.model.setPhysicalName(2, lower_crust_phys, "lower_crust")
    gmsh.model.setPhysicalName(2, upper_crust_phys, "upper_crust")
    #gmsh.model.setPhysicalName(2, domain_phys, "domain")
    for i in eachindex(surfaces_phys)
        gmsh.model.setPhysicalName(1, curved_loops_phys[i], "circle_seed_$i")
        gmsh.model.setPhysicalName(2, surfaces_phys[i], "seed_$i")
    end
    gmsh.model.geo.synchronize()
    gmsh.model.mesh.generate(2)
    gmsh.write(file)
    gmsh.finalize()

    return nothing
end

function generate_mesh3(x0,radius; Lx=80e3, Ly=30e3, H_bdt = 10e3, lowres=0.02, highres=0.001, el_type = Triangle, file)

    @assert length(x0) == length(radius)

    gmsh.initialize()
    gmsh.option.setNumber("General.Terminal", 1)
    gmsh.model.add("model")

    xmin, xmax = 0.0, Lx
    ymin, ymax = 0.0, Ly
    L_bdt = Ly-H_bdt

    p1 = gmsh.model.geo.addPoint(xmin, ymin, 0, lowres)
    p2 = gmsh.model.geo.addPoint(xmax, ymin,  0, lowres)
    p3 = gmsh.model.geo.addPoint(xmax, ymax, 0, lowres)
    p4 = gmsh.model.geo.addPoint(xmin, ymax, 0, lowres)
    pbdt1 = gmsh.model.geo.addPoint(xmin-1, L_bdt, 0, lowres)
    pbdt2 = gmsh.model.geo.addPoint(xmax+1, L_bdt, 0, lowres)
    l_bot = gmsh.model.geo.addLine(p1, p2)
    lr = gmsh.model.geo.addLine(p2, p3)
    l_top = gmsh.model.geo.addLine(p3, p4)
    ll = gmsh.model.geo.addLine(p4, p1)
    l_bdt = gmsh.model.geo.addLine(pbdt1, pbdt2)
    # cl_lcrust = gmsh.model.geo.addCurveLoop([l_bot, lr_bot, l_bdt2, ll_bot])
    # cl_ucrust = gmsh.model.geo.addCurveLoop([l_bdt1, lr_top, l_top, ll_top])
    cl_domain = gmsh.model.geo.addCurveLoop([l_bot,lr,l_top,ll])
    domain = gmsh.model.geo.addPlaneSurface([cl_domain])
    # Add two points that will be used as fixed in ux :
    # ptop = gmsh.model.geo.addPoint(0.5, 2, 0, lowres)
    # pbot = gmsh.model.geo.addPoint(0.5, 0, 0, lowres)
    # embed these points in the surface to force the mesh to define nodes at their location.
    # pbot = gmsh.model.geo.addPoint(xmax/2, ymin, 0, lowres)
    # ptop = gmsh.model.geo.addPoint(xmax/2, ymax, 0, lowres)

    #pmid = gmsh.model.geo.addPoint(xmax/2, ymax/2, 0, highres)
    #gmsh.finalize()

    #circ = gmsh.model.occ.addCircle(xmax/2, ymax/2, 0.0, xmax/20)
    T = typeof(p1)
    curved_loops = T[]
    curved_loops_phys = T[]
    surfaces = T[]
    surfaces_phys = T[]
    for i in eachindex(radius)
        x,y = x0[i]
        r = radius[i]
        pmid = gmsh.model.geo.addPoint(x, y, 0, 2highres)
        pc1 = gmsh.model.geo.addPoint(x, y-r, 0, 2highres)
        pc2 = gmsh.model.geo.addPoint(x+r, y, 0, 2highres)
        pc3 = gmsh.model.geo.addPoint(x, y+r, 0, 2highres)
        pc4 = gmsh.model.geo.addPoint(x-r, y, 0, 2highres)
        arc1 = gmsh.model.geo.addCircleArc(pc1,pmid,pc2)
        arc2 = gmsh.model.geo.addCircleArc(pc2,pmid,pc3)
        arc3 = gmsh.model.geo.addCircleArc(pc3,pmid,pc4)
        arc4 = gmsh.model.geo.addCircleArc(pc4,pmid,pc1)
        cl_seed = gmsh.model.geo.addCurveLoop([arc1,arc2,arc3,arc4])
        seed = gmsh.model.geo.addPlaneSurface([cl_seed])

        gmsh.model.geo.synchronize()
        gmsh.model.mesh.embed(0, [pc1,pc2,pc3,pc4], 2, domain)
        gmsh.model.mesh.embed(1, [arc1,arc2,arc3,arc4], 2, domain)
        #cl_seed_phys = gmsh.model.addPhysicalGroup(1, [cl_seed])
        #seed_phys = gmsh.model.addPhysicalGroup(2, [seed])

        push!(curved_loops, cl_seed)
        push!(surfaces, seed)
        #push!(curved_loops_phys, cl_seed_phys)
        #push!(surfaces_phys, seed_phys)
    end
    # domain excluding seeds
    # upper_crust = gmsh.model.geo.addPlaneSurface([cl_ucrust,curved_loops...])
    # upper_crust_phys = gmsh.model.addPhysicalGroup(2, [upper_crust])
    #
    # lower_crust = gmsh.model.geo.addPlaneSurface([cl_lcrust])
    # lower_crust_phys = gmsh.model.addPhysicalGroup(2, [lower_crust])

    gmsh.model.geo.synchronize()
    gmsh.model.mesh.embed(1, [l_bdt], 2, domain)

    # gmsh.model.geo.synchronize()
    #gmsh.model.mesh.embed(2, [s2], 2, s1)

    bottom = gmsh.model.addPhysicalGroup(1, [l_bot])
    right = gmsh.model.addPhysicalGroup(1, [lr])
    top = gmsh.model.addPhysicalGroup(1, [l_top])
    left = gmsh.model.addPhysicalGroup(1, [ll])
    domain_phys = gmsh.model.addPhysicalGroup(2, [domain])
    #l_bdt = gmsh.model.addPhysicalGroup(1, [l_bdt1])

    #gmsh.model.setPhysicalName(0, clamped, "clamped")
    gmsh.model.setPhysicalName(1, bottom, "bottom")
    gmsh.model.setPhysicalName(1, right, "right")
    gmsh.model.setPhysicalName(1, top, "top")
    gmsh.model.setPhysicalName(1, left, "left")
    #gmsh.model.setPhysicalName(1, l_bdt, "bdt")
    # gmsh.model.setPhysicalName(2, lower_crust_phys, "lower_crust")
    # gmsh.model.setPhysicalName(2, upper_crust_phys, "upper_crust")
    gmsh.model.setPhysicalName(2, domain_phys, "domain")
    for i in eachindex(surfaces_phys)
        gmsh.model.setPhysicalName(1, curved_loops_phys[i], "circle_seed_$i")
        gmsh.model.setPhysicalName(2, surfaces_phys[i], "seed_$i")
    end
    gmsh.model.geo.synchronize()
    #gmsh.option.setNumber("Mesh.Algorithm", 2)
    #gmsh.option.setNumber("Mesh.RecombineAll", 1)
    if el_type == Quadrilateral
        gmsh.option.setNumber("Mesh.RecombinationAlgorithm", 3)
        gmsh.model.mesh.setRecombine(2, domain)
    end
    gmsh.model.mesh.generate(2)
    gmsh.write(file)
    gmsh.finalize()

    return nothing
end

function t15()
    gmsh.initialize()
    gmsh.option.setNumber("General.Terminal", 1)

    # Copied from t1.py...
    lc = 1e-2
    gmsh.model.geo.addPoint(0, 0, 0, lc, 1)
    gmsh.model.geo.addPoint(.1, 0, 0, lc, 2)
    gmsh.model.geo.addPoint(.1, .3, 0, lc, 3)
    gmsh.model.geo.addPoint(0, .3, 0, lc, 4)
    gmsh.model.geo.addLine(1, 2, 1)
    gmsh.model.geo.addLine(3, 2, 2)
    gmsh.model.geo.addLine(3, 4, 3)
    gmsh.model.geo.addLine(4, 1, 4)
    gmsh.model.geo.addCurveLoop([4, 1, -2, 3], 1)
    gmsh.model.geo.addPlaneSurface([1], 1)

    # We change the mesh size to generate a coarser mesh
    lc = lc * 4
    gmsh.model.geo.mesh.setSize([(0, 1), (0, 2), (0, 3), (0, 4)], lc)

    # We define a new point
    gmsh.model.geo.addPoint(0.02, 0.02, 0., lc, 5)

    # We have to synchronize before embedding entites:
    gmsh.model.geo.synchronize()

    # One can force this point to be included ("embedded") in the 2D mesh, using the
    # `embed()' function:
    gmsh.model.mesh.embed(0, [5], 2, 1)

    # In the same way, one can use `embed()' to force a curve to be embedded in the
    # 2D mesh:
    gmsh.model.geo.addPoint(0.02, 0.12, 0., lc, 6)
    gmsh.model.geo.addPoint(0.04, 0.18, 0., lc, 7)
    gmsh.model.geo.addLine(6, 7, 5)

    gmsh.model.geo.synchronize()
    gmsh.model.mesh.embed(1, [5], 2, 1)

    # Points and curves can also be embedded in volumes
    gmsh.model.geo.extrude([(2, 1)], 0, 0, 0.1)

    p = gmsh.model.geo.addPoint(0.07, 0.15, 0.025, lc)

    gmsh.model.geo.synchronize()
    gmsh.model.mesh.embed(0, [p], 3, 1)

    gmsh.model.geo.addPoint(0.025, 0.15, 0.025, lc, p + 1)
    l = gmsh.model.geo.addLine(7, p + 1)

    gmsh.model.geo.synchronize()
    gmsh.model.mesh.embed(1, [l], 3, 1)

    # Finally, we can also embed a surface in a volume:
    gmsh.model.geo.addPoint(0.02, 0.12, 0.05, lc, p + 2)
    gmsh.model.geo.addPoint(0.04, 0.12, 0.05, lc, p + 3)
    gmsh.model.geo.addPoint(0.04, 0.18, 0.05, lc, p + 4)
    gmsh.model.geo.addPoint(0.02, 0.18, 0.05, lc, p + 5)

    gmsh.model.geo.addLine(p + 2, p + 3, l + 1)
    gmsh.model.geo.addLine(p + 3, p + 4, l + 2)
    gmsh.model.geo.addLine(p + 4, p + 5, l + 3)
    gmsh.model.geo.addLine(p + 5, p + 2, l + 4)

    ll = gmsh.model.geo.addCurveLoop([l + 1, l + 2, l + 3, l + 4])
    s = gmsh.model.geo.addPlaneSurface([ll])

    gmsh.model.geo.synchronize()
    gmsh.model.mesh.embed(2, [s], 3, 1)

    # Note that with the OpenCASCADE kernel (see `t16.py'), when the `fragment()'
    # function is applied to entities of different dimensions, the lower dimensional
    # entities will be autmatically embedded in the higher dimensional entities if
    # necessary.

    gmsh.model.mesh.generate(3)

    gmsh.write("t15.msh")

    gmsh.finalize()

end

function test_embed()
    gmsh.initialize()
    gmsh.option.setNumber("General.Terminal", 1)

    # Copied from t1.py...
    lc = 1e-2
    xmax, ymax = 1.0, 2.0
    gmsh.model.geo.addPoint(0, 0, 0, lc, 1)
    gmsh.model.geo.addPoint(xmax, 0, 0, lc, 2)
    gmsh.model.geo.addPoint(xmax, ymax, 0, lc, 3)
    gmsh.model.geo.addPoint(0, ymax, 0, lc, 4)
    gmsh.model.geo.addLine(1, 2, 1)
    gmsh.model.geo.addLine(3, 2, 2)
    gmsh.model.geo.addLine(3, 4, 3)
    gmsh.model.geo.addLine(4, 1, 4)
    gmsh.model.geo.addCurveLoop([4, 1, -2, 3], 1)
    gmsh.model.geo.addPlaneSurface([1], 1)

    # In the same way, one can use `embed()' to force a curve to be embedded in the
    # 2D mesh:
    gmsh.model.geo.addPoint(-0.0001, ymax/2, 0., lc, 5)
    gmsh.model.geo.addPoint(xmax+0.0001, ymax/2, 0., lc, 6)
    gmsh.model.geo.addLine(5, 6, 5)

    gmsh.model.geo.synchronize()
    gmsh.model.mesh.embed(1, [5], 2, 1)

    gmsh.model.mesh.generate(2)

    gmsh.write("test_embed.msh")

    gmsh.finalize()

end

function test_embed2()
    gmsh.initialize()
    gmsh.option.setNumber("General.Terminal", 1)

    # Copied from t1.py...
    lc = 1e-2
    xmax, ymax = 1.0, 2.0
    gmsh.model.geo.addPoint(0, 0, 0, lc, 1)
    gmsh.model.geo.addPoint(xmax, 0, 0, lc, 2)
    gmsh.model.geo.addPoint(xmax, ymax, 0, lc, 3)
    gmsh.model.geo.addPoint(0, ymax, 0, lc, 4)
    gmsh.model.geo.addPoint(0.0001, ymax/2, 0., lc, 5)
    gmsh.model.geo.addPoint(xmax-0.0001, ymax/2, 0., lc, 6)
    gmsh.model.geo.addLine(1, 2, 1)
    gmsh.model.geo.addLine(2, 6, 2)
    gmsh.model.geo.addLine(6, 5, 3)
    gmsh.model.geo.addLine(5, 1, 4)

    gmsh.model.geo.addLine(6, 3, 5)
    gmsh.model.geo.addLine(3, 4, 6)
    gmsh.model.geo.addLine(4, 5, 7)

    gmsh.model.geo.addCurveLoop([1, 2, 3, 4], 1)
    gmsh.model.geo.addCurveLoop([-3, 5, 6, 7], 2)
    gmsh.model.geo.addPlaneSurface([1,2], 1)

    # In the same way, one can use `embed()' to force a curve to be embedded in the
    # 2D mesh:
    # gmsh.model.geo.addPoint(0.0001, ymax/2, 0., lc, 5)
    # gmsh.model.geo.addPoint(xmax-0.0001, ymax/2, 0., lc, 6)
    # gmsh.model.geo.addLine(5, 6, 5)
    #
    # gmsh.model.geo.synchronize()
    # gmsh.model.mesh.embed(1, [5], 2, 1)

    gmsh.model.mesh.generate(2)

    gmsh.write("test_embed2.msh")

    gmsh.finalize()

end


using HetaSimulator, Plots

p = load_platform("./jak2stat5");

### test run

sim(p.models[:nameless], tspan = (0., 100.), alg = CVODE_BDF()) |> plot

### Load conditions

cond_df = read_conditions("./jak2stat5/Data/conditions.csv")
add_conditions!(p, cond_df)

# chapter 2.3.1, page 21, figure S11 => task1
dts1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_Long_log10-mod.csv");
add_measurements!(p, dts1_csv)

# chapter 2.3.2, page 23, figure S12
dts2_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_Concentrations_log10-mod.csv");
add_measurements!(p, dts2_csv)

# chapter 2.3.3, page 25, figure S13
dts3_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_RNA_log10-mod.csv");
add_measurements!(p, dts3_csv)

# chapter 2.3.4, page 28, figure S14
dts4_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_ActD0_log10-mod.csv");
dts4_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_ActD1_log10-mod.csv");
add_measurements!(p, dts4_0_csv)
add_measurements!(p, dts4_1_csv)

# chapter 2.3.5, page 31, figure S15
dts5_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_Fine_log10-mod.csv");
add_measurements!(p, dts5_csv)

# chapter 2.3.6, page 34, figure S16
dts6_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_CISoe0_log10-mod.csv");
dts6_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_CISoe1_log10-mod.csv");
add_measurements!(p, dts6_0_csv)
add_measurements!(p, dts6_1_csv)

# chapter 2.3.7, page 36, figure S17
dts7_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_CISoe_pEpoR0_log10-mod.csv");
dts7_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_CISoe_pEpoR1_log10-mod.csv");
add_measurements!(p, dts7_0_csv)
add_measurements!(p, dts7_1_csv)

# chapter 2.3.8, page 38, figure S18
dts8_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_SOCS3oe0_log10-mod.csv");
dts8_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_SOCS3oe1_log10-mod.csv");
add_measurements!(p, dts8_0_csv)
add_measurements!(p, dts8_1_csv)

# chapter 2.3.9, page 40, figure S19
dts9_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_SHP1oe0_log10-mod.csv");
dts9_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_SHP1oe1_log10-mod.csv");
add_measurements!(p, dts9_0_csv)
add_measurements!(p, dts9_1_csv)

# chapter 2.3.10, page 42, figure S20
dts10_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR72.5e-9_log10-mod.csv");
dts10_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR72.5e-8_log10-mod.csv");
dts10_2_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR72.5e-7_log10-mod.csv");
dts10_3_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR72.5e-6_log10-mod.csv");
dts10_4_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR72.5e-5_log10-mod.csv");
add_measurements!(p, dts10_0_csv)
add_measurements!(p, dts10_1_csv)
add_measurements!(p, dts10_2_csv)
add_measurements!(p, dts10_3_csv)
add_measurements!(p, dts10_4_csv)

# chapter 2.3.11, page 44, figure S21
dts11_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR302.5e-9_log10-mod.csv"); # 0
dts11_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR302.5e-8_log10-mod.csv"); # 1
dts11_2_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR301.25e-7_log10-mod.csv"); # 2
dts11_3_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR302.5e-7_log10-mod.csv"); # 3
dts11_4_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR301.25e-6_log10-mod.csv"); # 4
dts11_5_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR302.5e-6_log10-mod.csv"); # 5
add_measurements!(p, dts11_0_csv)
add_measurements!(p, dts11_1_csv)
add_measurements!(p, dts11_2_csv)
add_measurements!(p, dts11_3_csv)
add_measurements!(p, dts11_4_csv)
add_measurements!(p, dts11_5_csv)

# chapter 2.3.12, page 46, figure S22
dts12_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR101.25e-8_log10-mod.csv"); # 1.25e-8 # 0
dts12_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR101.75e-8_log10-mod.csv"); # 1.75e-8 # 1
dts12_2_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR101.25e-7_log10-mod.csv"); # 1.25e-7 # 2
dts12_3_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR101.7675e-7_log10-mod.csv"); # 1.7675e-7 # 3 # maybe mistake
dts12_4_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR102.5e-7_log10-mod.csv"); # 2.5e-7 # 4
dts12_5_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR102.5e-6_log10-mod.csv"); # 2.5e-6 # 5
add_measurements!(p, dts12_0_csv)
add_measurements!(p, dts12_1_csv)
add_measurements!(p, dts12_2_csv)
add_measurements!(p, dts12_3_csv)
add_measurements!(p, dts12_4_csv)
add_measurements!(p, dts12_5_csv)

# chapter 2.3.13, page 48, figure S23
dts13_0_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR902.5e-9_log10-mod.csv"); # 0
dts13_1_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR902.5e-8_log10-mod.csv"); # 1
dts13_2_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR901.25e-7_log10-mod.csv"); # 2
dts13_3_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR902.5e-7_log10-mod.csv"); # 3
dts13_4_csv = read_measurements("./jak2stat5/Data/JAK2STAT5_CFUE_DR902.5e-6_log10-mod.csv"); # 4
add_measurements!(p, dts13_0_csv)
add_measurements!(p, dts13_1_csv)
add_measurements!(p, dts13_2_csv)
add_measurements!(p, dts13_3_csv)
add_measurements!(p, dts13_4_csv)

res = sim(p; alg = CVODE_BDF())
plot(res) # very bad picture