__precompile__()
module Length

import DataStructures: counter
using SeqHax.Utils
using ArgParse

function add_args(argparser)
    add_common_args(argparser["length"])
    add_paired_args(argparser["length"])
    @add_arg_table argparser["length"] begin
        "input"
            help = "Input FASTQ read file(s) (may be gzipped)"
            required = true
            arg_type = String
    end
end

function main(args)
    r1ctr = counter(Int)
    paired = args["paired"]
    quiet = args["quiet"]
    r2ctr = ifelse(paired, counter(Int), r1ctr)

    if paired
        npair = foreach_read(args["input"], quiet=quiet) do idx, read
            push!(r1ctr, length(read.seq))
        end
    else
        npair = foreach_readpair(args["input"], quiet=quiet) do idx, r1, r2
            push!(r1ctr, length(r1.seq))
            push!(r2ctr, length(r2.seq))
        end
    end

    lengths = sort(union(keys(r1ctr), keys(r2ctr)))
    if paired
        println("read_length\tr1_count\tr2_count")
    else
        println("read_length\tcount")
    end
    for len in lengths
        if paired
            r1 = r1ctr[len]
            r2 = r2ctr[len]
            println("$len\t$r1\t$r2")
        else
            cnt = r1ctr[len]
            println("$len\t$cnt")
        end
    end
end

end # module Length


function alexnet(name::Symbol; kwargs...)
    assert_name_present_in(name, (:alexnet,))
    model = Lux.transform(AlexNet().layers)
    return initialize_model(name, model; kwargs...)
end

function resnet(name::Symbol; kwargs...)
    assert_name_present_in(name, (:resnet18, :resnet34, :resnet50, :resnet101, :resnet152))
    model = if name == :resnet18
        Lux.transform(ResNet(18).layers)
    elseif name == :resnet34
        Lux.transform(ResNet(34).layers)
    elseif name == :resnet50
        Lux.transform(ResNet(50).layers)
    elseif name == :resnet101
        Lux.transform(ResNet(101).layers)
    elseif name == :resnet152
        Lux.transform(ResNet(152).layers)
    end
    return initialize_model(name, model; kwargs...)
end

function resnext(name::Symbol; kwargs...)
    assert_name_present_in(name, (:resnext50, :resnext101, :resnext152))
    model = if name == :resnext50
        Lux.transform(ResNeXt(50).layers)
    elseif name == :resnext101
        Lux.transform(ResNeXt(101).layers)
    elseif name == :resnext152
        Lux.transform(ResNeXt(152).layers)
    end
    return initialize_model(name, model; kwargs...)
end

function googlenet(name::Symbol; kwargs...)
    assert_name_present_in(name, (:googlenet,))
    model = Lux.transform(GoogLeNet().layers)
    return initialize_model(name, model; kwargs...)
end

function densenet(name::Symbol; kwargs...)
    assert_name_present_in(name, (:densenet121, :densenet161, :densenet169, :densenet201))
    model = if name == :densenet121
        Lux.transform(DenseNet(121).layers)
    elseif name == :densenet161
        Lux.transform(DenseNet(161).layers)
    elseif name == :densenet169
        Lux.transform(DenseNet(169).layers)
    elseif name == :densenet201
        Lux.transform(DenseNet(201).layers)
    end
    return initialize_model(name, model; kwargs...)
end

function mobilenet(name::Symbol; kwargs...)
    assert_name_present_in(name,
                           (:mobilenet_v1, :mobilenet_v2, :mobilenet_v3_small,
                            :mobilenet_v3_large))
    model = if name == :mobilenet_v1
        Lux.transform(MobileNetv1().layers)
    elseif name == :mobilenet_v2
        Lux.transform(MobileNetv2().layers)
    elseif name == :mobilenet_v3_small
        Lux.transform(MobileNetv3(:small).layers)
    elseif name == :mobilenet_v3_large
        Lux.transform(MobileNetv3(:large).layers)
    end
    return initialize_model(name, model; kwargs...)
end

function convmixer(name::Symbol; kwargs...)
    assert_name_present_in(name, (:base, :large, :small))
    model = Lux.transform(ConvMixer(name).layers)
    return initialize_model(name, model; kwargs...)
end


let
    MultiStageCBS.State()
    MultiStageCBS.Action()
    MultiStageCBS.LowLevelEnv()
    s = MultiStageCBS.State(vtx=1)
    @test states_match(s,s)
    a = MultiStageCBS.Action()
    env = MultiStageCBS.LowLevelEnv()
    a = CRCBS.wait(s)
end
let
    G = Graph(3)
    add_edge!(G,1,2)
    add_edge!(G,2,2)
    add_edge!(G,2,3)
    env = MultiStageCBS.LowLevelEnv(graph=G)
    for a in get_possible_actions(env,MultiStageCBS.State(vtx=2))
        @test get_e(a).src == 2
        @test get_e(a).dst ∈ [1,2,3]
    end
end
let
    vtx_grid = initialize_regular_vtx_grid(;n_obstacles_x=1,n_obstacles_y=1)
    #  1   2   3   4   5   6
    #  7   8   9  10  11  12
    # 13  14   0   0  15  16
    # 17  18   0   0  19  20
    # 21  22  23  24  25  26
    # 27  28  29  30  31  32
    G = initialize_grid_graph_from_vtx_grid(vtx_grid)
    starts = [
        MultiStageCBS.State(vtx=1,stage=1,t=0),
        MultiStageCBS.State(vtx=2,stage=1,t=0),
        ]
    goals = [
        [MultiStageCBS.State(vtx=27),MultiStageCBS.State(vtx=13)],
        [MultiStageCBS.State(vtx=28),MultiStageCBS.State(vtx=14)],
        ]
    let
        heuristic = MultiStagePerfectHeuristic(
            G, map(g->map(s->s.vtx, g), goals)
        )
        env = MultiStageCBS.LowLevelEnv(graph=G,heuristic=heuristic)
        mapf = MAPF(env,starts,goals)
        solver = CBSSolver(AStar{cost_type(mapf)}())
        set_verbosity!(solver,global_verbosity())
        set_iteration_limit!(solver,10)
        node = initialize_root_node(mapf)
        @test low_level_search!(solver,mapf,node)
        # convert_to_vertex_lists(node.solution)
    end
    let
        heuristic = MultiStagePerfectHeuristic(
            G, map(g->map(s->s.vtx, g), goals)
        )
        env = MultiStageCBS.LowLevelEnv(graph=G,heuristic=heuristic)
        mapf = MAPF(env,starts,goals)
        solver = CBSSolver(AStar{cost_type(mapf)}())
        set_verbosity!(solver,global_verbosity())
        set_iteration_limit!(solver,10)
        node = initialize_root_node(mapf)
        solution, cost = CRCBS.solve!(solver,mapf)
        # @test cost == 16
    end
    let
        env = MultiStageCBS.LowLevelEnv(
            graph=G,
            cost_model = construct_composite_cost_model(
                FullCostModel(sum,NullCost()),
                FullCostModel(sum,TravelTime())
                ),
            heuristic = construct_composite_heuristic(
                NullHeuristic(),
                MultiStagePerfectHeuristic(G, map(g->map(s->s.vtx, g), goals))
                )
            )
        mapf = MAPF(env,starts,goals)
        node = initialize_root_node(mapf)
        solver = CBSSolver(AStar{cost_type(mapf)}())
        set_verbosity!(solver,global_verbosity())
        set_iteration_limit!(solver,10)
        solution, cost = CRCBS.solve!(solver,mapf);
        # @test cost[2] == 16
    end
end
let
    # solver = MultiStageCBS.CBSSolver()
    vtx_grid = initialize_regular_vtx_grid(;n_obstacles_x=2,n_obstacles_y=2,obs_offset = [1;1])
    #  1   2   3   4   5   6   7   8   9  10
    # 11  12  13  14  15  16  17  18  19  20
    # 21  22   0   0  23  24   0   0  25  26
    # 27  28   0   0  29  30   0   0  31  32
    # 33  34  35  36  37  38  39  40  41  42
    # 43  44  45  46  47  48  49  50  51  52
    # 53  54   0   0  55  56   0   0  57  58
    # 59  60   0   0  61  62   0   0  63  64
    # 65  66  67  68  69  70  71  72  73  74
    # 75  76  77  78  79  80  81  82  83  84
    G = initialize_grid_graph_from_vtx_grid(vtx_grid)
    starts = [
        MultiStageCBS.State(vtx=1,stage=1,t=0),
        MultiStageCBS.State(vtx=2,stage=1,t=0),
        # MultiStageCBS.State(vtx=26,stage=1,t=10),
        # MultiStageCBS.State(vtx=19,stage=1,t=10),
        # MultiStageCBS.State(vtx=41,stage=1,t=4)
        ]
    goals = [
        [MultiStageCBS.State(vtx=14),MultiStageCBS.State(vtx=81)],
        [MultiStageCBS.State(vtx=5),MultiStageCBS.State(vtx=55)],
        # [MultiStageCBS.State(vtx=60),MultiStageCBS.State(vtx=52)],
        # [MultiStageCBS.State(vtx=61)],
        # [MultiStageCBS.State(vtx=32),MultiStageCBS.State(vtx=1)]
        ]
    let
        heuristic = MultiStagePerfectHeuristic(
            G, map(g->map(s->s.vtx, g), goals)
        )
        env = MultiStageCBS.LowLevelEnv(graph=G,heuristic=heuristic)
        mapf = MAPF(env,starts,goals)
        solver = CBSSolver(AStar{cost_type(mapf)}())
        set_verbosity!(solver,global_verbosity())
        set_iteration_limit!(solver,10)
        node = initialize_root_node(mapf)
        solution, cost = CRCBS.solve!(solver,mapf)
    end
    let
        env = MultiStageCBS.LowLevelEnv(
            graph=G,
            cost_model = construct_composite_cost_model(
                FullCostModel(sum,NullCost()),
                FullCostModel(sum,TravelTime())
                ),
            heuristic = construct_composite_heuristic(
                NullHeuristic(),
                MultiStagePerfectHeuristic(G, map(g->map(s->s.vtx, g), goals))
                )
            )
        mapf = MAPF(env,starts,goals)
        node = initialize_root_node(mapf)
        solver = CBSSolver(AStar{cost_type(mapf)}())
        set_verbosity!(solver,global_verbosity())
        set_iteration_limit!(solver,10)
        solution, cost = CRCBS.solve!(solver,mapf);
    end
end


# code to compile the Stock Market Execss and the State of the Union Address texts
# into a matching DataFrame and Document-Term matrix

using Lasso, CSV, DataDeps, DataFrames, FamaFrenchData, TextAnalysis, SparseArrays

register(DataDep("sotu",
        """
        Dataset: State of the Union Adresses
        Website: http://stateoftheunion.onetwothree.net
        Author: Brad Borevitz
        """,
        "http://stateoftheunion.onetwothree.net/texts/stateoftheunion1790-2020.txt.zip",
        "f61dfaf9b487872173e664488e6daad1783fa42691ed8009b2c5245866939b97",
        post_fetch_method = unpack
    ))

# fast way to remove sparse terms using the dtm, returns a new dtm with only terms that occur at least minoccurances times in the corpus
function remove_sparse_terms(D::TextAnalysis.DocumentTermMatrix, minoccurances = 3)
  m,n=size(D.dtm)
  rows = rowvals(D.dtm)
  vals = nonzeros(D.dtm)
  newrows = Int[]
  newcols = Int[]
  newvals = Int[]
  newterms = String[]
  new_column_indices = Dict{String, Int}()
  colsums = sum(D.dtm, dims=1)
  col = 1
  for i = 1:n
    if colsums[i]>=minoccurances
      for j in nzrange(D.dtm, i)
        row = rows[j]
        val = vals[j]
        push!(newrows, row)
        push!(newcols, col)
        push!(newvals, val)
      end
      term = D.terms[i]
      push!(newterms, term)
      new_column_indices[term] = col
      col += 1
    end
  end
  newterms
  if length(newrows) > 0
      newdtm = sparse(newrows, newcols, newvals)
  else
      newdtm = spzeros(Int,m, 0)
  end
  DocumentTermMatrix(newdtm, newterms, new_column_indices)
end

# get the SOTU text
function loadsotu()
    sotutxt = datadep"sotu/stateoftheunion1790-2020.txt"
    sotustr = read(sotutxt, String)
    addresses = strip.(split(sotustr, "***", keepempty=false)[2:end-1])

    # typical structure:
    #
    # ***
    #
    # State of the Union Address
    # Harry S. Truman
    # January 6, 1947
    #
    # Mr. President, Mr. Speaker, Members of the Congress of the United States:
    #
    # It looks like a good many of you have moved over to the left since I was
    # here last! ...

    # vectors to save parsed documents and dates
    docs = NGramDocument[]
    years = Int[]

    for a in addresses[1:end]
        # an IOBuffer provides an efficient way of reading lines
        strbuf = IOBuffer(a)

        # read header
        title = readline(strbuf)
        speaker = readline(strbuf)
        datestr = readline(strbuf)
        year = parse(Int, datestr[end-3:end])
        if 1927 <= year <= 2019

            # skip "Dear Congress" at start
            readline(strbuf)
            readline(strbuf)
            readline(strbuf)

            speechtext = read(strbuf, String)
            # remove everything after "NOTE:"
            ix = findfirst("NOTE:", speechtext)
            if !isnothing(ix)
                speechtext = speechtext[1:ix[1]-1]
            end

            # initial preprocessing
            sdoc = StringDocument(speechtext)
            prepare!(sdoc, strip_non_letters | strip_case)
            prepare!(sdoc, strip_stopwords)
            prepare!(sdoc, strip_whitespace | strip_corrupt_utf8)

            # build TextAnalysis NGramDocument of bigrams (complexity=2)
            doc = NGramDocument(text(sdoc), 2)
            author!(doc, speaker)
            timestamp!(doc, datestr)

            # save parsed documents into vectors
            if length(years) > 0 && year == years[end]
                # overwrite last one added becasue sometimes (e.g. 1953) the outgoing president addresses too
                docs[end] = doc
            else
                push!(docs, doc)
                push!(years, year)
            end
        end
    end

    corpus = Corpus(docs)
    @info "built SOTU corpus with $(length(corpus)) speeches"

    update_lexicon!(corpus)
    D = DocumentTermMatrix(corpus)
    D = remove_sparse_terms(D, 20)
    @info "built Document Term Matrix with bigrams occuring at least 20 times in the corpus"

    sotu = DataFrame(
            Date=[parse(Int, ts[end-3:end]) for ts in timestamps(corpus)],
            President=author.(corpus)
        )

    sotu, D
end

# load fama french annual market excess Returns
function loadreturns()
    tables, tablenotes, filenotes = readFamaFrench("F-F_Research_Data_Factors")
    FF3_annual = tables[2]
    rename!(FF3_annual, "Mkt-RF"=>"Rem")
    FF3_annual
end

# load merged sotu + market excess returns dataset
function loadsoturet(;cache=true)
    csvfile = joinpath(datadep"sotu", "sotu.ret.csv")
    dtmfile = joinpath(datadep"sotu", "sotu.dtm.csv")
    termsfile = joinpath(datadep"sotu", "sotu.terms.csv")

    if cache && isfile(csvfile) && isfile(dtmfile) && isfile(termsfile)
        df = CSV.read(csvfile, DataFrame)

        countsIJV = CSV.read(dtmfile, DataFrame)
        counts = sparse(countsIJV.I, countsIJV.J, countsIJV.V)

        terms = CSV.read(termsfile, DataFrame).term
        @info "SOTU + Rem loaded from cached $(datadep"sotu")"
    else
        sotu, D = loadsotu()
        ret = loadreturns()

        # merged dataset
        ixret = findall(in(sotu.Date), ret.Date)
        @assert sotu.Date == ret.Date[ixret] "sotu and return years do not lineup as expected"

        df = innerjoin(ret[ixret, ["Date", "Rem"]], sotu, on="Date")

        # save to csv file
        df |> CSV.write(csvfile)

        # save dtm in IJV sparse format
        counts = dtm(D)
        I, J, V = findnz(counts)
        DataFrame(I=I, J=J, V=V) |> CSV.write(dtmfile)

        # save terms corresponding to the columns of the counts matrix
        terms = D.terms
        DataFrame(index=1:length(terms), term=terms) |> CSV.write(termsfile)

        @info "SOTU + Rem loaded and saved to $(datadep"sotu")"
    end

    df, counts, terms
end

"""
converts a python/scipy sprase matrix to julia
Code from https://github.com/JuliaPy/PyCall.jl/issues/735#issuecomment-577336332
"""
function scipyCSC_to_julia(A)
    m, n = A.shape
    colPtr = Int[i+1 for i in PyArray(A."indptr")]
    rowVal = Int[i+1 for i in PyArray(A."indices")]
    nzVal = Vector{Float64}(PyArray(A."data"))
    B = SparseMatrixCSC{Float64,Int}(m, n, colPtr, rowVal, nzVal)
    return PyCall.pyjlwrap_new(B)
end

"""
converts a python/pandas dataframe to julia
Code from https://discourse.julialang.org/t/converting-pandas-dataframe-returned-from-pycall-to-julia-dataframe/43001/3
"""
function pd_to_df(df_pd)
    colnames = map(Symbol, df_pd.columns)
    df = DataFrame(Any[Array(df_pd[c].values) for c in colnames], colnames)
end

covarsdf, counts, terms = loadsoturet(cache=false)


using MLStyle

"""
    callify(mycall, ast)

Replace every `f(args...; kwargs..)` with `mycall(f, args...; kwargs...)`
"""
function callify(mycall, ast)
    leaf(x) = x
    function branch(f, head, args)
        default() = Expr(head, map(f, args)...)
        head == :call || return default()

        if first(args) == :~ && length(args) == 3
            return default()
        end

        # At this point we know it's a function call
        length(args) == 1 && return Expr(:call, mycall, first(args))

        fun = args[1]
        arg2 = args[2]

        if arg2 isa Expr && arg2.head == :parameters
            # keyword arguments (try dump(:(f(x,y;a=1, b=2))) to see this)
            return Expr(:call, mycall, arg2, fun, map(f, Base.rest(args, 3))...)
        else
            return Expr(:call, mycall, map(f, args)...)
        end

    end

    foldast(leaf, branch)(ast)
end

# struct Provenance{T,S}
#     value::T
#     sources::S
# end

# getvalue(p::Provenance) = p.value
# getvalue(x) = x

# getsources(p::Provenance) = p.sources
# getsources(x) = Set()

# function trace_provenance(f, args...; kwargs...)
#     (newargs, arg_sources) = (getvalue.(args), union(getsources.(args)...))

#     k = keys(kwargs)
#     v = values(kwargs)
#     newkwargs = NamedTuple{k}(map(getvalue, v))

#     k = keys(kwargs)
#     v = values(NamedTuple(kwargs))
#     newkwargs = NamedTuple{k}(getvalue.(v))
#     kwarg_sources = union(getsources.(args)...)

#     sources = union(arg_sources, kwarg_sources)
#     Provenance(f(newargs...; newkwargs), sources)
# end

# macro call(expr)
#     callify(expr)
# end

# julia> callify(:(f(g(x,y))))
# :(call(f, call(g, x, y)))

# julia> callify(:(f(x; a=3)))
# :(call(f, x; a = 3))

# julia> callify(:(a+b))
# :(call(+, a, b))

# julia> callify(:(call(f,3)))
# :(call(f, 3))

# f(x) = x+1

# @call f(2)

# using SymbolicUtils

# @syms x::Vector{Float64} i::Int

# @call getindex(x,i)


using Random

using Dates, TimeZones
using MicroLogging
using NumericalIntegration
using DataFrames, DataFramesMeta, CSV

using StatsBase
using TimeSeries
using HypothesisTests
using Mise
using Impute
using CurveFit
using StatsPlots

function run_analysis()

    @info "Start Analysis"
    flush(stdout); flush(stderr)

    seoul_codes = [
        111121,111123,111131,111141,111142,
        111151,111152,111161,111171,111181,
        111191,111201,111212,111221,111231,
        111241,111251,111261,111262,111273,
        111274,111281,111291,111291,111301,
        111301,111311]
    seoul_names = [
        "중구","종로구","용산구","광진구","성동구",
        "중랑구","동대문구","성북구","도봉구","은평구",
        "서대문구","마포구","강서구","구로구","영등포구",
        "동작구","관악구","강남구","서초구","송파구",
        "강동구","금천구","강북구","강북구","양천구",
        "양천구","노원구"]
    # construct named tuple
    seoul_stations = (; zip(Symbol.(seoul_names), seoul_codes)...)

    # NamedTuple (::Symbol -> ::DataFrame)
    df = load_data_DNN("/input/jongro_seoul.csv", seoul_stations)
    #=
    first(df, 5)
    │ Row │ stationCode │ date                      │ lat      │ long      │ SO2     │ CO      │ O3      │ NO2     │ PM10    │ PM25    │ temp    │ u       │ v       │ pres    │ prep     │ snow     │
    │     │ Int64       │ TimeZones.ZonedDateTime   │ Float64  │ Float64   │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ Float64  │ Float64  │
    ├─────┼─────────────┼───────────────────────────┼──────────┼───────────┼─────────┼─────────┼─────────┼─────────┼─────────┼─────────┼─────────┼─────────┼─────────┤─────────┤──────────┤──────────┤
    │ 1   │ 111123      │ 2015-01-01T01:00:00+09:00 │ 37.572   │ 127.005   │ 0.004   │ 0.2     │ 0.02    │ 0.009   │ 57.0    │ 0.0     │ -7.4    │ 4.41656 │ 1.60749 │ 1011.8  │ missing  │ missing  │
    │ 2   │ 111123      │ 2015-01-01T02:00:00+09:00 │ 37.572   │ 127.005   │ 0.005   │ 0.2     │ 0.019   │ 0.008   │ 70.0    │ 3.0     │ -8.0    │ 4.22862 │ 1.53909 │ 1011.7  │ missing  │ missing  │
    │ 3   │ 111123      │ 2015-01-01T03:00:00+09:00 │ 37.572   │ 127.005   │ 0.005   │ 0.2     │ 0.02    │ 0.006   │ 92.0    │ 5.0     │ -8.4    │ 3.57083 │ 1.29968 │ 1012.1  │ missing  │ missing  │
    │ 4   │ 111123      │ 2015-01-01T04:00:00+09:00 │ 37.572   │ 127.005   │ 0.004   │ 0.2     │ 0.019   │ 0.005   │ 111.0   │ 2.0     │ -8.8    │ 4.60449 │ 1.6759  │ 1012.3  │ missing  │ missing  │
    │ 5   │ 111123      │ 2015-01-01T05:00:00+09:00 │ 37.572   │ 127.005   │ 0.005   │ 0.2     │ 0.019   │ 0.006   │ 127.0   │ 5.0     │ -9.1    │ 5.35625 │ 1.94951 │ 1011.8  │ missing  │ missing  │
    =#

    #===== start of parameter zone =====#
    total_fdate, total_tdate = get_date_range(df)
    train_fdate = ZonedDateTime(2008, 1, 1, 1, tz"Asia/Seoul")
    train_tdate = ZonedDateTime(2017, 12, 31, 23, tz"Asia/Seoul")
    test_fdate = ZonedDateTime(2018, 1, 1, 0, tz"Asia/Seoul")
    test_tdate = ZonedDateTime(2018, 12, 31, 23, tz"Asia/Seoul")

    # stations
    #=
    train_stn_names = [
        :중구,:종로구,:용산구,:광진구,:성동구,
        :중랑구,:동대문구,:성북구,:도봉구,:은평구,
        :서대문구,:마포구,:강서구,:구로구,:영등포구,
        :동작구,:관악구,:강남구,:서초구,:송파구,
        :강동구,:금천구,:강북구,:강북구,:양천구,
        :양천구,:노원구]
    =#
    #train_stn_names = [:종로구, :강서구, :송파구, :강남구]

    train_stn_names = [
        :중구,:종로구,:용산구,:광진구,:성동구,
        :중랑구,:동대문구,:성북구,:도봉구,:은평구,
        :서대문구,:마포구,:강서구,:구로구,:영등포구,
        :동작구,:관악구,:강남구,:서초구,:송파구,
        :강동구,:금천구,:강북구,:강북구,:양천구,
        :양천구,:노원구]
    train_stn_names = [:종로구]

    df = filter_raw_data(df, train_fdate, train_tdate, test_fdate, test_tdate)
    @show first(df, 10)

    features = [:PM10, :PM25]
    # If you want exclude some features, modify train_features
    # exclude :PM10, :PM25 temporarily for log transform
    train_features = [:PM10, :PM25]
    target_features = [:PM10, :PM25]

    # For GPU, change precision of Floating numbers
    eltype::DataType = Float32

    # simply collect dates, determine exact date for prediction (for 1h, 24h, and so on) later

    Base.Filesystem.mkpath("/mnt/analysis/")

    for ycol in target_features
        # DNN's windowed dataframe is already filterred with dates.
        # size(df) = (sample_size, length(features))
        @info "Analysis of $(string(ycol))"
        for name in train_stn_names
            code = seoul_stations[name]
            stn_df = filter_raw_data(df, code, train_fdate, train_tdate)
            # for imputatation
            CSV.write("/mnt/analysis/raw_data_analysis_$(string(ycol))_$((name)).csv", stn_df)
            dates = stn_df[!, :date]

            # Imputation
            # allow missing again
            DataFrames.allowmissing!(stn_df)

            zero2Missing!(stn_df, ycol)

            # Analysis : pdf
            @info "Estimate probability density function without imputation..."
            ycol_arr = collect(skipmissing(stn_df[!, ycol]))
            if ycol == :PM10
                npts = 256
            elseif ycol == :PM25
                npts = 256
            end
            ycol_pdfx, ycol_pdfy = pdf(ycol_arr, npts, method=:kernel)
            plot_anal_pdf(ycol_pdfx, ycol_pdfy, ycol,
                "PDF for $(ycol)", "/mnt/analysis/", "$(string(ycol))_$(string(name))_pdf_noimpute")
            ycol_logpdfx, ycol_logpdfy, = pdf(log10.(ycol_arr), npts, method=:kernel)
            plot_anal_pdf(ycol_logpdfx, ycol_logpdfy, ycol,
                "Log PDF for $(ycol)", "/mnt/analysis/", "$(string(ycol))_$(string(name))_logpdf_noimpute")

            # mean imputation
            #impute!(df, ycol, :mean; total_mean = total_mean)
            # simple sampling
            stn_df[!, ycol] = Impute.srs(stn_df[!, ycol])
            # knn imputation
            #nK = 10
            #ycols = [:SO2, :CO, :O3, :NO2, :PM10, :PM25, :temp, :u, :v, :pres, :humid, :prep, :snow]
            #impute!(stn_df, ycol, :knn; ycols = ycols, leafsize = nK)

            # disallow missing again
            DataFrames.disallowmissing!(stn_df)

            ta = TimeArray(dates, stn_df[!, ycol])

            @info "Estimate probability density function without imputation..."
            ycol_arr = stn_df[!, ycol]
            ycols = stn_df[!, ycol]
            logycols = log10.(stn_df[!, ycol])

            total_mean, total_std = StatsBase.mean_and_std(values(ta))

            @info "Total mean of $(string(ycol)) : ", total_mean
            @info "Total std  of $(string(ycol)) : ", total_std

            if ycol == :PM10
                npts = 200
            elseif ycol == :PM25
                npts = 200
            end
            ycol_pdfx, ycol_pdfy = pdf(ycol_arr, npts, method=:kernel)
            plot_anal_pdf(ycol_pdfx, ycol_pdfy, ycol,
                "PDF for $(ycol)", "/mnt/analysis/", "$(string(ycol))_$(string(name))_pdf_impute")
            ycol_logpdfx, ycol_logpdfy, = pdf(log10.(ycol_arr), npts, method=:kernel)
            plot_anal_pdf(ycol_logpdfx, ycol_logpdfy, ycol,
                "Log PDF for $(ycol)", "/mnt/analysis/", "$(string(ycol))_$(string(name))_logpdf_impute")
            DataFrames.insertcols!(stn_df, 3, Symbol(ycol, "_impute") => ycol_arr)
            DataFrames.insertcols!(stn_df, 3, Symbol(ycol, "_logimpute") => log10.(ycol_arr))

            # Analysis : HypothesisTests
            @info "Hypothesis Tests"
            max_lag = 30 * 24
            #@show HypothesisTests.ADFTest(values(ta), :none, max_lag)
            #@show HypothesisTests.ADFTest(values(ta), :constant, max_lag)
            #@show HypothesisTests.ADFTest(values(ta), :trend, max_lag)
            #@show HypothesisTests.ADFTest(values(ta), :squared_trend, max_lag)
            #@show HypothesisTests.LjungBoxTest(df[!, ycol], 365*24)

            # Analysis : Autocorrelation Functions]
            #=
            for _ycol in [ycol, Symbol(ycol, :_impute), Symbol(ycol, :_logimpute)]
                Base.Filesystem.mkpath("/mnt/analysis/$(string(_ycol))/")
                #max_time = Int(ceil(min(size(stn_df[!, ycol],1)-1, 10*log10(size(stn_df[!, ycol],1)))))
                max_time = 15 * 24
                @info "Autocorrelation..."
                lags = 0:max_time
                acf = StatsBase.autocor(stn_df[!, _ycol], lags)
                plot_anal_correlogram(acf, _ycol,
                    "Autocorrelation",
                    "/mnt/analysis/$(string(_ycol))/", "$(string(ycol))_$(string(name))_mt$(max_time)_acf")
                acf_df = DataFrame(time = collect(0:max_time), corr = acf)
                CSV.write("/mnt/analysis/$(string(_ycol))/$(string(ycol))_$(string(name))_mt$(max_time)_acf.csv", acf_df)
                @info "Integral Time Scale for raw data..", compute_inttscale(0:max_time, acf)

                pacf = StatsBase.pacf(stn_df[!, _ycol], lags)
                plot_anal_correlogram(pacf, _ycol,
                    "Partial Autocorrelation",
                    "/mnt/analysis/$(string(_ycol))/", "$(string(ycol))_$(string(name))_mt$(max_time)_pacf")

                # Analysis : Autocorrelation Functions
                # No need to compute pacf
                max_time = 365 * 24
                lags = 0:max_time
                acf = StatsBase.autocor(stn_df[!, _ycol], lags)
                plot_anal_correlogram(acf, ycol,
                    "Autocorrelation",
                    "/mnt/analysis/$(string(_ycol))/", "$(string(ycol))_$(string(name))_mt$(max_time)_acf")
                acf_df = DataFrame(time = collect(0:max_time), corr = acf)
                CSV.write("/mnt/analysis/$(string(_ycol))/$(string(ycol))_$(string(name))_mt$(max_time)_acf.csv", acf_df)
                @info "Integral Time Scale for raw data..", compute_inttscale(0:max_time, acf)
            end
            =#
            #=
            # Analysis : Integral Length Scale (Total)
            int_scale = integrate(1:length(acf), acf, SimpsonEven())
            @info "Integral Time Scale of $(ycol) at $(name) (total): ", int_scale

            plot_anal_autocor(acf, ycol,
                "Autocorrelation (total series)",
                "/mnt/analysis/", "$(string(ycol))_$(string(name))_total")
            =#
            # Analysis : Time difference
            #=
            @info "Time difference..."
            order = 1
            lag = 1
            diffed = TimeSeries.diff(ta, lag, padding=true, differences=order)
            val_diffed = copy(values(diffed))
            replace!(val_diffed, Inf => 0, -Inf => 0, NaN => 0)
            @info "mean of time $(order)th difference of $(ycol) at $(name): ", StatsBase.mean(val_diffed)
            @info "std  of time $(order)th difference of $(ycol) at $(name): ", StatsBase.std(val_diffed)
            plot_anal_lineplot(DateTime.(timestamp(diffed)), val_diffed, ycol,
                order, lag, "/mnt/analysis/", "$(string(ycol))_$(string(name))")

            order = 2
            lag = 1
            diffed = TimeSeries.diff(ta, lag, padding=true, differences=order)
            val_diffed = copy(values(diffed))
            replace!(val_diffed, Inf => 0, -Inf => 0, NaN => 0)
            @info "mean of time $(order)th difference of $(ycol) at $(name): ", StatsBase.mean(val_diffed)
            @info "std  of time $(order)th difference of $(ycol) at $(name): ", StatsBase.std(val_diffed)
            plot_anal_lineplot(DateTime.(timestamp(diffed)), val_diffed, ycol,
                order, lag, "/mnt/analysis/", "$(string(ycol))_$(string(name))")

            order = 1
            lag = 24
            diffed = TimeSeries.diff(ta, lag, padding=true, differences=order)
            val_diffed = copy(values(diffed))
            replace!(val_diffed, Inf => 0, -Inf => 0, NaN => 0)
            @info "mean of time $(order)th difference of $(ycol) at $(name): ", StatsBase.mean(val_diffed)
            @info "std  of time $(order)th difference of $(ycol) at $(name): ", StatsBase.std(val_diffed)
            plot_anal_lineplot(DateTime.(timestamp(diffed)), val_diffed, ycol,
                order, lag, "/mnt/analysis/", "$(string(ycol))_$(string(name))")
            =#
            # Analysis : Grouping by time
            @info "Groupby time..."
            #=
            for _ycol in [ycol, Symbol(ycol, :_impute), Symbol(ycol, :_logimpute)]
                Base.Filesystem.mkpath("/mnt/analysis/$(string(_ycol))/")
                period_df = copy(stn_df)

                # find periodicty
                pfuncs = (
                    hour = :hour,
                    week = :week,
                    month = :month)

                # Analysis : seasonality analysis, hourly, weekly, monthly
                for period_dir in [:hour, :week, :month]
                    @info "Periodic mean of $(string(_ycol)) ($(string(period_dir))ly...) "
                    pfunc = pfuncs[period_dir]
                    periodic_col = eval(pfunc).(period_df[!, :date])
                    DataFrames.insertcols!(period_df, 3, period_dir => periodic_col)

                    periodic_means, periodic_means_repeated = populate_periodic_mean(period_df, _ycol, period_dir)
                    # insert mean and fluctuation to DataFrame
                    DataFrames.insertcols!(period_df, 3, Symbol(period_dir, "_mean") => periodic_means_repeated)
                    DataFrames.insertcols!(period_df, 3, Symbol(period_dir, "_fluc") => period_df[!, ycol] .- periodic_means_repeated)

                    # violin plot mean
                    plot_anal_violin(period_df, _ycol, period_dir, periodic_means,
                        "Violin plot for $(ycol)", "/mnt/analysis/$(string(_ycol))/", "$(string(ycol))_$(string(name))")
                    # plot mean only
                    plot_anal_periodic_mean(period_df, _ycol, period_dir, periodic_means,
                        "$(uppercasefirst(string(period_dir)))ly mean for $(ycol)", "/mnt/analysis/$(string(_ycol))/", "$(string(ycol))_$(string(name))_time_mean")
                end

                # Analysis : fluctuation analysis, hourly, weekly, monthly
                # Use Split-Apply-Combine Strategy
                for period_dir in [:hour, :week, :month]
                    @info "Autocorrelation of periodic fluctuations of $(string(_ycol)) $(string(period_dir))ly..."

                    period_fluc_df = DataFrames.by(period_df, _ycol) do _df
                        (; period_dir => getproperty(_df, period_dir),
                        :date => _df.date,
                        :mean => getproperty(_df, Symbol(period_dir, "_mean")),
                        :fluc => getproperty(_df, _ycol) - getproperty(_df, Symbol(period_dir, "_mean")))
                    end

                    period_fluc_means = periodic_mean(period_fluc_df, _ycol, period_dir)
                    plot_anal_periodic_mean(period_fluc_df, _ycol, period_dir, period_fluc_means,
                        "$(uppercasefirst(string(period_dir)))ly fluctuation mean for $(ycol)", "/mnt/analysis/$(string(_ycol))/",
                        "$(string(ycol))_$(string(name))_$(string(_ycol))_period_flucmean")

                    #max_time = Int(ceil(min(size(period_fluc_df[!, _ycol],1)-1, 10*log10(size(period_fluc_df[!, _ycol],1)))))
                    max_time = 15 * 24
                    lags = 0:max_time
                    @info "$(string(period_dir))ly Fluctuation Autocorrelation..."
                    acf = StatsBase.autocor(period_fluc_df[!, :fluc], lags)
                    plot_anal_correlogram(acf, _ycol,
                        "$(uppercasefirst(string(period_dir)))ly Fluctuation Autocorrelation",
                        "/mnt/analysis/$(string(_ycol))/",
                        "$(string(ycol))_$(string(name))_mt$(max_time)_fluc_$(string(period_dir))ly_acf")

                    max_time = 365 * 24
                    lags = 0:max_time
                    acf = StatsBase.autocor(period_fluc_df[!, :fluc], lags)
                    plot_anal_correlogram(acf, _ycol,
                        "$(uppercasefirst(string(period_dir)))ly Fluctuation Autocorrelation",
                        "/mnt/analysis/$(string(_ycol))/",
                        "$(string(ycol))_$(string(name))_mt$(max_time)_fluc_$(string(period_dir))ly_acf")
                end
            end
            =#
            # decompose seasonal components
            Base.Filesystem.mkpath("/mnt/analysis/decomposition/$(string(ycol))/")
            #lee_year_sea1, lee_year_sea2, lee_year_res2, lee_day_sea1, lee_day_res1 =
            #    season_adj_lee(stn_df, Symbol(ycol, "_impute"))

            decomposed_df = decompose_seasonality(stn_df, Symbol(ycol, "_impute"))
            CSV.write("/mnt/analysis/decomposition/$(string(ycol))/" * "$(string(ycol))_$(string(name))_new_sea_decompose.csv", decomposed_df)
            plot_year = 2016
            plot_seasonality(ycols, decomposed_df, ycol, plot_year, DateTime(train_fdate, Local),
                "Seasonal Decomposition in $(plot_year)",
                "/mnt/analysis/decomposition/$(string(ycol))/", "$(string(ycol))_$(string(name))_new_sea_decompose")
            #=
            plot_seasonality(ycols, lee_year_sea1, lee_year_sea2, lee_day_sea1, lee_day_res1, ycol, plot_year,
                DateTime(stn_df[1, :date], Local),
                ["Original data", "Annual seasonality", "Daily seasonality", "Residual"],
                "Seasonal Decomposition in $(plot_year)",
                "/mnt/analysis/decomposition/$(string(ycol))/", "$(string(ycol))_$(string(name))_seasonality")
            plot_seasonality(ycols, lee_year_sea1, lee_year_sea2, lee_day_sea1, lee_day_res1, ycol, 2015, 2017,
                DateTime(stn_df[1, :date], Local),
                ["Original data", "Annual seasonality", "Daily seasonality", "Residual"],
                "Seasonal Decomposition from 2015 to 2017",
                "/mnt/analysis/decomposition/$(string(ycol))/", "$(string(ycol))_$(string(name))_seasonality_3year")
            =#
            # Autocorrelation
            max_time = 15*24
            acf_raw = StatsBase.autocor(decomposed_df[!, :raw], 0:max_time)
            acf_yres2_s = StatsBase.autocor(decomposed_df[!, :year_res_s], 0:max_time)
            acf_yres2 = StatsBase.autocor(decomposed_df[!, :year_res], 0:max_time)
            acf_dres1 = StatsBase.autocor(decomposed_df[!, :day_res], 0:max_time)
            #acf_dres1_log = StatsBase.autocor(log.(lee_day_res1), 0:15*24)

            plot_anal_correlogram(acf_raw, ycol,
                "Raw Autocorrelation",
                "/mnt/analysis/decomposition/$(string(ycol))/",
                "$(string(ycol))_$(string(name))_mt$(max_time)_acf_raw")
            plot_anal_correlogram(acf_yres2_s, ycol,
                "Annual Residual (Smoothed) Autocorrelation",
                "/mnt/analysis/decomposition/$(string(ycol))/",
                "$(string(ycol))_$(string(name))_mt$(max_time)_acf_yres2_s")
            plot_anal_correlogram(acf_yres2, ycol,
                "Annual Residual (Raw) Autocorrelation",
                "/mnt/analysis/decomposition/$(string(ycol))/",
                "$(string(ycol))_$(string(name))_mt$(max_time)_acf_yres2")
            plot_anal_correlogram(acf_dres1, ycol,
                "Daily Residual Autocorrelation",
                "/mnt/analysis/decomposition/$(string(ycol))/",
                "$(string(ycol))_$(string(name))_mt$(max_time)_acf_dres1")
            acf_df = DataFrame(time = collect(0:max_time), corr = acf_yres2)

            Base.mkpath("/mnt/analysis/$(string(ycol))")
            CSV.write("/mnt/analysis/$(string(ycol))/$(string(ycol))_$(string(name))_mt$(max_time)_acf_yres2.csv", acf_df)

            intT = compute_inttscale(0:max_time, acf_yres2)
            @info "Integral Time Scale for Annual Residual..", intT
            # not to log negative value when fitting
            positiveIntT = max(1, Int(round(intT)) - 1)
            fit = exp_fit(acf_df[1:positiveIntT, :time], acf_df[1:positiveIntT, :corr])
            @info "Coef a and b in a * exp(b * x) for Annual Residual.. to 0:$(positiveIntT)", fit[1], fit[2]

            fit = exp_fit(acf_df[1:positiveIntT, :time], acf_df[1:positiveIntT, :corr])
            fit_x = acf_df[1:positiveIntT, :time]
            fit_y = fit[1] .* exp.(fit_x .* fit[2])
            intT = compute_inttscale(fit_x, fit_y)
            @info "Integral Time Scale for Fitted Annual Residual..", intT
            @info "Coef a and b in a * exp(b * x / T) for Fitted Annual Residual.. to 0:$(positiveIntT)", fit[1], fit[2] / intT

            acf_df = DataFrame(time = collect(0:max_time), corr = acf_dres1)
            CSV.write("/mnt/analysis/$(string(ycol))/$(string(ycol))_$(string(name))_mt$(max_time)_acf_dres1.csv", acf_df)
            intT = compute_inttscale(0:max_time, acf_dres1)
            @info "Integral Time Scale for Daily Residual..", intT
            positiveIntT = max(1, Int(round(intT)) - 1)
            fit = exp_fit(acf_df[1:positiveIntT, :time], acf_df[1:positiveIntT, :corr])
            @info "Coef a and b in a * exp(b * T) for Daily Residual.. to 0:$(positiveIntT)", fit[1], fit[2]

            fit = exp_fit(acf_df[1:positiveIntT, :time], acf_df[1:positiveIntT, :corr])
            fit_x = acf_df[1:positiveIntT, :time]
            fit_y = fit[1] .* exp.(fit_x .* fit[2])
            intT = compute_inttscale(fit_x, fit_y)
            @info "Integral Time Scale for Fitted Daily Residual..", intT
            @info "Coef a and b in a * exp(b * x / T) for Fitted Daily Residual.. to 0:$(positiveIntT)", fit[1], fit[2] / intT

            max_time = 365*24
            acf_yres2_long = StatsBase.autocor(decomposed_df[!, :year_res], 0:max_time)
            acf_dres1_long = StatsBase.autocor(decomposed_df[!, :day_res], 0:max_time)
            plot_anal_correlogram(acf_yres2_long, ycol,
                "Annual Residual Autocorrelation",
                "/mnt/analysis/decomposition/$(string(ycol))/",
                "$(string(ycol))_$(string(name))_mt$(max_time)_acf_yres2_long")
            plot_anal_correlogram(acf_dres1_long, ycol,
                "Daily Residual Autocorrelation",
                "/mnt/analysis/decomposition/$(string(ycol))/",
                "$(string(ycol))_$(string(name))_mt$(max_time)_acf_dres1_long")
            acf_df = DataFrame(time = collect(0:max_time), corr = acf_yres2_long)
            CSV.write("/mnt/analysis/$(string(ycol))/$(string(ycol))_$(string(name))_mt$(max_time)_acf_yres2.csv", acf_df)
            intT = compute_inttscale(0:max_time, acf_yres2_long)
            @info "Integral Time Scale for Annual Residual..", intT
            acf_df = DataFrame(time = collect(0:max_time), corr = acf_dres1_long)
            CSV.write("/mnt/analysis/$(string(ycol))/$(string(ycol))_$(string(name))_mt$(max_time)_acf_dres1.csv", acf_df)
            intT = compute_inttscale(0:max_time, acf_dres1_long)
            @info "Integral Time Scale for Daily Residual..", compute_inttscale(0:max_time, acf_dres1_long)
        end
    end

end

run_analysis()


# This is used to store experiment details
abstract type AbstractExperiment end

mutable struct Experiment <: AbstractExperiment
    data::DataFrame
    description::String
    selected_features::Array{Int64,1}
    selected_entries::Array{Int64,1}
end

# Constructor
# WARNING: data is not copied so any transformation of the Experiment's data
# will modify the original `data` DataFrame fed to the constructor.
function Experiment(data; description = "No description provided")
    return Experiment(data, description, 1:ncol(data), 1:nrow(data))
end

function Base.show(io::IO, e::Experiment)
    compact = get(io, :compact, false)

    if compact
        show(io, "Exp.: "*string(length(e.selected_entries))*"/"*string(nrow(e.data))*
             "-"*string(length(e.selected_features))*"/"*string(ncol(e.data)))
    else
        show(io, "Experiment with "*string(length(e.selected_entries))*"/"*string(nrow(e.data))*
             " entries and "*string(length(e.selected_features))*"/"*string(ncol(e.data))*
             " features selected.")
    end
end

abstract type AbstractReduce end
abstract type AbstractFilter <: AbstractReduce end
abstract type AbstractSimpleFilter <: AbstractFilter end
abstract type AbstractCombinationFilter <: AbstractFilter end
abstract type AbstractMissingFilter <: AbstractFilter end

mutable struct Filter <: AbstractSimpleFilter
    value::Any
    feature::Symbol
    compare::Function
    description::String
end

# Constructor
function Filter(value, feature;
                compare = isequal, description = "No description provided")
    return Filter(value, feature, compare, description)
end

mutable struct CombinationFilter <: AbstractCombinationFilter
    filter1::AbstractFilter
    filter2::AbstractFilter
    operator::Function
end

mutable struct MissingFilter <: AbstractMissingFilter end

# Constructor
function MissingFilter(feature;
                        description = "No description provided")
    return Filter(1, feature, (x,y) -> !ismissing(x), description)
end

# Methods

"""Return filtered entries in an Experiment `e` based on filter `f`
"""
function filter_entries(e::AbstractExperiment, f::AbstractSimpleFilter)
    expEntries = e.data[e.selected_entries, f.feature]
    return(e.selected_entries[f.compare.(expEntries, f.value)])
end

function filter_entries(e::AbstractExperiment, f::AbstractCombinationFilter)
    e1 = filter_entries(e, f.filter1)
    e2 = filter_entries(e, f.filter2)
    return(sort(f.operator(e1, e2)))
end

function filter_entries(e::AbstractExperiment, f::AbstractMissingFilter)
    return(sort(∩([filter_entries(e, MissingFilter(f))
                   for f in Symbol.(names(e.data)[e.selected_features])]...)))
end

"""Filter entries in an Experiment `e` based on filter(s) `f`,
updating `e.selected_entries` in place accordingly.
"""
function filter_entries!(e::AbstractExperiment, f::AbstractFilter)
    # Currently returns the indices kept
    e.selected_entries = filter_entries(e,f)
end

function filter_entries!(e::AbstractExperiment, filters::Array{T,1}) where {T<:AbstractFilter}
    for f in filters
        filter_entries!(e, f)
    end
end

abstract type AbstractReduce end
abstract type AbstractSelector <: AbstractReduce end
abstract type AbstractSimpleSelector <: AbstractSelector end
abstract type AbstractNameSelector <: AbstractSelector end
abstract type AbstractCombinationSelector <: AbstractSelector end

mutable struct Selector <: AbstractSimpleSelector
    summarize::Function
    subset::Union{Function, Nothing}
    description::String
end

# Constructor
function Selector(summarize; subset = nothing,
                  description = "No description provided")
    return Selector(summarize, subset, description)
end

mutable struct NameSelector <: AbstractNameSelector
    summarize::Function
    description::String
end

# Constructor
function NameSelector(summarize; description = "No description provided")
    return NameSelector(summarize, description)
end

mutable struct CombinationSelector <: AbstractCombinationSelector
    selector1::AbstractSelector
    selector2::AbstractSelector
    operator::Function
end

# Methods

"""Return selected features in an Experiment `e` based on selectors `s`
"""
function select_features(e::AbstractExperiment, s::AbstractSimpleSelector)
    # NB: isnothing was not implemented in 1.0
    if s.subset === nothing
        data = e.data[e.selected_entries, e.selected_features]
    else
        subIndices = s.subset(e.data[e.selected_entries,:])
        data = e.data[e.selected_entries[subIndices], e.selected_features]
    end
    selectedFtDF = mapcols(s.summarize, data)
    return(e.selected_features[[x for x in selectedFtDF[1,:]]])
end

function select_features(e::AbstractExperiment, s::AbstractNameSelector)
    selectedFtDF = map(s.summarize, names(e.data[:,e.selected_features]))
    return(e.selected_features[selectedFtDF])
end

function select_features(e::AbstractExperiment, s::AbstractCombinationSelector)
    f1 = select_features(e, s.selector1)
    f2 = select_features(e, s.selector2)
    return(sort(s.operator(f1, f2)))
end

"""Return selected features in an Experiment `e` based on selectors `s`,
updating `e.selected_features` in place accordingly.
"""
function select_features!(e::AbstractExperiment, s::AbstractSelector)
    # Currently returns the indices kept
    e.selected_features = select_features(e,s)
end

function select_features!(e::AbstractExperiment, selectors::Array{T,1}) where {T<:AbstractSelector}
    for s in selectors
        select_features!(e, s)
    end
end

# Not exported in favor of the shorter "filter!" and "select!"
"""For an experiment `e`, update in place `e.selected_features` and
`e.selected_entries` based on an array `arr` of feature selectors and
entry filters. Filters and selectors are applied sequentially.
"""
function filter_experiment!(e::AbstractExperiment, arr::Array{T,1}) where {T<:AbstractReduce}
    for a in arr
        filter!(e, a)
    end
end

"""For an experiment `e`, update in place `e.selected_features` and
`e.selected_entries` based on a feature selector `s`.
"""
function filter_experiment!(e::AbstractExperiment, s::AbstractSelector)
    select_features!(e,s)
end

"""For an experiment `e`, update in place `e.selected_features` and
`e.selected_entries` based on an entry filter `f`.
"""
filter_experiment!(e::AbstractExperiment, f::AbstractFilter) = filter_entries!(e,f)

# Aliases
@doc (@doc filter_experiment!)
Base.filter!(e::AbstractExperiment,
                   arr::Array{T,1}) where {T<:AbstractReduce} = filter_experiment!(e,arr)
Base.filter!(e::AbstractExperiment, s::AbstractSelector) = filter_experiment!(e,s)
Base.filter!(e::AbstractExperiment, f::AbstractFilter) = filter_experiment!(e,f)

@doc (@doc filter_experiment!)
DataFrames.select!(e::AbstractExperiment,
                   arr::Array{T,1}) where {T<:AbstractReduce} = filter_experiment!(e,arr)
DataFrames.select!(e::AbstractExperiment, s::AbstractSelector) = filter_experiment!(e,s)
DataFrames.select!(e::AbstractExperiment, f::AbstractFilter) = filter_experiment!(e,f)

"""Return a negative Filter or Selector by inverting
the entries or features that are kept and excluded.
"""
function negation(r::Union{AbstractNameSelector,AbstractSimpleSelector})
    neg_r = deepcopy(r)
    neg_r.summarize = !neg_r.summarize
    neg_r.description = "Do not "*neg_r.description
    return(neg_r)
end

function negation(r::AbstractSimpleFilter)
    neg_r = deepcopy(r)
    neg_r.compare = !neg_r.compare
    neg_r.description = "Do not "*neg_r.description
    return(neg_r)
end

"""Return a copy of the data in Experiment `e` for its
selected entries and features.
"""
function getdata(e::Experiment)
    return(e.data[e.selected_entries, e.selected_features])
end


# set current limit
# SI
export psp_setilimit

"""
    psp_setilimit(io, current_limit)

Set the current limit.
"""
function psp_setilimit(io_psp::IO,ilim::Real)
  @assert ilim<10 "current limit must be < 10A"
  ilimstring = @sprintf("%01.2f",ilim)
  set_current_limit_string = "SI "*ilimstring*"\r"
  write(io_psp, set_current_limit_string)
  return nothing
end


module Stations

using Utils
import Utils.GeoLoc

using Calendar
import Calendar.CalendarTime

import Base.show

abstract AbstractStation

type Observation

    # station of origin
    station::AbstractStation

    # observation time
    tm::CalendarTime

    # observed value
    value::Float64

    # name of the variable
    obs_type::String

    # variance of the observation
    var :: Float64

    # default constructor
    Observation(s, tm, value, obs_type, var) = new(s, tm, value, obs_type, var)

end

type Station <: AbstractStation

    # station id
    id::String

    # station name
    name::String

    # station location (lat/lon decimal)
    loc::GeoLoc

    # nearest grid point (tuple)
    ngp::(Int64, Int64)

    # station elevation (meters)
    elevation::Float64

    # dictionary mapping variable names to observations
    obs::Dict{String,Array{Observation}}

    # available observation types from the station
    sensors :: Array{String}

    Station() = new("", "", GeoLoc(0.0, 0.0), (-1,-1), 0.0,  Dict{String,Array{Observation}}(), Array(String,0))

    Station(id, name, loc, elevation) = new(id, name, loc, (-1,-1), elevation, Dict{String,Array{Observation}}(), Array(String,0))

end

obs_variance(o::Observation) = o.var
obs_value(o::Observation) = o.value
obs_type(o::Observation) = o.obs_type
obs_station(o::Observation) = o.station
obs_station_id(o::Observation) = o.station.id
nearest_grid_point(o::Observation) = o.station.ngp

function show(io::IO, o::Observation)
#    write(io, "Obs($o.value of $o.obs_type at $o.station.id, $o.tm with var $o.var)")
    print(io, "Obs(", o.obs_type, "=", o.value, " at ", o.station.id, ", ", o.tm, " with var ", o.var, ")")
end

obs_times(s::Station, obs_type::String) = [o.tm for o in observations(s, obs_type)]
obs_var(s::Station, obs_type::String) = s.obs[obs_type]
id(s::Station) = s.id
name(s::Station) = s.name
observations(s::Station, obs_type::String) = s.obs[obs_type]

function register_to_grid(s::Station, lat::Array{Float64,2}, lon::Array{Float64,2})
    s.ngp = ind2sub(size(lat), indmin( (s.loc.lat - lat).^2 + (s.loc.lon - lon).^2 ))
end

function observations(s::Station, obs_type::String, tm::Array{CalendarTime})

    # retrieve all the observations for the given type
    obs = s.obs[obs_type]
    otm = [o.tm for o in obs]
    obs_ret = Dict(CalendarTime, Observation)

    i, j = 1, 1
    while (i <= length(tm)) && (j <= length(otm))
        if tm[i] == otm[j]
            obs_ret[tm[i]] = obs[i]
            i += 1
            j += 1
        elseif tm[i] > obst[j]
            j += 1
        elseif tm[i] < obst[j]
            j += 1
        end
    end

    return obs_ret
end

function load_station_info(fname :: String)

    s = Station()

    # open file
    io = open(fname, "r")

    # read station id
    s.id = strip(readline_skip_comments(io))

    # read station name
    s.name = strip(readline_skip_comments(io))

    # read station geo location
    s.loc = Utils.parse_geolocation(readline_skip_comments(io))

    # read elevation
    s.elevation = float(strip(readline_skip_comments(io)))

    # read all observation types acquired by the station
    s.sensors = map(x -> strip(x), split(readline_skip_comments(io), ","))

    # create var time series containers
    for v in s.sensors
        s.obs[v] = Array(Float64, 0)
    end

    # close the info file
    close(io)

    return s
end

function load_station_data(s::Station, fname::String)
    # open file
    io = open(fname, "r")

    # read observations, time is first (and in GMT), then one value per measurement or nan if not available
    while !eof(io)

        # read the date (if cannot read date, file is empty)
        tm_line = readline_skip_comments(io)
        if tm_line == nothing  break end
        tm_str = strip(tm_line)
        tm = Calendar.parse("yyyy-MM-dd_HH:mm", tm_str, "GMT")

        # read in observed variables
        variables = map(x -> strip(x), split(readline_skip_comments(io), ","))

        # read in observed values
        vals = map(x -> float(x), split(readline_skip_comments(io), ","))

        # read in variances of observations
        variances = map(x -> float(x), split(readline_skip_comments(io), ","))

        # add each observation to the station dictionary
        for i in 1:length(variables)
            var = variables[i]
            push!(s.obs[var], Observation(s, tm, vals[i], var, variances[i]))
        end

    end

    # cleanup
    close(io)

    # return the created station
    return s

end

function build_observation_data(ss::Array{Station}, obs_type::String)
    Ns = length(ss)
    obs = Observation[]

    # observation data
    for s in ss
        append!(obs, observations(s, obs_type))
    end

    # repackage into a time-indexed structure
    obs_data = Dict{CalendarTime,Array{Observation}}()
    for o in obs
        if haskey(obs_data, o.tm) push!(obs_data[o.tm], o) else obs_data[o.tm] = [o] end
    end

    return obs_data
end

function readline_skip_comments(io :: IO)
    s = "#"
    while (length(s) == 0) || (s[1] == '#')
        if !eof(io)
            s = readline(io)
        else
            return nothing
        end
    end
    return s
end

end


#
#  Split - Apply - Combine operations
#

##############################################################################
##
## GroupedDataFrame...
##
##############################################################################

"""
The result of a `groupby` operation on an AbstractDataFrame; a
view into the AbstractDataFrame grouped by rows.

Not meant to be constructed directly, see `groupby`.
"""
type GroupedDataFrame
    parent::AbstractDataFrame
    cols::Vector         # columns used for sorting
    idx::Vector{Int}     # indexing vector when sorted by the given columns
    starts::Vector{Int}  # starts of groups
    ends::Vector{Int}    # ends of groups
end

#
# Split
#

function groupsort_indexer(x::AbstractVector, ngroups::Integer, null_last::Bool=false)
    # translated from Wes McKinney's groupsort_indexer in pandas (file: src/groupby.pyx).

    # count group sizes, location 0 for NULL
    n = length(x)
    # counts = x.pool
    counts = fill(0, ngroups + 1)
    for i = 1:n
        counts[x[i] + 1] += 1
    end

    # mark the start of each contiguous group of like-indexed data
    where = fill(1, ngroups + 1)
    if null_last
        for i = 3:ngroups+1
            where[i] = where[i - 1] + counts[i - 1]
        end
        where[1] = where[end] + counts[end]
    else
        for i = 2:ngroups+1
            where[i] = where[i - 1] + counts[i - 1]
        end
    end

    # this is our indexer
    result = fill(0, n)
    for i = 1:n
        label = x[i] + 1
        result[where[label]] = i
        where[label] += 1
    end
    result, where, counts
end

"""
A view of an AbstractDataFrame split into row groups

```julia
groupby(d::AbstractDataFrame, cols)
groupby(cols)
```

### Arguments

* `d` : an AbstractDataFrame to split (optional, see [Returns](#returns))
* `cols` : data frame columns to group by

### Returns

* `::GroupedDataFrame` : a grouped view into `d`
* `::Function`: a function `x -> groupby(x, cols)` (if `d` is not specified)

### Details

An iterator over a `GroupedDataFrame` returns a `SubDataFrame` view
for each grouping into `d`. A `GroupedDataFrame` also supports
indexing by groups and `map`.

See the following for additional split-apply-combine operations:

* `by` : split-apply-combine using functions
* `aggregate` : split-apply-combine; applies functions in the form of a cross product
* `combine` : combine (obviously)
* `colwise` : apply a function to each column in an AbstractDataFrame or GroupedDataFrame

Piping methods `|>` are also provided.

See the
[DataFramesMeta](https://github.com/JuliaStats/DataFramesMeta.jl)
package for more operations on GroupedDataFrames.

### Examples

```julia
df = DataFrame(a = repeat([1, 2, 3, 4], outer=[2]),
               b = repeat([2, 1], outer=[4]),
               c = randn(8))
gd = groupby(df, :a)
gd[1]
last(gd)
vcat([g[:b] for g in gd]...)
for g in gd
    println(g)
end
map(d -> mean(dropnull(d[:c])), gd)   # returns a GroupApplied object
combine(map(d -> mean(dropnull(d[:c])), gd))
df |> groupby(:a) |> [sum, length]
df |> groupby([:a, :b]) |> [sum, length]
```

"""
function groupby{T}(d::AbstractDataFrame, cols::Vector{T})
    ## a subset of Wes McKinney's algorithm here:
    ##     http://wesmckinney.com/blog/?p=489

    ncols = length(cols)
    # use CategoricalArray to get a set of integer references for each unique item
    nv = NullableCategoricalArray(d[cols[ncols]])
    # if there are NULLs, add 1 to the refs to avoid underflows in x later
    anynulls = (findfirst(nv.refs, 0) > 0 ? 1 : 0)
    # use UInt32 instead of the original array's integer size since the number of levels can be high
    x = similar(nv.refs, UInt32)
    for i = 1:nrow(d)
        if nv.refs[i] == 0
            x[i] = 1
        else
            x[i] = CategoricalArrays.order(nv.pool)[nv.refs[i]] + anynulls
        end
    end
    # also compute the number of groups, which is the product of the set lengths
    ngroups = length(levels(nv)) + anynulls
    # if there's more than 1 column, do roughly the same thing repeatedly
    for j = (ncols - 1):-1:1
        nv = NullableCategoricalArray(d[cols[j]])
        anynulls = (findfirst(nv.refs, 0) > 0 ? 1 : 0)
        for i = 1:nrow(d)
            if nv.refs[i] != 0
                x[i] += (CategoricalArrays.order(nv.pool)[nv.refs[i]] + anynulls - 1) * ngroups
            end
        end
        ngroups = ngroups * (length(levels(nv)) + anynulls)
        # TODO if ngroups is really big, shrink it
    end
    (idx, starts) = groupsort_indexer(x, ngroups)
    # Remove zero-length groupings
    starts = _uniqueofsorted(starts)
    ends = starts[2:end] - 1
    GroupedDataFrame(d, cols, idx, starts[1:end-1], ends)
end
groupby(d::AbstractDataFrame, cols) = groupby(d, [cols])

# add a function curry
groupby{T}(cols::Vector{T}) = x -> groupby(x, cols)
groupby(cols) = x -> groupby(x, cols)

Base.start(gd::GroupedDataFrame) = 1
Base.next(gd::GroupedDataFrame, state::Int) =
    (sub(gd.parent, gd.idx[gd.starts[state]:gd.ends[state]]),
     state + 1)
Base.done(gd::GroupedDataFrame, state::Int) = state > length(gd.starts)
Base.length(gd::GroupedDataFrame) = length(gd.starts)
Base.endof(gd::GroupedDataFrame) = length(gd.starts)
Base.first(gd::GroupedDataFrame) = gd[1]
Base.last(gd::GroupedDataFrame) = gd[end]

Base.getindex(gd::GroupedDataFrame, idx::Int) =
    sub(gd.parent, gd.idx[gd.starts[idx]:gd.ends[idx]])
Base.getindex(gd::GroupedDataFrame, I::AbstractArray{Bool}) =
    GroupedDataFrame(gd.parent, gd.cols, gd.idx, gd.starts[I], gd.ends[I])

Base.names(gd::GroupedDataFrame) = names(gd.parent)
_names(gd::GroupedDataFrame) = _names(gd.parent)

##############################################################################
##
## GroupApplied...
##    the result of a split-apply operation
##    TODOs:
##      - better name?
##      - ref
##      - keys, vals
##      - length
##      - start, next, done -- should this return (k,v) or just v?
##      - make it a real associative type? Is there a need to look up key columns?
##
##############################################################################

"""
The result of a `map` operation on a GroupedDataFrame; mainly for use
with `combine`

Not meant to be constructed directly, see `groupby` abnd
`combine`. Minimal support is provided for this type. `map` is
provided for a GroupApplied object.

"""
immutable GroupApplied{T<:AbstractDataFrame}
    gd::GroupedDataFrame
    vals::Vector{T}

    @compat function (::Type{GroupApplied})(gd::GroupedDataFrame, vals::Vector)
        length(gd) == length(vals) ||
            throw(DimensionMismatch("GroupApplied requires keys and vals be of equal length (got $(length(gd)) and $(length(vals)))."))
        new{eltype(vals)}(gd, vals)
    end
end

#
# Apply / map
#

# map() sweeps along groups
function Base.map(f::Function, gd::GroupedDataFrame)
    GroupApplied(gd, [wrap(f(df)) for df in gd])
end
function Base.map(f::Function, ga::GroupApplied)
    GroupApplied(ga.gd, [wrap(f(df)) for df in ga.vals])
end

wrap(df::AbstractDataFrame) = df
wrap(A::Matrix) = convert(DataFrame, A)
wrap(s::Any) = DataFrame(x1 = s)

"""
Combine a GroupApplied object (rudimentary)

```julia
combine(ga::GroupApplied)
```

### Arguments

* `ga` : a GroupApplied

### Returns

* `::DataFrame`

### Examples

```julia
df = DataFrame(a = repeat([1, 2, 3, 4], outer=[2]),
               b = repeat([2, 1], outer=[4]),
               c = randn(8))
combine(map(d -> mean(dropnull(d[:c])), gd))
```

"""
function combine(ga::GroupApplied)
    gd, vals = ga.gd, ga.vals
    valscat = vcat(vals)
    idx = Vector{Int}(size(valscat, 1))
    j = 0
    @inbounds for (start, val) in zip(gd.starts, vals)
        n = size(val, 1)
        idx[j + (1:n)] = gd.idx[start]
        j += n
    end
    hcat!(gd.parent[idx, gd.cols], valscat)
end

"""
Apply a function to each column in an AbstractDataFrame or
GroupedDataFrame

```julia
colwise(f::Function, d)
colwise(d)
```

### Arguments

* `f` : a function or vector of functions
* `d` : an AbstractDataFrame of GroupedDataFrame

If `d` is not provided, a curried version of groupby is given.

### Returns

* various, depending on the call

### Examples

```julia
df = DataFrame(a = repeat([1, 2, 3, 4], outer=[2]),
               b = repeat([2, 1], outer=[4]),
               c = randn(8))
colwise(sum, df)
colwise(sum, groupby(df, :a))
```

"""
colwise(f::Function, d::AbstractDataFrame) = Any[vcat(f(d[idx])) for idx in 1:size(d, 2)]
colwise(f::Function, gd::GroupedDataFrame) = map(colwise(f), gd)
colwise(f::Function) = x -> colwise(f, x)
colwise(f) = x -> colwise(f, x)
# apply several functions to each column in a DataFrame
colwise{T<:Function}(fns::Vector{T}, d::AbstractDataFrame) =
    reshape(Any[vcat(f(d[idx])) for f in fns, idx in 1:size(d, 2)],
            length(fns)*size(d, 2))
colwise{T<:Function}(fns::Vector{T}, gd::GroupedDataFrame) = map(colwise(fns), gd)
colwise{T<:Function}(fns::Vector{T}) = x -> colwise(fns, x)

"""
Split-apply-combine in one step; apply `f` to each grouping in `d`
based on columns `col`

```julia
by(d::AbstractDataFrame, cols, f::Function)
by(f::Function, d::AbstractDataFrame, cols)
```

### Arguments

* `d` : an AbstractDataFrame
* `cols` : a column indicator (Symbol, Int, Vector{Symbol}, etc.)
* `f` : a function to be applied to groups; expects each argument to
  be an AbstractDataFrame

`f` can return a value, a vector, or a DataFrame. For a value or
vector, these are merged into a column along with the `cols` keys. For
a DataFrame, `cols` are combined along columns with the resulting
DataFrame. Returning a DataFrame is the clearest because it allows
column labeling.

A method is defined with `f` as the first argument, so do-block
notation can be used.

`by(d, cols, f)` is equivalent to `combine(map(f, groupby(d, cols)))`.

### Returns

* `::DataFrame`

### Examples

```julia
df = DataFrame(a = repeat([1, 2, 3, 4], outer=[2]),
               b = repeat([2, 1], outer=[4]),
               c = randn(8))
by(df, :a, d -> sum(d[:c]))
by(df, :a, d -> 2 * dropnull(d[:c]))
by(df, :a, d -> DataFrame(c_sum = sum(d[:c]), c_mean = mean(dropnull(d[:c]))))
by(df, :a, d -> DataFrame(c = d[:c], c_mean = mean(dropnull(d[:c]))))
by(df, [:a, :b]) do d
    DataFrame(m = mean(dropnull(d[:c])), v = var(dropnull(d[:c])))
end
```

"""
by(d::AbstractDataFrame, cols, f::Function) = combine(map(f, groupby(d, cols)))
by(f::Function, d::AbstractDataFrame, cols) = by(d, cols, f)

#
# Aggregate convenience functions
#

# Applies a set of functions over a DataFrame, in the from of a cross-product
"""
Split-apply-combine that applies a set of functions over columns of an
AbstractDataFrame or GroupedDataFrame

```julia
aggregate(d::AbstractDataFrame, cols, fs)
aggregate(gd::GroupedDataFrame, fs)
```

### Arguments

* `d` : an AbstractDataFrame
* `gd` : a GroupedDataFrame
* `cols` : a column indicator (Symbol, Int, Vector{Symbol}, etc.)
* `fs` : a function or vector of functions to be applied to vectors
  within groups; expects each argument to be a column vector

Each `fs` should return a value or vector. All returns must be the
same length.

### Returns

* `::DataFrame`

### Examples

```julia
df = DataFrame(a = repeat([1, 2, 3, 4], outer=[2]),
               b = repeat([2, 1], outer=[4]),
               c = randn(8))
aggregate(df, :a, sum)
aggregate(df, :a, [sum, x->mean(dropnull(x))])
aggregate(groupby(df, :a), [sum, x->mean(dropnull(x))])
df |> groupby(:a) |> [sum, x->mean(dropnull(x))]   # equivalent
```

"""
aggregate(d::AbstractDataFrame, fs::Function) = aggregate(d, [fs])
function aggregate{T<:Function}(d::AbstractDataFrame, fs::Vector{T})
    headers = _makeheaders(fs, _names(d))
    _aggregate(d, fs, headers)
end

# Applies aggregate to non-key cols of each SubDataFrame of a GroupedDataFrame
aggregate(gd::GroupedDataFrame, f::Function) = aggregate(gd, [f])
function aggregate{T<:Function}(gd::GroupedDataFrame, fs::Vector{T})
    headers = _makeheaders(fs, _setdiff(_names(gd), gd.cols))
    combine(map(x -> _aggregate(without(x, gd.cols), fs, headers), gd))
end
(|>)(gd::GroupedDataFrame, fs::Function) = aggregate(gd, fs)
(|>){T<:Function}(gd::GroupedDataFrame, fs::Vector{T}) = aggregate(gd, fs)

# Groups DataFrame by cols before applying aggregate
function aggregate{S <: ColumnIndex, T <:Function}(d::AbstractDataFrame,
                                     cols::@compat(Union{S, AbstractVector{S}}),
                                     fs::@compat(Union{T, Vector{T}}))
    aggregate(groupby(d, cols), fs)
end

function _makeheaders{T<:Function}(fs::Vector{T}, cn::Vector{Symbol})
    fnames = _fnames(fs) # see other/utils.jl
    reshape([Symbol(colname,'_',fname) for fname in fnames, colname in cn],
            length(fnames)*length(cn))
end

function _aggregate{T<:Function}(d::AbstractDataFrame, fs::Vector{T}, headers::Vector{Symbol})
    DataFrame(colwise(fs, d), headers)
end


const TIMER_OUTPUTS = true
const being_timed = Ref{Bool}(false)

if TIMER_OUTPUTS
    using TimerOutputs

    macro timer(name, expr)
        :(if being_timed[]
              @timeit $(esc(name)) $(esc(expr))
          else
              $(esc(expr))
          end)
    end

    macro iftimer(expr)
        esc(expr)
    end

else
    macro timer(name, expr)
        esc(expr)
    end

    macro iftimer(expr)
    end
end

using Base: ImmutableDict

# Linked List interface
@inline assoc(d::ImmutableDict, k, v) = ImmutableDict(d, k=>v)

struct LL{V}
    v::V
    i::Int
end

islist(::Any) = false
islist(l::Union{LL, Term, AbstractArray, Tuple}) = true

Base.empty(l::LL) = empty(l.v)
Base.isempty(l::LL) = l.i > length(l.v)

Base.length(l::LL) = length(l.v)-l.i+1
@inline car(l::LL) = l.v[l.i]
@inline cdr(l::LL) = isempty(l) ? empty(l) : LL(l.v, l.i+1)

Base.length(t::Term) = length(arguments(t)) + 1 # PIRACY
Base.isempty(t::Term) = false
@inline car(t::Term) = operation(t)
@inline cdr(t::Term) = arguments(t)

@inline car(v) = first(v)
@inline cdr(v) = isempty(v) ? empty(l) : LL(v, 2)

@inline take_n(ll::LL, n) = isempty(ll) || n == 0 ? empty(ll) : @views ll.v[ll.i:n+ll.i-1] # @views handles Tuple
@inline take_n(ll, n) = @views ll[1:n]

drop_n(ll, n) = n === 0 ? ll : drop_n(cdr(ll), n-1)

@inline drop_n(ll::Term, n) = drop_n(arguments(ll), n-1)
@inline drop_n(ll::AbstractArray, n) = drop_n(LL(ll, 1), n)
@inline drop_n(ll::LL, n) = LL(ll.v, ll.i+n)

pow(x,y) = y==0 ? 1 : y<0 ? inv(x)^(-y) : x^y
pow(x::Symbolic,y) = y==0 ? 1 : Base.:^(x,y)
pow(x, y::Symbolic) = Base.:^(x,y)
pow(x::Symbolic,y::Symbolic) = Base.:^(x,y)

# Simplification utilities
has_trig(x) = false
function has_trig(term::Term)
    fns = (sin, cos, tan, cot, sec, csc)
    op = operation(term)

    if Base.@nany 6 i->fns[i] === op
        return true
    else
        return any(has_trig, arguments(term))
    end
end

fold(x) = x
function fold(t::Term)
    tt = map(fold, arguments(t))
    if !any(x->x isa Symbolic, tt)
        # evaluate it
        return operation(t)(tt...)
    else
        return Term{symtype(t)}(operation(t), tt)
    end
end

### Predicates

sym_isa(::Type{T}) where {T} = @nospecialize(x) -> x isa T || symtype(x) <: T
is_operation(f) = @nospecialize(x) -> (x isa Term) && (operation(x) == f)

isliteral(::Type{T}) where {T} = x -> x isa T
isnumber(x) = isliteral(Number)(x)

_iszero(t) = false
_iszero(x::Number) = iszero(x)
_isone(t) = false
_isone(x::Number) = isone(x)

issortedₑ(args) = issorted(args, lt=<ₑ)

# are there nested ⋆ terms?
function isnotflat(⋆)
    function (args)
        for t in args
            if t isa Term && operation(t) === (⋆)
                return true
            end
        end
        return false
    end
end

function hasrepeats(x)
    length(x) <= 1 && return false
    for i=1:length(x)-1
        if isequal(x[i], x[i+1])
            return true
        end
    end
    return false
end

function merge_repeats(merge, xs)
    length(xs) <= 1 && return false
    merged = Any[]
    i=1

    while i<=length(xs)
        l = 1
        for j=i+1:length(xs)
            if isequal(xs[i], xs[j])
                l += 1
            else
                break
            end
        end
        if l > 1
            push!(merged, merge(xs[i], l))
        else
            push!(merged, xs[i])
        end
        i+=l
    end
    return merged
end

# Numbers to the back
function flatten_term(⋆, args)
    # flatten nested ⋆
    flattened_args = []
    for t in args
        if t isa Term && operation(t) === (⋆)
            append!(flattened_args, arguments(t))
        else
            push!(flattened_args, t)
        end
    end
    Term(⋆, flattened_args)
end

function sort_args(f, args)
    if length(args) < 2
        return Term(f, args)
    elseif length(args) == 2
        x, y = args
        return Term(f, x <ₑ y ? [x,y] : [y,x])
    end
    args = args isa Tuple ? [args...] : args
    Term(f, sort(args, lt=<ₑ))
end


## LINEAR LEARNING NETWORKS (FOR INTERNAL USE ONLY)

# constructs and returns an unsupervised node, and its machine, for a
# given model and input. the node is static if model is a function
# instead of bona fide model. Example: `node_(pca, X)` (dynamic) or
# `node_(MLJ.matrix, X)` (static).
function node_(model, X)
    if model isa Model
        mach = machine(model, X)
        return transform(mach, X), mach
    else
        n = node(model, X)
        return n, n.machine
    end
end

# `models_and_functions` can include both functions (static
# operatations) and bona fide models. If `ys == nothing` the learning
# network is assumed to be unsupervised. Otherwise: If `target ===
# nothing`, then no target transform is applied; if `target !==
# nothing` and `inverse === nothing`, then corresponding `transform`
# and `inverse_transform` are applied; if neither `target` nor
# `inverse` are `nothing`, then both `target` and `inverse` are
# assumed to be StaticTransformations, to be applied
# respectively to `ys` and to the output of the `models_and_functions`
# pipeline. Note that target inversion is applied to the output of the
# *last* nodal machine in the pipeline, corresponding to the last
# element of `models_and_functions`.

# No checks whatsoever are performed. Returns a learning network.
function linear_learning_network(Xs, ys, target, inverse, models_and_functions...)

    n = length(models_and_functions)

    if ys !== nothing && target !== nothing
        yt, target_machine = node_(target, ys)
    else
        yt  = ys
    end

    nodes = Vector(undef, n + 1)
    nodes[1] = Xs

    for i = 2:(n + 1)
        m = models_and_functions[i-1]
        if m isa Supervised
            supervised_machine = machine(m, nodes[i-1], yt)
            nodes[i] = predict(supervised_machine, nodes[i-1])
       else
            nodes[i] = node_(m, nodes[i-1]) |> first
        end
    end

    if target === nothing
        terminal_node=nodes[end]
    else
        if inverse === nothing
            terminal_node = inverse_transform(target_machine, nodes[end])
        else
            terminal_node = node_(inverse, nodes[end]) |> first
        end
    end

    return terminal_node

end

## PREPROCESSING

function eval_and_reassign(modl, ex)
    s = gensym()
    evaluated = modl.eval(ex)
    modl.eval(:($s = $evaluated))
    return s, evaluated
end

pipe_alert(message) = throw(ArgumentError("@pipeline error.\n"*
                                     string(message)))
pipe_alert(k::Int) = throw(ArgumentError("@pipeline error $k. "))

# does expression processing, syntax and semantic
# checks. is_probabilistic is `true`, `false` or `missing`.
function pipeline_preprocess(modl, ex, is_probabilistic::Union{Missing,Bool})

    ex isa Expr || pipe_alert(2)
    length(ex.args) > 1 || pipe_alert(3)
    ex.head == :call || pipe_alert(4)
    pipetype_ = ex.args[1]
    pipetype_ isa Symbol || pipe_alert(5)

    fieldnames_ = []               # for `from_network`
    models_ = []                   # for `from_network`
    models_and_functions_ = []     # for `linear_learning_network`
    models_and_functions  = []     # for `linear_learning_network`
    for ex in ex.args[2:end]
        if ex isa Expr
            if ex.head == :kw
                variable_ = ex.args[1]
                variable_ isa Symbol || pipe_alert(8)
                value_, value = eval_and_reassign(modl, ex.args[2])
                if variable_ == :target
                    if value isa Function
                        value_, value =
                            eval_and_reassign(modl,
                                              :(MLJ.StaticTransformer($value)))
                    end
                    value isa Unsupervised ||
                        pipe_alert("Got $value where a function or "*
                                   "Unsupervised model instance was "*
                                   "expected. ")
                    target_ = value_
                    target = value
                    push!(fieldnames_, :target)
                elseif variable_ == :inverse
                    if value isa Function
                        value_, value =
                            eval_and_reassign(modl,
                                              :(MLJ.StaticTransformer($value)))
                    else
                        pipe_alert(10)
                    end
                    inverse_ = value_
                    inverse = value
                    push!(fieldnames_, :inverse)
                else
                    value isa Model || pipe_alert(11)
                    push!(models_and_functions_, value_)
                    push!(models_and_functions, value)
                    push!(fieldnames_, variable_)
                end
                push!(models_, value_)
            else
                f = modl.eval(ex)
                f isa Function ||
                    pipe_alert("Perhaps a missing name, as `name=$f'?")
                push!(models_and_functions_, ex)
                push!(models_and_functions, f)
            end
        else
            f = modl.eval(ex)
            f isa Function || pipe_alert(7)
            push!(models_and_functions_, ex)
            push!(models_and_functions, f)
        end
    end

    (@isdefined target)  || (target = nothing; target_ = :nothing)
    (@isdefined inverse) || (inverse = nothing; inverse_ = :nothing)
    inverse !== nothing && target === nothing &&
        pipe_alert("You have specified `inverse=...` but no `target`.")

    supervised(m) = !(m isa Unsupervised) && !(m isa Function)

    supervised_components = filter(supervised, models_and_functions)

    length(supervised_components) < 2 ||
        pipe_alert("More than one component of the pipeline is a "*
                   "supervised model .")

    is_supervised  =
        length(supervised_components) == 1

    # `kind_` is defined in composites.jl:
    kind = kind_(is_supervised, is_probabilistic)
    ismissing(kind) &&
        pipe_alert("Network has no supervised components and so "*
                  "`is_probabilistic=true` "*
                  "declaration is not allowed. ")

    target isa StaticTransformer && inverse == nothing &&
        pipe_alert("It appears `target` is a function. "*
                   "You must therefore specify `inverse=...` .")

    target == nothing || is_supervised ||
        pipe_alert("`target=...` has been specified but no "*
                   "supervised components have been specified. ")

    return (pipetype_, fieldnames_, models_,
            models_and_functions_, target_, inverse_, kind)

end

function pipeline_preprocess(modl, ex, kw_ex)
    kw_ex isa Expr || pipe_alert(10)
    kw_ex.head == :(=) || pipe_alert(11)
    kw_ex.args[1] == :is_probabilistic ||
        pipe_alert("Unrecognized keywork `$(kw_ex.args[1])`.")
    value_ = kw_ex.args[2]
    if value_ == :missing
        value = missing
    else
        value = value_
        value isa Bool ||
            pipe_alert("`is_probabilistic` can only be `true` or `false`.")
    end
    return pipeline_preprocess(modl, ex, value)
end

function pipeline_(modl, ex, kw_ex)

    (pipetype_, fieldnames_, models_, models_and_functions_,
     target_, inverse_, kind) =
         pipeline_preprocess(modl, ex, kw_ex)

    if kind === :UnsupervisedNetwork
        ys_ = :nothing
    else
        ys_ = :(source(nothing, kind=:target))
    end

    N_ex = quote

        MLJ.linear_learning_network(source(nothing), $ys_,
                                    $target_, $inverse_,
                                    $(models_and_functions_...))
    end

    from_network_(modl, pipetype_, fieldnames_, models_, N_ex, kind)

    return pipetype_

end

pipeline_(modl, ex) = pipeline_(modl, ex, :(is_probabilistic=missing))

"""
    @pipeline NewPipeType(fld1=model1, fld2=model2, ...)
    @pipeline NewPipeType(fld1=model1, fld2=model2, ...) is_probabilistic=false

Create a new pipeline model type `NewPipeType` that composes the types of
the specified models `model1`, `model2`, ... . The models are composed
in the specified order, meaning the input(s) of the pipeline goes to
`model1`, whose output is sent to `model2`, and so forth.

At most one of the models may be a supervised model, in which case
`NewPipeType` is supervised. Otherwise it is unsupervised.

The new model type `NewPipeType` has hyperparameters (fields) named
`:fld1`, `:fld2`, ..., whose default values for an automatically
generated keyword constructor are deep copies of `model1`, `model2`,
... .

*Important.* If the overall pipeline is supervised and makes probabilistic
predictions, then one must declare `is_probabilistic=true`. In the
deterministic case the keyword argument can be omitted.

Static (unlearned) transformations - that is, ordinary functions - may
also be inserted in the pipeline as shown in the following example
(the classifier is probabilistic but the pipeline itself is
deterministic):

    @pipeline MyPipe(X -> coerce(X, :age=>Continuous),
                     hot=OneHotEncoder(),
                     cnst=ConstantClassifier(),
                     yhat -> mode.(yhat))

### Return value

An instance of the new type, with default
hyperparameters (see above), is returned.

### Target transformation and inverse transformation

A learned target transformation (such as standardization) can also be
specified, using the keyword `target`, provided the transformer
provides an `inverse_transform` method:

    @load KNNRegressor
    @pipeline MyPipe(hot=OneHotEncoder(),
                     knn=KNNRegressor(),
                     target=UnivariateTransformer())

A static transformation can be specified instead, but then
an `inverse` must also be given:

    @load KNNRegressor
    @pipeline MyPipe(hot=OneHotEncoder(),
                     knn=KNNRegressor(),
                     target = v -> log.(v),
                     inverse = v -> exp.(v))

*Important.* While the supervised model in a pipeline providing a
 target transformation can appear anywhere in the pipeline (as in
 `ConstantClassifier` example above), the inverse operation is always
 performed on the output of the *final* model or static
 transformation in the pipeline.

See also: [@from_network](@ref)

"""
macro pipeline(exs...)
    pipetype_ = pipeline_(__module__, exs...)

    esc(quote
        $pipetype_()
        end)

end