"""
`module  JAC.PhotoRecombination`
    ... a submodel of JAC that contains all methods for computing photo-recomnbination, i.e. radiative recombination
        and radiative electron capture properties between some initial and final-state multiplets.
"""
module PhotoRecombination

    using Printf, ..AngularMomentum, ..Basics, ..Continuum, ..ManyElectron, ..PhotoEmission, ..Radial, ..Nuclear, ..TableStrings
    ##x global JAC_counter = 0

    """
    `struct  Settings`  ... defines a type for the details and parameters of computing photo recombination lines.

        + multipoles              ::Array{EmMultipole}           ... Specifies the multipoles of the radiation field that are to be included.
        + gauges                  ::Array{UseGauge}              ... Specifies the gauges to be included into the computations.
        + electronEnergies        ::Array{Float64,1}             ... List of electron energies.
        + ionEnergies             ::Array{Float64,1}             ... List of ion energies.
        + useIonEnergies          ::Bool                         ... Make use of ion energies in [MeV/u] to obtain the electron energies.
        + calcAnisotropy          ::Bool                         ... True, if the overall anisotropy is to be calculated.
        + calcTensors             ::Bool                         ... True, if the statistical tensors are to be calculated and false otherwise.
        + printBeforeComputation  ::Bool                         ... True, if all energies and lines are printed before their evaluation.
        + selectLines             ::Bool                         ... True, if lines are selected individually for the computations.
        + selectedLines           ::Array{Tuple{Int64,Int64},1}  ... List of lines, given by tupels (inital-level, final-level).
    """
    struct Settings
        multipoles                ::Array{EmMultipole}
        gauges                    ::Array{UseGauge}
        electronEnergies          ::Array{Float64,1}
        ionEnergies               ::Array{Float64,1}
        useIonEnergies            ::Bool
        calcAnisotropy            ::Bool
        calcTensors               ::Bool
        printBeforeComputation    ::Bool
        selectLines               ::Bool
        selectedLines             ::Array{Tuple{Int64,Int64},1}
    end

    """
    `PhotoRecombination.Settings()`  ... constructor for the default values of photo recombination line computations
    """
    function Settings()
        Settings(EmMultipole[], UseGauge[], Float64[], Float64[], false, false, false, false, false, Tuple{Int64,Int64}[])
    end

    # `Base.show(io::IO, settings::PhotoRecombination.Settings)`
    #		... prepares a proper printout of the variable settings::PhotoRecombination.Settings.
    function Base.show(io::IO, settings::PhotoRecombination.Settings)
        println(io, "multipoles:               $(settings.multipoles)  ")
        println(io, "gauges:                   $(settings.gauges)  ")
        println(io, "electronEnergies:         $(settings.electronEnergies)  ")
        println(io, "ionEnergies:              $(settings.ionEnergies)  ")
        println(io, "useIonEnergies:           $(settings.useIonEnergies)  ")
        println(io, "calcAnisotropy:           $(settings.calcAnisotropy)  ")
        println(io, "calcTensors:              $(settings.calcTensors)  ")
        println(io, "printBeforeComputation:   $(settings.printBeforeComputation)  ")
        println(io, "selectLines:              $(settings.selectLines)  ")
        println(io, "selectedLines:            $(settings.selectedLines)  ")
    end

    """
    `struct  PhotoRecombination.Channel`
        ... defines a type for a photorecombination channel to help characterize a single multipole and scattering
            (continuum) state of many electron-states with a single free electron.

        + multipole      ::EmMultipole          ... Multipole of the photon emission/absorption.
        + gauge          ::EmGauge              ... Gauge for dealing with the (coupled) radiation field.
        + kappa          ::Int64                ... partial-wave of the free electron
        + symmetry       ::LevelSymmetry        ... total angular momentum and parity of the scattering state
        + phase          ::Float64              ... phase of the partial wave
        + amplitude      ::Complex{Float64}     ... Rec amplitude associated with the given channel.
    """
    struct  Channel
        multipole        ::EmMultipole
        gauge            ::EmGauge
        kappa            ::Int64
        symmetry         ::LevelSymmetry
        phase            ::Float64
        amplitude        ::Complex{Float64}
    end

    """
    `struct  PhotoRecombination.Line`
        ... defines a type for a Photorecombination line that may include the definition of channels.

        + initialLevel   ::Level                  ... initial-(state) level
        + finalLevel     ::Level                  ... final-(state) level
        + electronEnergy ::Float64                ... Energy of the (incoming free) electron.
        + photonEnergy   ::Float64                ... Energy of the emitted photon.
        + crossSection   ::EmProperty             ... Cross section for this electron capture.
        + hasChannels    ::Bool                   ... Determines whether the individual (sub-) channels are defined in terms of their
                                                      free-electron energy, kappa, multipole, etc., or not.
        + channels       ::Array{PhotoRecombination.Channel,1}    ... List of photorecombination channels of this line.
    """
    struct  Line
        initialLevel     ::Level
        finalLevel       ::Level
        electronEnergy   ::Float64
        photonEnergy     ::Float64
        crossSection     ::EmProperty
        hasChannels      ::Bool
        channels         ::Array{PhotoRecombination.Channel,1}
    end

    """
    `PhotoRecombination.Line()`
        ... constructor for an `empty instance` of a photorecombination line between a specified initial
            and final level.
    """
    function Line(initialLevel::Level, finalLevel::Level)
        Line(initialLevel::Level, finalLevel::Level, 0., 0., EmProperty(0., 0.), false, Channel[])
    end

    # `Base.show(io::IO, line::PhotoRecombination.Line)`  ... prepares a proper printout of the variable line::PhotoRecombination.Line.
    function Base.show(io::IO, line::PhotoRecombination.Line)
        println(io, "initialLevel:      $(line.initialLevel)  ")
        println(io, "finalLevel:        $(line.finalLevel)  ")
        println(io, "electronEnergy:    $(line.electronEnergy)  ")
        println(io, "photonEnergy:      $(line.photonEnergy)  ")
        println(io, "crossSection:      $(line.crossSection)  ")
        println(io, "hasChannels:       $(line.hasChannels)  ")
        println(io, "channels:          $(line.channels)  ")
    end

    """
    `PhotoRecombination.amplitude(kind::String, channel::PhotoRecombination.Channel, energy::Float64, finalLevel::Level,
                                  continuumLevel::Level, grid::Radial.Grid)`
        ... to compute the kind = (photorecombination) amplitude
            < alpha_f J_f || O^(photorecombination) || (alpha_i J_i, epsilon kappa) J_t>  due to the electron-photon
            interaction for the given final and continuum level, the partial wave of the outgoing electron as well as
            the given multipole and gauge. A value::ComplexF64 is returned.
    """
    function amplitude(kind::String, channel::PhotoRecombination.Channel, energy::Float64, finalLevel::Level,
                       continuumLevel::Level, grid::Radial.Grid)
        if      kind in [ "photorecombination"]
        #-----------------------------------
            amplitude = PhotoEmission.amplitude("emission", channel.multipole, channel.gauge, energy, finalLevel, continuumLevel, grid)
            amplitude = im^Basics.subshell_l(Subshell(101, channel.kappa)) * exp( im*channel.phase ) * amplitude
        else    error("stop b")
        end

        return( amplitude )
    end

    """
    `PhotoRecombination.computeAmplitudesProperties(line::PhotoRecombination.Line, nm::Nuclear.Model, grid::Radial.Grid, nrContinuum::Int64,
                                                    settings::PhotoRecombination.Settings)`
        ... to compute all amplitudes and properties of the given line; a line::PhotoRecombination.Line is returned for
            which the amplitudes and properties are now evaluated.
    """
    function  computeAmplitudesProperties(line::PhotoRecombination.Line, nm::Nuclear.Model, grid::Radial.Grid, nrContinuum::Int64,
                                          settings::PhotoRecombination.Settings)
        newChannels = PhotoRecombination.Channel[];;   contSettings = Continuum.Settings(false, nrContinuum);    csC = 0.;    csB = 0.
        for channel in line.channels
            newfLevel = Basics.generateLevelWithSymmetryReducedBasis(line.finalLevel)
            newfLevel = Basics.generateLevelWithExtraSubshell(Subshell(101, channel.kappa), newfLevel)
            newiLevel = Basics.generateLevelWithSymmetryReducedBasis(line.initialLevel)
            cOrbital, phase  = Continuum.generateOrbitalForLevel(line.electronEnergy, Subshell(101, channel.kappa), newiLevel, nm, grid, contSettings)
            newcLevel  = Basics.generateLevelWithExtraElectron(cOrbital, channel.symmetry, newiLevel)
            newChannel = PhotoRecombination.Channel(channel.multipole, channel.gauge, channel.kappa, channel.symmetry, phase, 0.)
            amplitude  = PhotoRecombination.amplitude("photorecombination", channel, line.photonEnergy, newfLevel, newcLevel, grid)
            push!( newChannels, PhotoRecombination.Channel(newChannel.multipole, newChannel.gauge, newChannel.kappa, newChannel.symmetry,
                                                           newChannel.phase, amplitude) )
            if       channel.gauge == Basics.Coulomb     csC = csC + abs(amplitude)^2
            elseif   channel.gauge == Basics.Babushkin   csB = csB + abs(amplitude)^2
            elseif   channel.gauge == Basics.Magnetic    csB = csB + abs(amplitude)^2;   csC = csC + abs(amplitude)^2
            end
        end
        Ji2 = AngularMomentum.twoJ(line.initialLevel.J)
        csFactor     = 8 * pi^3 * Defaults.getDefaults("alpha")^3 * line.photonEnergy / (Ji2 + 1)
        crossSection = EmProperty(csFactor * csC, csFactor * csB)
        newLine = PhotoRecombination.Line( line.initialLevel, line.finalLevel, line.electronEnergy, line.photonEnergy, crossSection, true, newChannels)
        return( newLine )
    end

    """
    `PhotoRecombination.computeAnisotropyParameter(nu::Int64, gauge::EmGauge, line::PhotoRecombination.Line)`
        ... to compute the anisotropy parameter of the emitted photons for the photorecombination of an initially unpolarized ion.
            A value::ComplexF64 is returned.
    """
    function  computeAnisotropyParameter(nu::Int64, gauge::EmGauge, line::PhotoRecombination.Line)
        if  !line.hasChannels   error("No channels are defined for the given PhotoRecombination.line.")         end
        wa = 0.0im;    Ji = line.initialLevel.J;    Jf = line.finalLevel.J
        wn = 0.;
        for  ch in line.channels
            if  gauge != ch.gauge  &&  gauge != Basics.Magnetic   continue    end
            wn = wn + conj(ch.amplitude) * ch.amplitude
        end

        for  cha  in line.channels
            if  gauge != cha.gauge  &&  gauge != Basics.Magnetic   continue    end
            J = cha.symmetry.J;    L = cha.multipole.L;    if  cha.multipole.electric   p = 1   else    p = 0   end
            j = AngularMomentum.kappa_j(cha.kappa);    l = AngularMomentum.kappa_l(cha.kappa)
            #
            for  chp  in line.channels
                if  gauge != chp.gauge  &&  gauge != Basics.Magnetic   continue    end
                Jp = chp.symmetry.J;    Lp = chp.multipole.L;    if  chp.multipole.electric   pp = 1   else    pp = 0   end
                jp = AngularMomentum.kappa_j(chp.kappa);     lp = AngularMomentum.kappa_l(chp.kappa)
                #
                if  1 + (-1)^(L + p + Lp + pp - nu) == 0    continue    end
                wa = wa + 1.0im^(L + p - Lp - pp) * AngularMomentum.phaseFactor([Ji, -1, AngularJ64(1//2), -1, Jf]) *
                                 sqrt( AngularMomentum.bracket([AngularJ64(L), AngularJ64(Lp), l, lp, j, jp, J, Jp]) ) *
                                 AngularMomentum.ClebschGordan( l, AngularM64(0), lp, AngularM64(0),  AngularJ64(nu),  AngularM64(0)) *
                                 AngularMomentum.ClebschGordan( AngularJ64(L), AngularM64(1), AngularJ64(Lp), AngularM64(-1),
                                                                    AngularJ64(nu), AngularM64(0)) *
                                 AngularMomentum.Wigner_6j(J, Jp, AngularJ64(nu), AngularJ64(Lp), AngularJ64(L), Jf) *
                                 AngularMomentum.Wigner_6j(J, Jp, AngularJ64(nu), jp, j, Ji) *
                                 AngularMomentum.Wigner_6j(j, jp, AngularJ64(nu), lp, l, AngularJ64(1//2)) *
                                 conj(cha.amplitude) * chp.amplitude
            end
        end

        value = - 0.5 * wa / wn
        return( value )
    end

    """
    `PhotoRecombination.computeLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                         settings::PhotoRecombination.Settings; output::Bool=true)`
        ... to compute the photo recombination transition amplitudes and all properties as requested by the given settings.
            A list of lines::Array{PhotoRecombination.Lines} is returned.
    """
    function  computeLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                           settings::PhotoRecombination.Settings; output::Bool=true)
        println("")
        printstyled("PhotoRecombination.computeLines(): The computation of photo-recombination properties starts now ... \n", color=:light_green)
        printstyled("--------------------------------------------------------------------------------------------------- \n", color=:light_green)
        println("")
        lines = PhotoRecombination.determineLines(finalMultiplet, initialMultiplet, settings)
        # Display all selected lines before the computations start
        if  settings.printBeforeComputation    PhotoRecombination.displayLines(lines)    end
        # Determine maximum energy and check for consistency of the grid
        maxEnergy = 0.;   for  line in lines   maxEnergy = max(maxEnergy, line.electronEnergy)   end
        nrContinuum = Continuum.gridConsistency(maxEnergy, grid)
        # Calculate all amplitudes and requested properties
        newLines = PhotoRecombination.Line[]
        for  line in lines
            newLine = PhotoRecombination.computeAmplitudesProperties(line, nm, grid, nrContinuum, settings)
            push!( newLines, newLine)
        end
        # Print all results to screen
        PhotoRecombination.displayResults(stdout, newLines, settings)
        printSummary, iostream = Defaults.getDefaults("summary flag/stream")
        if  printSummary   PhotoRecombination.displayResults(iostream, newLines, settings)    end
        #
        if    output    return( lines )
        else            return( nothing )
        end
    end

    """
    `PhotoRecombination.determineChannels(finalLevel::Level, initialLevel::Level, settings::PhotoRecombination.Settings)`
        ... to determine a list of RecChannel for a transitions from the initial to final level and by taking into account
            the particular settings of for this computation; an Array{PhotoRecombination.Channel,1} is returned.
    """
    function determineChannels(finalLevel::Level, initialLevel::Level, settings::PhotoRecombination.Settings)
        channels = PhotoRecombination.Channel[];
        symi = LevelSymmetry(initialLevel.J, initialLevel.parity);    symf = LevelSymmetry(finalLevel.J, finalLevel.parity)
        for  mp in settings.multipoles
            for  gauge in settings.gauges
                symList = AngularMomentum.allowedMultipoleSymmetries(symf, mp)
                for  symt in symList
                    kappaList = AngularMomentum.allowedKappaSymmetries(symi, symt)
                    for  kappa in kappaList
                        # Include further restrictions if appropriate
                        if     string(mp)[1] == 'E'  &&   gauge == Basics.UseCoulomb
                            push!(channels, PhotoRecombination.Channel(mp, Basics.Coulomb,   kappa, symt, 0., Complex(0.)) )
                        elseif string(mp)[1] == 'E'  &&   gauge == Basics.UseBabushkin
                            push!(channels, PhotoRecombination.Channel(mp, Basics.Babushkin, kappa, symt, 0., Complex(0.)) )
                        elseif string(mp)[1] == 'M'
                            push!(channels, PhotoRecombination.Channel(mp, Basics.Magnetic,  kappa, symt, 0., Complex(0.)) )
                        end
                    end
                end
            end
        end
        return( channels )
    end

    """
    `PhotoRecombination.determineLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, settings::PhotoRecombination.Settings)`
        ... to determine a list of PhotoRecombination.Line's for transitions between levels from the initial- and final-state
            multiplets, and by taking into account the particular selections and settings for this computation;
            an Array{PhotoRecombination.Line,1} is returned. Apart from the level specification, all physical properties are set
            to zero during the initialization process.
    """
    function  determineLines(finalMultiplet::Multiplet, initialMultiplet::Multiplet, settings::PhotoRecombination.Settings)
        if    settings.selectLines    selectLines   = true;
                       selectedLines = Basics.determineSelectedLines(settings.selectedLines, initialMultiplet, finalMultiplet)
        else                          selectLines   = false
        end

        lines = PhotoRecombination.Line[]
        for  i = 1:length(initialMultiplet.levels)
            for  f = 1:length(finalMultiplet.levels)
                if  selectLines  &&  !((i,f) in selectedLines )    continue   end
                ##x println("PhotoRecombination.determineLines-aa:  i = $i, f = $f")
                #
                if  settings.useIonEnergies
                    error("Not yet implemented: Given ion energies.")
                else
                    for  en in settings.electronEnergies
                        if  en < 0    continue   end
                        # Electron energies are still in 'pre-defined' units; convert to Hartree
                        en_au = Defaults.convertUnits("energy: to atomic", en)
                        omega = en_au + initialMultiplet.levels[i].energy - finalMultiplet.levels[f].energy

                        channels = PhotoRecombination.determineChannels(finalMultiplet.levels[f], initialMultiplet.levels[i], settings)
                        push!( lines, PhotoRecombination.Line(initialMultiplet.levels[i], finalMultiplet.levels[f], en_au, omega,
                                                              EmProperty(0., 0.), true, channels) )
                    end
                end
            end
        end
        return( lines )
    end

    """
    `PhotoRecombination.displayLines(lines::Array{PhotoRecombination.Line,1})`
        ... to display a list of lines and channels that have been selected due to the prior settings. A neat table of all selected
            transitions and energies is printed but nothing is returned otherwise.
    """
    function  displayLines(lines::Array{PhotoRecombination.Line,1})
        println(" ")
        println("  Selected photorecombination lines:")
        println(" ")
        println("  ", TableStrings.hLine(175))
        sa = "  ";   sb = "  "
        sa = sa * TableStrings.center(18, "i-level-f"; na=2);                                sb = sb * TableStrings.hBlank(20)
        sa = sa * TableStrings.center(18, "i--J^P--f"; na=4);                                sb = sb * TableStrings.hBlank(22)
        sa = sa * TableStrings.center(10, "Energy_if"; na=2);
        sb = sb * TableStrings.center(10, TableStrings.inUnits("energy"); na=2)
        sa = sa * TableStrings.center(12, "Energy e_r"; na=1);
        sb = sb * TableStrings.center(12, TableStrings.inUnits("energy"); na=2)
        sa = sa * TableStrings.center(10, "omega"; na=5);
        sb = sb * TableStrings.center(10, TableStrings.inUnits("energy"); na=4)
        sa = sa * TableStrings.flushleft(57, "List of multipoles, gauges, kappas and total symmetries"; na=4)
        sb = sb * TableStrings.flushleft(57, "partial (multipole, gauge, total J^P)                  "; na=4)
        println(sa);    println(sb);    println("  ", TableStrings.hLine(175))
        #
        for  line in lines
            sa  = "  ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                           fsym = LevelSymmetry( line.finalLevel.J,   line.finalLevel.parity)
            sa = sa * TableStrings.center(18, TableStrings.levels_if(line.initialLevel.index, line.finalLevel.index); na=2)
            sa = sa * TableStrings.center(18, TableStrings.symmetries_if(isym, fsym); na=4)
            energy = line.initialLevel.energy - line.finalLevel.energy
            sa = sa * @sprintf("%.4e", Defaults.convertUnits("energy: from atomic", energy))              * "   "
            sa = sa * @sprintf("%.4e", Defaults.convertUnits("energy: from atomic", line.electronEnergy)) * "   "
            sa = sa * @sprintf("%.4e", Defaults.convertUnits("energy: from atomic", line.photonEnergy))   * "    "
            kappaMultipoleSymmetryList = Tuple{Int64,EmMultipole,EmGauge,LevelSymmetry}[]
            for  i in 1:length(line.channels)
                push!( kappaMultipoleSymmetryList, (line.channels[i].kappa, line.channels[i].multipole, line.channels[i].gauge,
                                                    line.channels[i].symmetry) )
            end
            wa = TableStrings.kappaMultipoleSymmetryTupels(85, kappaMultipoleSymmetryList)
            sb = sa * wa[1];    println( sb )
            for  i = 2:length(wa)
                sb = TableStrings.hBlank( length(sa) ) * wa[i];    println( sb )
            end
        end
        println("  ", TableStrings.hLine(175), "\n")
        #
        return( nothing )
    end

    """
    `PhotoRecombination.displayResults(stream::IO, lines::Array{PhotoRecombination.Line,1}, settings::PhotoRecombination.Settings)`
        ... to list all results, energies, cross sections, etc. of the selected lines. A neat table is printed but nothing is
            returned otherwise.
    """
    function  displayResults(stream::IO, lines::Array{PhotoRecombination.Line,1}, settings::PhotoRecombination.Settings)
        println(stream, " ")
        println(stream, "  Photorecombination cross sections for beta^2 * gamma^2 = 1:")
        println(stream, "  ")
        println(stream, "  ", TableStrings.hLine(133))
        sa = "  ";   sb = "  "
        sa = sa * TableStrings.center(18, "i-level-f"   ; na=0);                       sb = sb * TableStrings.hBlank(20)
        sa = sa * TableStrings.center(16, "i--J^P--f"   ; na=3);                       sb = sb * TableStrings.hBlank(19)
        sa = sa * TableStrings.center(12, "i--Energy--f"; na=4)
        sb = sb * TableStrings.center(12,TableStrings.inUnits("energy"); na=4)
        sa = sa * TableStrings.center(12, "omega"     ; na=4)
        sb = sb * TableStrings.center(12, TableStrings.inUnits("energy"); na=4)
        sa = sa * TableStrings.center(12, "Energy e_p"; na=3)
        sb = sb * TableStrings.center(12, TableStrings.inUnits("energy"); na=3)
        sa = sa * TableStrings.center(10, "Multipoles"; na=1);                              sb = sb * TableStrings.hBlank(13)
        sa = sa * TableStrings.center(30, "Cou -- Cross section -- Bab"; na=3)
        sb = sb * TableStrings.center(30, TableStrings.inUnits("cross section") * "          " *
                                              TableStrings.inUnits("cross section"); na=3)
        println(stream, sa);    println(stream, sb);    println(stream, "  ", TableStrings.hLine(133))
        #
        for  line in lines
            sa  = " ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                          fsym = LevelSymmetry( line.finalLevel.J,   line.finalLevel.parity)
            sa = sa * TableStrings.center(17, TableStrings.levels_if(line.initialLevel.index, line.finalLevel.index); na=2)
            sa = sa * TableStrings.center(17, TableStrings.symmetries_if(isym, fsym); na=3)
            en = line.initialLevel.energy - line.finalLevel.energy
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", en))                  * "    "
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", line.photonEnergy))   * "    "
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", line.electronEnergy)) * "    "
            multipoles = EmMultipole[]
            for  ch in line.channels
                multipoles = push!( multipoles, ch.multipole)
            end
            multipoles = unique(multipoles);   mpString = TableStrings.multipoleList(multipoles) * "          "
            sa = sa * TableStrings.flushleft(11, mpString[1:10];  na=2)
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("cross section: from atomic", line.crossSection.Coulomb))     * "    "
            sa = sa * @sprintf("%.6e", Defaults.convertUnits("cross section: from atomic", line.crossSection.Babushkin))   * "    "
            println(stream, sa)
        end
        println(stream, "  ", TableStrings.hLine(133))
        #
        #
        if  settings.calcAnisotropy
            println(stream, " ")
            println(stream, "  Anisotropy angular parameters beta_nu of the emitted photons for initially unpolarized ions:")
            println(stream, " ")
            println(stream, "  ", TableStrings.hLine(133))
            sa = "  ";   sb = "  "
            sa = sa * TableStrings.center(18, "i-level-f"   ; na=0);                       sb = sb * TableStrings.hBlank(20)
            sa = sa * TableStrings.center(16, "i--J^P--f"   ; na=3);                       sb = sb * TableStrings.hBlank(19)
            sa = sa * TableStrings.center(12, "i--Energy--f"; na=4)
            sb = sb * TableStrings.center(12,TableStrings.inUnits("energy"); na=4)
            sa = sa * TableStrings.center(12, "omega"     ; na=4)
            sb = sb * TableStrings.center(12, TableStrings.inUnits("energy"); na=4)
            sa = sa * TableStrings.center(12, "Energy e_p"; na=3)
            sb = sb * TableStrings.center(12, TableStrings.inUnits("energy"); na=3)
            sa = sa * TableStrings.center(10, "Multipoles"; na=5);                         sb = sb * TableStrings.hBlank(15)
            sa = sa * TableStrings.flushleft(57, "beta's"; na=4)
            println(stream, sa);    println(stream, sb);    println(stream, "  ", TableStrings.hLine(133))
            #
            for  line in lines
                sa  = " ";    isym = LevelSymmetry( line.initialLevel.J, line.initialLevel.parity)
                              fsym = LevelSymmetry( line.finalLevel.J,   line.finalLevel.parity)
                sa = sa * TableStrings.center(17, TableStrings.levels_if(line.initialLevel.index, line.finalLevel.index); na=2)
                sa = sa * TableStrings.center(17, TableStrings.symmetries_if(isym, fsym); na=3)
                en = line.initialLevel.energy - line.finalLevel.energy
                sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", en))                  * "    "
                sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", line.photonEnergy))   * "    "
                sa = sa * @sprintf("%.6e", Defaults.convertUnits("energy: from atomic", line.electronEnergy)) * "    "
                multipoles = EmMultipole[]
                for  ch in line.channels
                    multipoles = push!( multipoles, ch.multipole)
                end
                multipoles = unique(multipoles);   mpString = TableStrings.multipoleList(multipoles) * "          "
                sa = sa * TableStrings.flushleft(11, mpString[1:10];  na=3)
                be = PhotoRecombination.computeAnisotropyParameter(1, Basics.Coulomb,   line);   sa = sa * @sprintf("%.4e", be.re) * " (1, C);  "
                be = PhotoRecombination.computeAnisotropyParameter(1, Basics.Babushkin, line);   sa = sa * @sprintf("%.4e", be.re) * " (1, B);  "
                println(stream, sa);   sa = TableStrings.hBlank(102)
                be = PhotoRecombination.computeAnisotropyParameter(2, Basics.Coulomb,   line);   sa = sa * @sprintf("%.4e", be.re) * " (2, C);  "
                be = PhotoRecombination.computeAnisotropyParameter(2, Basics.Babushkin, line);   sa = sa * @sprintf("%.4e", be.re) * " (2, B);  "
                println(stream, sa);   sa = TableStrings.hBlank(102)
                be = PhotoRecombination.computeAnisotropyParameter(3, Basics.Coulomb,   line);   sa = sa * @sprintf("%.4e", be.re) * " (3, C);  "
                be = PhotoRecombination.computeAnisotropyParameter(3, Basics.Babushkin, line);   sa = sa * @sprintf("%.4e", be.re) * " (3, B);  "
                println(stream, sa);   sa = TableStrings.hBlank(102)
                be = PhotoRecombination.computeAnisotropyParameter(4, Basics.Coulomb,   line);   sa = sa * @sprintf("%.4e", be.re) * " (4, C);  "
                be = PhotoRecombination.computeAnisotropyParameter(4, Basics.Babushkin, line);   sa = sa * @sprintf("%.4e", be.re) * " (4, B);  "
                println(stream, sa);   sa = TableStrings.hBlank(102)
            end
            println(stream, "  ", TableStrings.hLine(133))
        end
        #
        #
        if  settings.calcTensors
            println(stream, " ")
            println(stream, "  Reduced statistical tensors of the recombined ion ... not yet implemented !!")
            println(stream, " ")
        end
        #
        return( nothing )
    end

end # module


###############################################################################
#
#  NfRelOrd
#
###############################################################################

# I don't like that we have to drag S around
# Still hoping for julia/#18466

mutable struct NfRelOrdSet{T}
  nf::RelativeExtension{T}

  function NfRelOrdSet{T}(K::RelativeExtension{T}) where {T}
    a = new(K)
    return a
  end
end

mutable struct NfRelOrd{T, S} <: Ring
  nf::RelativeExtension{T}
  basis_nf::Vector{RelativeElement{T}}
  basis_mat::Generic.Mat{T}
  basis_mat_inv::Generic.Mat{T}
  basis_pmat::PMat{T, S}
  pseudo_basis::Vector{Tuple{RelativeElement{T}, S}}

  disc_abs::NfOrdIdl # used if T == nf_elem
  disc_rel#::NfRelOrdIdl{T} # used otherwise; is a forward declaration
  parent::NfRelOrdSet{T}

  isequation_order::Bool

  ismaximal::Int                   # 0 Not known
                                   # 1 Known to be maximal
                                   # 2 Known to not be maximal

  trace_mat::Generic.Mat{T}

  inv_coeff_ideals::Vector{S}

  function NfRelOrd{T, S}(K::RelativeExtension{T}) where {T, S}
    z = new{T, S}()
    z.nf = K
    z.parent = NfRelOrdSet{T}(K)
    z.isequation_order = false
    z.ismaximal = 0
    return z
  end

  function NfRelOrd{T, S}(K::RelativeExtension{T}, M::PMat{T, S}) where {T, S}
    z = NfRelOrd{T, S}(K)
    z.nf = K
    z.parent = NfRelOrdSet{T}(K)
    z.basis_pmat = M
    z.basis_mat = M.matrix
    return z
  end

  function NfRelOrd{T, S}(K::RelativeExtension{T}, M::Generic.Mat{T}) where {T, S}
    z = NfRelOrd{T, S}(K)
    z.nf = K
    z.parent = NfRelOrdSet{T}(K)
    z.basis_mat = M
    z.basis_pmat = pseudo_matrix(M)
    return z
  end
end

###############################################################################
#
#  NfRelOrdElem
#
###############################################################################

mutable struct NfRelOrdElem{T} <: RingElem
  parent#::NfRelOrd{T, S} # I don't want to drag the S around
  elem_in_nf::RelativeElement{T}
  elem_in_basis::Vector{T}
  has_coord::Bool

  function NfRelOrdElem{T}(O::NfRelOrd{T}) where {T}
    z = new{T}()
    z.parent = O
    z.elem_in_nf = zero(nf(O))
    z.elem_in_basis = Vector{T}(degree(O))
    z.has_coord = false
    return z
  end

  function NfRelOrdElem{T}(O::NfRelOrd{T}, a::RelativeElement{T}) where {T}
    z = new{T}()
    z.parent = O
    z.elem_in_nf = a
    z.elem_in_basis = Vector{T}(degree(O))
    z.has_coord = false
    return z
  end

  function NfRelOrdElem{T}(O::NfRelOrd{T}, a::RelativeElement{T}, arr::Vector{T}) where {T}
    z = new{T}()
    z.parent = O
    z.elem_in_nf = a
    z.elem_in_basis = arr
    z.has_coord = true
    return z
  end
end

###############################################################################
#
#  NfRelOrdFracIdl
#
###############################################################################

mutable struct NfRelOrdFracIdlSet{T, S}
  order::NfRelOrd{T, S}

  function NfRelOrdFracIdlSet{T, S}(O::NfRelOrd{T, S}) where {T, S}
    a = new(O)
    return a
  end
end

mutable struct NfRelOrdFracIdl{T, S}
  order::NfRelOrd{T, S}
  parent::NfRelOrdFracIdlSet{T, S}
  basis_pmat::PMat{T, S}
  pseudo_basis::Vector{Tuple{RelativeElement{T}, S}}
  basis_mat::Generic.Mat{T}
  basis_mat_inv::Generic.Mat{T}
  den::fmpz

  norm
  has_norm::Bool

  function NfRelOrdFracIdl{T, S}(O::NfRelOrd{T, S}) where {T, S}
    z = new{T, S}()
    z.order = O
    z.parent = NfRelOrdFracIdlSet{T, S}(O)
    z.has_norm = false
    return z
  end

  function NfRelOrdFracIdl{T, S}(O::NfRelOrd{T, S}, M::PMat{T, S}) where {T, S}
    z = NfRelOrdFracIdl{T, S}(O)
    z.basis_pmat = M
    z.basis_mat = M.matrix
    return z
  end

  function NfRelOrdFracIdl{T, S}(O::NfRelOrd{T, S}, a::Array{Tuple{T1, S}}) where {T1 <: RelativeElement{T}, S} where T
    z = NfRelOrdFracIdl{T, S}(O)
    z.pseudo_basis = a
    return z
  end
end

###############################################################################
#
#  NfRelOrdIdl
#
###############################################################################

mutable struct NfRelOrdIdlSet{T, S}
  order::NfRelOrd{T, S}

  function NfRelOrdIdlSet{T, S}(O::NfRelOrd{T, S}) where {T, S}
    a = new(O)
    return a
  end
end

mutable struct NfRelOrdIdl{T, S}
  order::NfRelOrd{T, S}
  parent::NfRelOrdIdlSet{T, S}
  basis_pmat::PMat{T, S}
  pseudo_basis::Vector{Tuple{RelativeElement{T}, S}}
  basis_mat::Generic.Mat{T}
  basis_mat_inv::Generic.Mat{T}

  norm
  has_norm::Bool
  is_prime::Int            # 0: don't know
                           # 1 known to be prime
                           # 2 known to be not prime
  splitting_type::Tuple{Int, Int}

  minimum
  non_index_div_poly::fq_poly # only used if the ideal is a prime ideal not dividing the index
  p_uniformizer::NfRelOrdElem{T}
  anti_uniformizer::RelativeElement{T}

  function NfRelOrdIdl{T, S}(O::NfRelOrd{T, S}) where {T, S}
    z = new{T, S}()
    z.order = O
    z.parent = NfRelOrdIdlSet{T, S}(O)
    z.has_norm = false
    z.is_prime = 0
    z.splitting_type = (0,0)
    return z
  end

  function NfRelOrdIdl{T, S}(O::NfRelOrd{T, S}, M::PMat{T, S}) where {T, S}
    z = NfRelOrdIdl{T, S}(O)
    z.basis_pmat = M
    z.basis_mat = M.matrix
    return z
  end

  function NfRelOrdIdl{T, S}(O::NfRelOrd{T, S}, a::Array{Tuple{T1, S}}) where {T1 <: RelativeElement{T}, S} where T
    z = NfRelOrdIdl{T, S}(O)
    z.pseudo_basis = a
    return z
  end
end


abstract type AbstractSemiconductor{T} <: AbstractObject{T} end

"""
    struct Semiconductor{T,G,MT,CDM,IDM} <: AbstractSemiconductor{T}

Semiconductor bulk of a [`SolidStateDetector`](@ref).

This is the volume in which electrons and holes will drift during the signal development.

## Parametric types
* `T`: Precision type.
* `G`: Type of `geometry`.
* `MT`: Type of `material`.
* `CDM`: Type of `charge_drift_model`.
* `IDM`: Type of `impurity_density_model`.

## Fields
* `temperature::T`: Temperature (in K) of the semiconductor.
* `material::MT`: Material of the semiconductor.
* `impurity_density_model::IDM`: Impurity density model for the points inside the semiconductor.
* `charge_drift_model::CDM`: Model that describes the drift of electrons and holes inside the semiconductor.
* `geometry::G`: Geometry of the semiconductor, see [Constructive Solid Geometry (CSG)](@ref).

## Definition in Configuration File

A `Semiconductor` is defined through the `semiconductor` field of a detector.
It needs `material` and `geometry`, and can optionally be given a `temperature`, `impurity_density` and `charge_drift_model`.

An example definition of a semiconductor looks like this:
```yaml
semiconductor:
  material: HPGe
  temperature: 78
  impurity_density: # ...
  charge_drift_model: # ...
  geometry: # ...
```
"""
struct Semiconductor{T,G,MT,CDM,IDM} <: AbstractSemiconductor{T}
    temperature::T
    material::MT
    impurity_density_model::IDM
    charge_drift_model::CDM
    geometry::G
end

function Semiconductor{T}(dict::Dict, input_units::NamedTuple, outer_transformations) where T <: SSDFloat
    impurity_density_model = if haskey(dict, "impurity_density")
        ImpurityDensity(T, dict["impurity_density"], input_units)
    elseif haskey(dict, "charge_density_model")
        @warn "Config file deprication: The field \"charge_density_model\" under semiconductor is deprecated.
            It should be changed to \"impurity_density\". In later version this will result in an error.
            For now, it will be treated as an impurity density."
        ImpurityDensity(T, dict["charge_density_model"], input_units)
    else
        ConstantImpurityDensity{T}(0)
    end
    charge_drift_model = if haskey(dict, "charge_drift_model") && haskey(dict["charge_drift_model"], "model")
        cdm = getfield(Main, Symbol(dict["charge_drift_model"]["model"])){T}
        cdm <: AbstractChargeDriftModel{T} ? cdm() : ElectricFieldChargeDriftModel{T}()
    else
        ElectricFieldChargeDriftModel{T}()
    end
    material = material_properties[materials[dict["material"]]]
    temperature = if haskey(dict, "temperature")
        T(dict["temperature"])
    elseif material.name == "High Purity Germanium"
        T(78)
    else
        T(293)
    end
    inner_transformations = parse_CSG_transformation(T, dict, input_units)
    transformations = combine_transformations(inner_transformations, outer_transformations)
    geometry = Geometry(T, dict["geometry"], input_units, transformations)
    return Semiconductor(temperature, material, impurity_density_model, charge_drift_model, geometry)
end

function println(io::IO, d::Semiconductor{T}) where {T <: SSDFloat}
    println("\t---General Properties---")
    println("\t-Detector Material: \t $(d.material.name)")
    println("\t-Charge Drift Model:\t $(typeof(d.charge_drift_model).name.name)")
end

print(io::IO, d::Semiconductor{T}) where {T} = print(io, "Semiconductor{$T} - $(d.material.name)")

show(io::IO, d::Semiconductor) = print(io, d)
show(io::IO,::MIME"text/plain", d::Semiconductor) = show(io, d)

function Semiconductor(sc::Semiconductor{T,G,MT,CDM,IDM}, impurity_density::AbstractImpurityDensity{T}) where {T,G,MT,CDM,IDM}
    Semiconductor(sc.temperature, sc.material, impurity_density, sc.charge_drift_model, sc.geometry)
end
function Semiconductor(sc::Semiconductor{T,G,MT,CDM,IDM}, chargedriftmodel::AbstractChargeDriftModel{T}) where {T,G,MT,CDM,IDM}
    Semiconductor(sc.temperature, sc.material, sc.impurity_density_model, chargedriftmodel, sc.geometry)
end

"""
    scaling_factor_for_permittivity_in_undepleted_region(sc::Semiconductor{T})::T where {T}

This function is called in the calculations of weighting potentials of undepleted detectors.
The electric permittivity, ``ϵ_{r}``, is scaled with this function in areas where the detector is undepleted.
A value between `[0, +Inf]` should be returned. However, `Inf` should not be returned but instead
a very high value should be returned in order to mimic perfect conductivity if that is desired.

## Arguments
* `sc::Semiconductor{T}`: Semiconductor for which the dielectric permittivity should be scaled up.

!!! danger "Experimental feature!"
    This feature is under research. The goal is to study the properties / signal response of undepleted detector.
    This function is indented to be overwritten by the user to study the response.
"""
function scaling_factor_for_permittivity_in_undepleted_region(sc::Semiconductor{T})::T where {T}
    100000
end


module Spectra

export power_spectrum_1d, power_spectrum_2d, power_spectrum_3d

using FFTW
using MPI

using ..ConfigTypes
using ..Mesh.Grids
using ..MPIStateArrays

include("power_spectrum_les.jl")
include("power_spectrum_gcm.jl")

end


# this file will have the "master" function that's actually exposed to the user
include("pmg_graphs.jl")
include("smiles_rdkit_features.jl")

function featurize_inputs(input_data_folder, SMILES_atom_features, SMILES_bond_features, other_atom_features, graphs_folder, features_folder)
    # determine type of data in input folder (must be all SMILES or all CIFs)
    # check whether user has asked for features that make sense (may require some ancillary fcns)
    # loop over input files and farm out actual parsing to specialized functions for CIF vs. SMILES
    # eventually, save out adjacency matrices to graphs_folder and feature vectors/metadata to features_folder
end

# downstream methods for concatenating atom features generated by using SMILES
concat_smiles_atom_feat(atom_feat::OrderedDict) = vcat(values(atom_feat)...)
concat_smiles_bond_feat(bond_feat::OrderedDict) = vcat(values(bond_feat)...)
## Weave Featurization

# data structure for storing features returned from weave_featurizer
mutable struct WeaveMol
    atom_features
    pair_features
    num_atoms
    atom_feat_num
end

"""
weave featurizer

input:
    inputs(Array{String}): List of SIMLES strings
    atom_feature_list(Array{String}): list of atom features of interest
    bond_feature_list(Array{String}): list of bond features of interest
"""
function weave_featurize(inputs; atom_feature_list::Array{String}=atom_feat_list, bond_feature_list::Array{String}=bond_feat_list)
    outputs = Array{WeaveMol}(undef,length(inputs))
    for (i,smiles) in enumerate(inputs)
        mol = chem.MolFromSmiles(smiles)
        new_order = chem.rdmolfiles.CanonicalRankAtoms(mol)  # reorder the atoms so that they're always in the same canonical order.
        mol = chem.rdmolops.RenumberAtoms(mol, new_order)

        atoms = [atom for atom in mol.GetAtoms()]  # convert PyObject to Julia array
        atoms = sort!(atoms, by = x -> x.GetIdx()+1)  # sort by index to ensure same order as rdkit
        n_atom_feat = sum([length(atom_feat_bins[feat]) for feat in atom_feature_list])
        n_atoms = mol.GetNumAtoms()
        atom_feats = zeros(Float32,n_atoms,n_atom_feat)
        for (a_i,atom) in enumerate(atoms)
            atom_feats[a_i,:] = concat_smiles_atom_feat(smiles_atom_features(atom,feature_list=atom_feature_list))
        end

        bonds = [bond for bond in mol.GetBonds()]  # convert PyObject to Julia array
        bonds = sort!(bonds,by = x->(x.GetBeginAtomIdx()+1,x.GetEndAtomIdx()+1))
        bond_feats = OrderedDict()
        adj_list = [[] for n=1:n_atoms]
        for bond in bonds
            # sort index
            begin_atom_idx = bond.GetBeginAtomIdx()+1
            end_atom_idx = bond.GetEndAtomIdx()+1
            index = begin_atom_idx<end_atom_idx ? (begin_atom_idx,end_atom_idx) : (end_atom_idx, begin_atom_idx)
            bond_feats[index] = concat_smiles_bond_feat(smiles_bond_features(bond,feature_list=bond_feature_list))
            push!(adj_list[begin_atom_idx],end_atom_idx)
            push!(adj_list[end_atom_idx],begin_atom_idx)
        end
        # extract pair features
        bf_len = sum([length(bond_feat_bins[feat]) for feat in bond_feature_list])
        pair_feats = pair_features(mol,bond_feats,adj_list,bf_len = bf_len)
        outputs[i] = WeaveMol(atom_feats, pair_feats, mol.GetNumAtoms(), size(atom_feats,2))
    end
    return outputs
end


module deSolveDiffEq

using Reexport, RCall
@reexport using DiffEqBase

solver = Ref{Module}()

function __init__()
    solver[] = rimport("deSolve")
end

abstract type deSolveAlgoritm <: DiffEqBase.AbstractODEAlgorithm end

struct lsoda <: deSolveAlgoritm end
struct lsode <: deSolveAlgoritm end
struct lsodes <: deSolveAlgoritm end
struct lsodar <: deSolveAlgoritm end
struct vode <: deSolveAlgoritm end
struct daspk <: deSolveAlgoritm end
struct euler <: deSolveAlgoritm end
struct rk4 <: deSolveAlgoritm end
struct ode23 <: deSolveAlgoritm end
struct ode45 <: deSolveAlgoritm end
struct radau <: deSolveAlgoritm end
struct bdf <: deSolveAlgoritm end
struct bdf_d <: deSolveAlgoritm end
struct adams <: deSolveAlgoritm end
struct impAdams <: deSolveAlgoritm end
struct impAdams_d <: deSolveAlgoritm end
struct iteration <: deSolveAlgoritm end

function DiffEqBase.__solve(
    prob::DiffEqBase.AbstractODEProblem,
    alg::deSolveAlgoritm,timeseries=[],ts=[],ks=[];
    saveat=eltype(prob.tspan)[],timeseries_errors=true,
    reltol = 1e-3, abstol = 1e-6,
    maxiters = 100000,
    kwargs...)

    p = prob.p
    tspan = prob.tspan
    u0 = prob.u0

    if DiffEqBase.isinplace(prob)
        f = function (t,u,__p)
            du = similar(u)
            prob.f(du,u,p,t)
            R"list($du)" # Error message says a list return is required!
        end
    else
        f = function (t,u,__p)
            du = prob.f(u,p,t)
            R"list($du)" # Error message says a list return is required!
        end
    end

    _saveat = isempty(saveat) ? nothing : saveat
    if _saveat isa Array
        __saveat = _saveat
    elseif _saveat isa Number
        __saveat = Array(tspan[1]:_saveat:tspan[2])
    elseif _saveat isa Nothing
        __saveat = [tspan[1],tspan[2]]
    else
        __saveat = Array(_saveat)
    end

    out = rcopy(solver[].ode(times = __saveat, y = u0, func = f,
                             method = string(alg)[15:end-2],
                             parms=nothing, maxsteps = maxiters,
                             rtol = reltol, atol = abstol))

    ts = @view out[:,1]

    if typeof(u0) <: AbstractArray
        timeseries = Vector{typeof(u0)}(undef,length(ts))
        for i=1:length(ts)
            timeseries[i] = @view out[i,2:end]
        end
    else
        timeseries = out[i,end]
    end

    DiffEqBase.build_solution(prob,alg,ts,timeseries,
                              dense = false,
                              timeseries_errors = timeseries_errors)
end

end # module


using ClimateModels
import PlutoSliderServer

function fig1to5(x::ModelConfig)
    pth0=pwd()
    pth01=joinpath(pth0,"page","__site")
    pth=joinpath(x.folder,string(x.ID))
    cd(pth)

    for ii in x.inputs[:notebookIDs]
        fil_in="notebook_0$(ii).jl"
        cp(joinpath(pth0,"IPCC",fil_in),fil_in)
        PlutoSliderServer.export_notebook(fil_in)

        fil_out="notebook_0$(ii).html"
        isdir(pth01) ? cp(fil_out,joinpath(pth01,fil_out))  : nothing
    end

    cd(pth0)

    return x
end

MC=ModelConfig(model="IPCC-AR6-WG1",configuration=fig1to5,inputs=Dict(:notebookIDs => 1:5))

#setup(MC)
mkdir(joinpath(MC.folder,string(MC.ID)))

fig1to5(MC)


using Vinyl: @reset, shift
using Test

struct FakeArray end
Base.length(::FakeArray) = shift(k -> k)

@testset "Continuations" begin

k = @reset shift(k -> k)+5
@test k(1) == 6
@test k(2) == 7

@test @reset(2*(shift(k -> k(5))+1)) == 12

@test @reset(2*shift(k -> k(k(4)))) == 16

# TODO fix #1
@reset begin
  for i = 1:2
    x = shift(k -> (i,k()))
  end
  ()
end

f(x) = length(x) > 10 ? :big : :small
k = @reset f(FakeArray())

# see #2
@test k(5) == :small
@test k(15) == :big

end


# views that provide efficient construction and element access

abstract AbstractUnsafeView{T, N} <: AbstractArray{T, N}

typealias ContiguousArray{T<:Number, N} Union(Array{T, N}, AbstractUnsafeView{T, N})
typealias ContiguousVector{T<:Number} ContiguousArray{T, 1}
typealias ContiguousMatrix{T<:Number} ContiguousArray{T, 2}
typealias ContiguousCube{T<:Number} ContiguousArray{T, 3}

typealias ContiguousArrOrNum{T<:Number} Union(ContiguousArray{T}, T)

typealias ContiguousRealArray{T<:Real,N} ContiguousArray{T,N}
typealias ContiguousRealArrOrNum{T<:Real} Union(ContiguousArray{T}, T)

similar(a::AbstractUnsafeView, T::DataType, shape::NTuple) = Array(T, shape)
similar{T}(a::AbstractUnsafeView{T}) = Array(T, size(a))

function copy{T}(a::AbstractUnsafeView{T})
    r = similar(a)
    if isbits(T)
        unsafe_copy!(pointer(r), pointer(a), length(a))
    else
        for i = 1 : length(a)
            r[i] = a[i]
        end
    end
    r
end

immutable UnsafeVectorView{T} <: AbstractUnsafeView{T, 1}
    ptr::Ptr{T}
    len::Int
end

pointer(a::UnsafeVectorView) = a.ptr
pointer{T}(a::UnsafeVectorView{T}, i::Int) = a.ptr + (i - 1) * sizeof(T)
size(a::UnsafeVectorView) = (a.len,)
size(a::UnsafeVectorView, d::Int) = d == 1 ? a.len : 1
length(a::UnsafeVectorView) = a.len

getindex(a::UnsafeVectorView, i::Int) = unsafe_load(a.ptr, i)
setindex!(a::UnsafeVectorView, v, i::Int) = unsafe_store!(a.ptr, v, i)

immutable UnsafeMatrixView{T} <: AbstractUnsafeView{T, 2}
    ptr::Ptr{T}
    dim1::Int
    dim2::Int
    len::Int

    UnsafeMatrixView(ptr::Ptr{T}, d1::Int, d2::Int) = new(ptr, d1, d2, d1 * d2)
end

pointer(a::UnsafeMatrixView) = a.ptr
pointer{T}(a::UnsafeMatrixView{T}, i::Int) = a.ptr + (i - 1) * sizeof(T)
size(a::UnsafeMatrixView) = (a.dim1, a.dim2)
size(a::UnsafeMatrixView, d::Int) = d == 1 ? a.dim1 : d == 2 ? a.dim2 : 1
length(a::UnsafeMatrixView) = a.len

getindex(a::UnsafeMatrixView, i::Int) = unsafe_load(a.ptr, i)
getindex(a::UnsafeMatrixView, i::Int, j::Int) = unsafe_load(a.ptr, i + (j-1) * a.dim1)

setindex!(a::UnsafeMatrixView, v, i::Int) = unsafe_store!(a.ptr, v, i)
setindex!(a::UnsafeMatrixView, v, i::Int, j::Int) = unsafe_store!(a.ptr, v, i + (j-1) * a.dim1)

immutable UnsafeCubeView{T} <: AbstractUnsafeView{T, 3}
    ptr::Ptr{T}
    dim1::Int
    dim2::Int
    dim3::Int
    len::Int
    plen::Int

    UnsafeCubeView(ptr::Ptr{T}, d1::Int, d2::Int, d3::Int) = new(ptr, d1, d2, d3, d1 * d2 * d3, d1 * d2)
end

pointer(a::UnsafeCubeView) = a.ptr
pointer{T}(a::UnsafeCubeView{T}, i::Int) = a.ptr + (i - 1) * sizeof(T)
size(a::UnsafeCubeView) = (a.dim1, a.dim2, a.dim3)
size(a::UnsafeCubeView, d::Int) = d == 1 ? a.dim1 : d == 2 ? a.dim2 : d == 3 ? a.dim3 : 1
length(a::UnsafeCubeView) = a.len

getindex(a::UnsafeCubeView, i::Int) = unsafe_load(a.ptr, i)
getindex(a::UnsafeCubeView, i::Int, j::Int) = unsafe_load(a.ptr, i + (j-1) * a.dim1)
getindex(a::UnsafeCubeView, i::Int, j::Int, k::Int) = unsafe_load(a.ptr, i + (j-1) * a.dim1 + (k-1) * a.plen)

setindex!(a::UnsafeCubeView, v, i::Int) = unsafe_store!(a.ptr, v, i)
setindex!(a::UnsafeCubeView, v, i::Int, j::Int) = unsafe_store!(a.ptr, v, i + (j-1) * a.dim1)
setindex!(a::UnsafeCubeView, v, i::Int, j::Int, k::Int) = unsafe_store!(a.ptr, v, i + (j-1) * a.dim1 + (k-1) * a.plen)

# calculate offset of an element

offset(a::ContiguousArray, i::Int) = (i - 1)
offset(a::ContiguousArray, i::Int, j::Int) = (i - 1) + size(a, 1) * (j - 1)
offset(a::ContiguousArray, i::Int, j::Int, k::Int) = (i - 1) + size(a, 1) * ((j - 1) + size(a, 2) * (k - 1))

offset(a::UnsafeMatrixView, i::Int, j::Int) = (i - 1) + a.dim1 * (j - 1)
offset(a::UnsafeCubeView, i::Int, j::Int, k::Int) = (i - 1) + a.dim1 * ((j - 1) + a.dim2 * (k - 1))

# view constructors

offset_view{T}(a::ContiguousArray{T}, offset::Int, d1::Int) = UnsafeVectorView(pointer(a) + sizeof(T) * offset, d1)
offset_view{T}(a::ContiguousArray{T}, offset::Int, d1::Int, d2::Int) = UnsafeMatrixView{T}(pointer(a) + sizeof(T) * offset, d1, d2)

function offset_view{T}(a::ContiguousArray{T}, offset::Int, d1::Int, d2::Int, d3::Int)
    UnsafeCubeView{T}(pointer(a) + sizeof(T) * offset, d1, d2, d3)
end

unsafe_view(v) = v  # fallback

# 1D

unsafe_view(a::ContiguousVector) = UnsafeVectorView(pointer(a), length(a))

unsafe_view(a::ContiguousArray, r::Colon) = UnsafeVectorView(pointer(a), length(a))
unsafe_view(a::ContiguousArray, r::Colon, j::Int) = offset_view(a, offset(a, 1, j), size(a, 1))
unsafe_view(a::ContiguousArray, r::Colon, j::Int, k::Int) = offset_view(a, offset(a, 1, j, k), size(a, 1))

unsafe_view(a::ContiguousArray, r::Range1{Int}) = offset_view(a, r[1] - 1, length(r))
unsafe_view(a::ContiguousArray, r::Range1{Int}, j::Int) = offset_view(a, offset(a, r[1], j), length(r))
unsafe_view(a::ContiguousArray, r::Range1{Int}, j::Int, k::Int) = offset_view(a, offset(a, r[1], j, k), length(r))

# 2D

unsafe_view{T}(a::ContiguousMatrix{T}) = UnsafeMatrixView{T}(pointer(a), size(a, 1), size(a, 2))

unsafe_view{T}(a::ContiguousVector{T}, r1::Colon, r2::Colon) = UnsafeMatrixView{T}(pointer(a), length(a), 1)
unsafe_view{T}(a::ContiguousMatrix{T}, r1::Colon, r2::Colon) = UnsafeMatrixView{T}(pointer(a), size(a, 1), size(a, 2))

function unsafe_view{T}(a::ContiguousArray{T}, r1::Colon, r2::Colon)
    siz = size(a)
    UnsafeMatrixView{T}(pointer(a), siz[1], succ_length(siz, 1))
end

unsafe_view(a::ContiguousArray, r::Colon, r2::Range1{Int}) = offset_view(a, offset(a, 1, r2[1]), size(a, 1), length(r2))
unsafe_view(a::ContiguousArray, r::Colon, r2::Range1{Int}, k::Int) = offset_view(a, offset(a, 1, r2[1], k), size(a, 1), length(r2))
unsafe_view(a::ContiguousArray, r::Colon, r2::Colon, k::Int) = offset_view(a, offset(a, 1, 1, k), size(a, 1), size(a, 2))

# 3D

unsafe_view{T}(a::ContiguousCube{T}) = UnsafeCubeView{T}(pointer(a), size(a, 1), size(a, 2), size(a, 3))

unsafe_view{T}(a::ContiguousVector{T}, r::Colon, r2::Colon, r3::Colon) = UnsafeCubeView{T}(pointer(a), size(a, 1), 1, 1)
unsafe_view{T}(a::ContiguousMatrix{T}, r::Colon, r2::Colon, r3::Colon) = UnsafeCubeView{T}(pointer(a), size(a, 1), size(a, 2), 1)

function unsafe_view{T}(a::ContiguousArray{T}, r::Colon, r2::Colon, r3::Colon)
    siz = size(a)
    UnsafeCubeView{T}(pointer(a), siz[1], siz[2], succ_length(siz, 2))
end

function unsafe_view(a::ContiguousArray, r::Colon, r2::Colon, r3::Range1{Int})
    offset_view(a, offset(a, 1, 1, r3[1]), size(a, 1), size(a, 2), length(r3))
end


function (parareal::Parareal)(solution::PararealSolution, problem::AbstractInitialValueProblem; mode::String="SERIAL")
    cache = PararealCache(problem, parareal)
    if nprocs() == 1 || mode == "SERIAL"
        parareal_serial!(cache, solution, problem, parareal)
    elseif nprocs() > 1 && mode == "DISTRIBUTED"
        parareal_distributed!(cache, solution, problem, parareal)
    elseif nprocs() > 1 && mode == "MPI"
        parareal_mpi!(cache, solution, problem, parareal)
    else
        error("$mode is not available. Please choose between SERIAL, DISTRIBUTED, and MPI.")
    end
    return solution
end

function (parareal::Parareal)(problem::AbstractInitialValueProblem; mode::String="SERIAL", saveiterates::Bool=false)
    solution = PararealSolution(problem, parareal; saveiterates=saveiterates)
    parareal(solution, problem; mode=mode)
    return solution
end


using DeepUncertainty
using Test
using Flux
using Flux.CUDA
using Flux: cpu, gpu
using DistributionsAD

@testset "Layers" begin
    # MC layers
    include("./layers/mclayers.jl")
    # Batch ensembe layers
    include("./layers/batchensemble.jl")
    # Variational layers
    include("./layers/variational.jl")
    # Variational Batch ensembe layers
    include("./layers/bayesian_be.jl")
    # Spectral layers
    include("./layers/spectrallayers.jl")
end

@testset "Bayesian utils" begin
    include("bayesian.jl")
end

@testset "CUDA" begin
    if CUDA.functional()
        include("cuda/runtests.jl")
    else
        @warn "CUDA unavailable, not testing GPU support"
    end
end


using PSCOPF

using Test
using Dates

@testset verbose=true "test_init_firmness" begin

    gen1 = PSCOPF.Networks.Generator("fuel_1_0", "bus_1", PSCOPF.Networks.PILOTABLE, 10., 100., 0., 10.,
                                    Dates.Second(210*60), Dates.Second(210*60)) #dmo, dp
    gen2 = PSCOPF.Networks.Generator( "fuel_1_1", "bus_1", PSCOPF.Networks.PILOTABLE, 10., 100., 0., 10.,
                                    Dates.Second(4*60*60), Dates.Second(210*60)) #dmo, dp
    gen_no_pmin = PSCOPF.Networks.Generator("fuel_1_2", "bus_1", PSCOPF.Networks.PILOTABLE, 0., 0., 0., 10.,
                                    Dates.Second(4*60*60), Dates.Second(210*60)) #dmo, dp
    lim1 = PSCOPF.Networks.Generator("wind_1_0", "bus_1", PSCOPF.Networks.LIMITABLE, 0., 0., 0., 10.,
                                    Dates.Second(4*60*60), Dates.Second(210*60)) #dmo, dp
    TS = [DateTime("2015-01-01T14:00:00"), DateTime("2015-01-01T14:30:00")]

    #=

      7h             10h30                      14h   14h30
     ECH          TS-Delta  NextECH             TS1   TS2
       |                 |  |                     |     |
                          <---------------------->
                                     3h30
                               |<---------------------->
                       TS2-Delta
                            11h
    =#
    @testset "gen_with_no_pmin" begin
        generators = [gen_no_pmin]
        ech = Dates.DateTime("2015-01-01T07:00:00")

        next_ech = Dates.DateTime("2015-01-01T10:45:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test ismissing(PSCOPF.get_commitment_firmness(firmness, "fuel_1_2"))
        @test ismissing(PSCOPF.get_commitment_firmness(firmness, "fuel_1_2", TS[1]))
        @test ismissing(PSCOPF.get_commitment_firmness(firmness, "fuel_1_2", TS[2]))
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_2")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_2", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_2", TS[2]) == PSCOPF.FREE
    end

    @testset "limitable" begin
        generators = [lim1]
        ech = Dates.DateTime("2015-01-01T07:00:00")

        next_ech = Dates.DateTime("2015-01-01T10:45:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test ismissing(PSCOPF.get_commitment_firmness(firmness, "wind_1_0"))
        @test ismissing(PSCOPF.get_commitment_firmness(firmness, "wind_1_0", TS[1]))
        @test ismissing(PSCOPF.get_commitment_firmness(firmness, "wind_1_0", TS[2]))
        @test length(PSCOPF.get_power_level_firmness(firmness, "wind_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "wind_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "wind_1_0", TS[2]) == PSCOPF.FREE
    end

    #=
       10h30         12h            14h   14h30
    TS1-Delta        ECH            TS1    TS2
            |          |             |      |
             <---------------------->
                          3h30
                   |<---------------------->
           TS2-Delta
                 11h
    =#
    @testset "ech_late" begin
        generators = [gen1]
        ech = Dates.DateTime("2015-01-01T12:00:00")

        next_ech = Dates.DateTime("2015-01-01T13:00:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)

        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.DECIDED
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.DECIDED
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.DECIDED
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.DECIDED
    end

    #=
       10h30                      14h   14h30
    TS-Delta                       TS    TS2
         ECH
           |                        |     |
           |<---------------------->|     |
                           3h30           |
                  <---------------------->|
                11h
    =#
    @testset "ech_last_moment" begin
        generators = [gen1]
        ech = Dates.DateTime("2015-01-01T10:30:00")

        next_ech = Dates.DateTime("2015-01-01T10:30:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.FREE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.FREE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE

        next_ech = Dates.DateTime("2015-01-01T10:45:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE

        next_ech = Dates.DateTime("2015-01-01T13:00:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.TO_DECIDE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.TO_DECIDE

        next_ech = nothing
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.TO_DECIDE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.TO_DECIDE
    end

    #=
      7h             10h30                      14h   14h30
     ECH          TS-Delta                      TS1    TS2
       |                 |                        |     |
                          <---------------------->
                                     3h30
                               |<---------------------->
                       TS2-Delta
                            11h
    =#
    @testset "ech_early" begin
        generators = [gen1]
        ech = Dates.DateTime("2015-01-01T07:00:00")

        next_ech = Dates.DateTime("2015-01-01T10:30:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.FREE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.FREE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE

        next_ech = Dates.DateTime("2015-01-01T10:45:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE

        next_ech = Dates.DateTime("2015-01-01T13:00:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)
        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.TO_DECIDE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.TO_DECIDE
    end

    #=
      7h             10h30                      14h   14h30
     ECH            TS1-DP                      TS1    TS2
       |                 |                        |     |
       |                 |<---------------------->|     |
       |                 |           3h30               |
       |                 |     |<---------------------->
       |                 | TS2-DP
       |                 |   11h
       |          |      |
            TS1-DMO TS2-DMO
                10h   10h30

    =#
    @testset "two_generators" begin
        generators = [gen1, gen2]
        ech = Dates.DateTime("2015-01-01T07:00:00")

        next_ech = Dates.DateTime("2015-01-01T10:45:00")
        firmness = PSCOPF.compute_firmness(ech, next_ech, TS, generators)

        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_0")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_0", TS[2]) == PSCOPF.FREE

        @test length(PSCOPF.get_commitment_firmness(firmness, "fuel_1_1")) == 2
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_1", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_commitment_firmness(firmness, "fuel_1_1", TS[2]) == PSCOPF.TO_DECIDE
        @test length(PSCOPF.get_power_level_firmness(firmness, "fuel_1_1")) == 2
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_1", TS[1]) == PSCOPF.TO_DECIDE
        @test PSCOPF.get_power_level_firmness(firmness, "fuel_1_1", TS[2]) == PSCOPF.FREE

    end

end


"""
Code for convenience functions for calculating CCFs on RvSpectMLBase's types
At some point could consider moving this to RvSpectML.

Author: Eric Ford
Created: August 2020
"""

using Statistics

using NamedTupleTools
include("util.jl")

include("ccf_chunk.jl")

include("ccf_chunklist.jl")

using ThreadedIterables
include("ccf_chunklist_timeseries.jl")

include("order_ccf_chunklist.jl")

#using ThreadTools
include("order_ccf_chunklist_timeseries.jl")


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    GeoStats

`ScientificTypes.jl` convention adopted in the GeoStats.jl project.
"""
struct GeoStats <: Convention end

# Basic values
scitype(::Integer,        ::GeoStats) = Count
scitype(::AbstractFloat,  ::GeoStats) = Continuous
scitype(::AbstractString, ::GeoStats) = Textual

# Categorical values
function scitype(c::CategoricalValue, ::GeoStats)
  L = length(levels(c))
  ifelse(isordered(pool(c)), OrderedFactor{L}, Multiclass{L})
end
function scitype(A::CategoricalArray{T,N}, ::GeoStats) where {T,N}
  L = length(levels(A))
  S = ifelse(isordered(A), OrderedFactor{L}, Multiclass{L})
  T >: Missing && (S = Union{S,Missing})
  AbstractArray{S,N}
end

# Scitype for fast array broadcasting
Scitype(::Type{<:Integer},        ::GeoStats) = Count
Scitype(::Type{<:AbstractFloat},  ::GeoStats) = Continuous
Scitype(::Type{<:AbstractString}, ::GeoStats) = Textual


abstract Func{Ti,To}

typealias ScalarFunc{T} Func{T,T}
typealias ScalarFunc2d{T} Func{Point2{T},T}
typealias ScalarFunc3d{T} Func{Point3{T},T}

typealias FuncArray{T<:Func} AbstractArray{T}
typealias ScalarFuncArray{T<:ScalarFunc} AbstractArray{T}
typealias ScalarFunc2dArray{T<:ScalarFunc2d} AbstractArray{T}
typealias ScalarFunc3dArray{T<:ScalarFunc3d} AbstractArray{T}

macro vectorize(f,Ti,To)
    f = esc(f)
    Ti = esc(Ti)
    To = esc(To)

    quote
        function $f(dst::AbstractArray{$To}, c::AbstractArray{$Ti})
            @assert size(dst) == size(c)
            for i in eachindex(c)
                dst[i] = $f(c[i])
            end
        end

        function $f(c::AbstractArray{$Ti})
            dst = Array{$To}(size(c))
            $f(dst,c)
            dst
        end
    end
end

macro vectorize2(f,T,Ti,To)
    f = esc(f)
    T = esc(T)
    Ti = esc(Ti)
    To = esc(To)

    quote
        function $f(f::$T, dst::AbstractArray{$To}, c::AbstractArray{$Ti})
            @assert size(dst) == size(c)
            for i in eachindex(c)
                dst[i] = $f(f,c[i])
            end
        end

        function $f(f::$T, c::AbstractArray{$Ti})
            dst = Array{$To}(size(c))
            $f(f,dst,c)
            dst
        end
    end
end


include("../../../PlotUtils/PlotUtils.jl")
include("../../../PlotUtils/imports.jl")

using Distributed
rmprocs(filter(w -> w > 1, workers()))
addprocs(9)

if length(ARGS) == 0
  results_dir = "/scratchdata/alui2/cytof/results/repfam/test-sim-6-7-14/"
else
  results_dir = ARGS[1]  # path to results directory
end

@everywhere include("../../../PlotUtils/PlotUtils.jl")
@everywhere include("../../../PlotUtils/imports.jl")

@everywhere function plot_params(samples, simdat, imgdir)
  # Create a directory for images / txt if needed.
  mkpath("$(imgdir)/txt/")

  # Extract samples
  extract(s) = map(o -> o[s], samples)
  Zs = extract(:theta__Z)
  Ws = extract(:theta__W)
  lams = extract(:theta__lam)
  alphas = extract(:theta__alpha)
  vs = extract(:theta__v)
  deltas = extract(:theta__delta)
  sig2s = extract(:theta__sig2)
  etas = extract(:theta__eta)
  omegas = extract(:omega)
  ps = [MCMC.sigmoid.(o) for o in omegas]

  # Number of samples
  I = length(sig2s[1])

  # Print Zmean
  open("$(imgdir)/txt/Zmean.txt", "w") do io
    writedlm(io, mean(Zs))
  end

  # Print Wmean
  open("$(imgdir)/txt/Wmean.txt", "w") do io
    writedlm(io, mean(Ws))
  end

  # Print R
  computeR(w) = vec(sum(w .> 0, dims=2))
  open("$(imgdir)/txt/Rcounts.txt", "w") do io
    I, K = size(Ws[1])
    Rs = map(computeR, Ws)
    Rcounts = zeros(Int, I, K)
    for R in Rs
      for i in 1:I
        Rcounts[i, R[i]] += 1
      end
    end
    writedlm(io, Rcounts)
  end

  # Plot parameters
  PlotUtils.plot_W(Ws, imgdir=imgdir, W_true=simdat[:W])
  PlotUtils.plot_v(vs, imgdir)
  PlotUtils.plot_alpha(alphas, imgdir)
  PlotUtils.plot_mus(deltas, imgdir)
  PlotUtils.plot_sig2(sig2s, imgdir, sig2_true=simdat[:sig2])
  PlotUtils.plot_p(ps, imgdir)

  # Plot y / Z
  PlotUtils.make_yz(simdat[:y], Zs, Ws, lams, imgdir, vlim=(-4,4),
                    Z_true=simdat[:Z], w_thresh=0.0)
end

@everywhere function makeplots(path_to_output)
  output_dir = splitdir(path_to_output)[1]
  println(path_to_output)

  # Create img directory
  imgdir = joinpath(output_dir, "img")
  mkpath("$(imgdir)/txt")

  # Load simulated data
  path_to_simdat = joinpath(output_dir, "simdat.bson")
  simdat = BSON.load(path_to_simdat)[:simdat]

  # Load outputs
  output = BSON.load(path_to_output)
  samples = output[:samples][1]

  # Plot parameters
  plot_params(samples, simdat, imgdir)

  # Number of burn in iterations
  nburn = output[:nburn]

  # Plot loglikelihood
  println("Loglikes ...")
  if :ll in keys(output)
    ll = output[:ll]
    PlotUtils.plot_loglike(ll, imgdir, fname="loglike.pdf")
    PlotUtils.plot_loglike(ll[(nburn + 1):end],
                           imgdir, fname="loglike_postburn.pdf")
  end

  # Extract samples
  println("Extracting samples ...")
  extract(s) = map(o -> o[s], samples)
  Zs = extract(:theta__Z)
  Ws = extract(:theta__W)
  deltas = extract(:theta__delta)
  sig2s = extract(:theta__sig2)
  etas = extract(:theta__eta)

  # Number of samples
  I = length(sig2s[1])

  # Plot missing mechanism
  println("Plotting missmechs ...")
  for i in 1:I
    PlotUtils.plot_missmech(output[:c].constants.beta, i, xlim=[-5, 0])
    plt.savefig("$(imgdir)/missmech_$(i).pdf", bbox_inches="tight")
    plt.close()
  end

  # Print missing mechanism
  open(joinpath(imgdir, "txt/beta.txt"), "w") do io
    writedlm(io, output[:c].constants.beta, ',')
  end

  # Plot data density
  println("Plotting dden ...")
  PlotUtils.plot_dden(ddens=output[:dden],
                      etas=etas, Ws=Ws, Zs=Zs, sig2s=sig2s, deltas=deltas,
                      ygrid=output[:c].constants.y_grid,
                      imgdir=imgdir, simdat=simdat)
end

### MAIN ###

# Name of output file
OUTPUT_FILE = "output.bson"

# Path to all output files
output_paths = [joinpath(root, OUTPUT_FILE)
                for (root, _, files) in walkdir(results_dir)
                if OUTPUT_FILE in files]

# Make plots in parallel
error_msg = pmap(makeplots, output_paths, on_error=identity)
println(error_msg)

println("DONE!")


using Random
using PyCall
using ArviZ: attributes

try
    ArviZ.initialize_bokeh()
catch
    @info "bokeh backend not available. bokeh tests will be skipped."
end

py"""
class PyNullObject(object):
   def __init__(self):
       pass
"""

function create_model(seed = 10)
    rng = MersenneTwister(seed)
    J = 8
    nchains = 4
    ndraws = 500
    data = Dict(
        "J" => J,
        "y" => [28.0, 8.0, -3.0, 7.0, -1.0, 1.0, 18.0, 12.0],
        "sigma" => [15.0, 10.0, 16.0, 11.0, 9.0, 11.0, 10.0, 18.0],
    )
    posterior = Dict(
        "mu" => randn(rng, nchains, ndraws),
        "tau" => abs.(randn(rng, nchains, ndraws)),
        "eta" => randn(rng, nchains, ndraws, J),
        "theta" => randn(rng, nchains, ndraws, J),
    )
    posterior_predictive = Dict("y" => randn(rng, nchains, ndraws, J))
    sample_stats = Dict(
        "energy" => randn(rng, nchains, ndraws),
        "diverging" => (randn(rng, nchains, ndraws) .> 0.90),
        "max_depth" => (randn(rng, nchains, ndraws) .> 0.90),
        "log_likelihood" => randn(rng, nchains, ndraws, J),
    )
    prior = Dict(
        "mu" => randn(rng, nchains, ndraws) / 2,
        "tau" => abs.(randn(rng, nchains, ndraws)) / 2,
        "eta" => randn(rng, nchains, ndraws, data["J"]) / 2,
        "theta" => randn(rng, nchains, ndraws, data["J"]) / 2,
    )
    prior_predictive = Dict("y" => randn(rng, nchains, ndraws, J) / 2)
    sample_stats_prior = Dict(
        "energy" => randn(rng, nchains, ndraws),
        "diverging" => Int.(randn(rng, nchains, ndraws) .> 0.95),
    )
    model = from_dict(
        posterior = posterior,
        posterior_predictive = posterior_predictive,
        sample_stats = sample_stats,
        prior = prior,
        prior_predictive = prior_predictive,
        sample_stats_prior = sample_stats_prior,
        observed_data = Dict("y" => data["y"]),
        dims = Dict("y" => ["obs_dim"], "log_likelihood" => ["obs_dim"]),
        coords = Dict("obs_dim" => 1:J),
    )
    return model
end

models() = (model_1 = create_model(10), model_2 = create_model(11))

function noncentered_schools_data()
    return Dict(
        "J" => 8,
        "y" => [28.0, 8.0, -3.0, 7.0, -1.0, 1.0, 18.0, 12.0],
        "sigma" => [15.0, 10.0, 16.0, 11.0, 9.0, 11.0, 10.0, 18.0],
    )
end

function check_multiple_attrs(test_dict, parent)
    failed_attrs = []
    for (dataset_name, attributes) in test_dict
        if Symbol(dataset_name) ∈ propertynames(parent)
            dataset = getproperty(parent, Symbol(dataset_name))
            for attribute in attributes
                if Symbol(attribute) ∉ propertynames(dataset)
                    push!(failed_attrs, (dataset_name, attribute))
                end
            end
        else
            push!(failed_attrs, dataset_name)
        end
    end
    return failed_attrs
end

dimsizes(ds) = ds._dims
convertindex(x::AbstractArray) = x
convertindex(o::PyObject) = o.array.values
vardict(ds) = Dict(k => convertindex(v._data) for (k, v) in ds._variables)
dimdict(ds) = Dict(k => v._dims for (k, v) in ds._variables)