#TODO: Add Keyword Constructors for all datatypes
#TODO MAYBE: Create + methods to build cells
nca_default() = struct_electrode(
                                  "NCA",                # Chemistry type
                                  0.03,                 # binder weight fraction
                                  0.03,                 # conductive material weight fraction
                                  4.71,                 # AM density
                                  1.77,                 # binder density
                                  1.77,                 # conductive material density
                                  192.0,                # first charge specific capacity
                                  180.0,                # reversible specific capacity
                                  150.0,                # total thickness cathode
                                  0.2,                  # porosity
                                  0.0,                  # extra length
                                  15.0,                 # Pos foil (Al) thickness                      microns
                                  2.7,                  # Positive foil density
                                  7.0,                  # Tab thickness                                microns
                                  6.0,                  # Tab width                                      mm
                                  8.88,                 # Tab density (Ni)                              g/cm3
                                )

nmc622_default() = struct_electrode(
                                    "NMC622",     # Chemistry type
                                    0.03,         # binder weight fraction
                                    0.03,         # Conductive material weight fraction
                                    4.8,          # AM density
                                    1.77,         # binder density
                                    1.77,         # conductive material density
                                    190.0,        # first charge specific capacity
                                    171.0,        # reversible specific capactity
                                    150.0,        # total thickness cathode
                                    0.2,          # porosity
                                    0.0,          # extra length
                                    15.0,         # Pos foil (Al) thickness                      microns
                                    2.7,          # Pos foil density
                                    7.0,          # Tab thickness                                microns
                                    6.0,          # Tab width                                      mm
                                    8.88,         # Tab density
                                  )

lfp_default() = struct_electrode(
                                  "LFP",        # Chemistry type
                                  0.03,         # binder weight fraction
                                  0.03,         # conductive material weight fraction
                                  3.6,          # AM density
                                  1.77,         # binder density
                                  1.77,         # conductive material density
                                  160.0,        # first charge specific capacity
                                  150.0,        # reversible specific capacity
                                  150.0,        # total thickness cathode
                                  0.2,          # porosity
                                  0.0,          # extra length
                                  15.0,         # Pos foil (Al) thickness                      microns
                                  2.7,          # Pos foil density
                                  7.0,          # Tab thickness                                microns
                                  6.0,          # Tab width                                      mm
                                  8.88,         # Tab density
                                )

graphite_default() = struct_electrode(

                                  "Graphite",               # Chemistry type

                                  0.03,                     # binder weight fraction
                                  0.03,                     # conductive weight fraction

                                  2.24,                     # AM density
                                  1.77,                     # binder density
                                  1.77,                     # conductive density

                                  371.93,                   # first charge specific capacity
                                  350.0,                    # reversible specific capacity

                                  150.0,                    # total thickness cathode
                                  0.3,                      # porosity

                                  40.0,                     # Extra length                                   mm

                                  8.0,                      # Neg foil (Cu) thickness                      microns
                                  8.92,                     # Neg foil (Cu) density                         g/cm3

                                  7.0,                      # Tab thickness                                microns
                                  6.0,                      # Tab width                                      mm
                                  8.88,                     # Tab density (Ni)                              g/cm3
                                  )

cell(;form_factor="Cyl", size="18650", dimensional_delta=0.01,
    can_th=0.05, can_rho=7.7, sep_th=20.0, sep_rho=1.2,
    sep_por=0.4, v_nom=3.6, electrolyte_name="EC:DMC",
    electrolyte_rho=1.2, NP=1.15,
    anode=graphite_default(), cathode=nca_default()) = struct_cell_general(
                                      form_factor,              # Form factor                           ["Cyl","Pris","Pou"]
                                       size,                    # Cell size                                    5 digits
                                       dimensional_delta,       # dimensional_delta                               %
                                       can_th,                  # canister thickness                             (mm)
                                       can_rho,                 # Stainless Steel density                        g/cm3

                                       sep_th,                  # Separator thickness                          microns
                                       sep_rho,                 # Separator density                             g/cm3
                                       sep_por,                 # Separator porosity

                                       v_nom,                   # Nominal voltage                                 V

                                       electrolyte_name,        # Electrolyte name
                                       electrolyte_rho,         # Electrolyte density                           g/cm3

                                       NP,                      # N to P ratio

                                       anode,                   # Anode Struct
                                       cathode,                 # Cathode Struct
                                       )

cathode_default_cost() = struct_electrode_costs(
                                                [24.0, 0.9],            # Cost of Positive Active Material                       # USD/kg
                                                [6.6, 1.0],             # Cost of Positive Conductive                            # USD/kg
                                                [9.5, 1.0],             # Cost of Positive Binder                                # USD/kg
                                                [3.1, 1.0],             # Cost of Positive Binder Solvent                        # USD/kg
                                                [10193620.403506],      # Baseline annual positive active material               # kg/year
                                                [13744207.2856261],     # Baseline annual positive binder solvent evaporated     # kg/year
                                                [96.0 / 4.0],           # Multiplier for binder solvent requirement
                                                [0.3, 1.0],             # Cost of Positive Current Collector                     # USD/m2
                                                [0.15, 0.8],            # Cost of Positive Terminal Assembly                     # USD/cell
                                                [0.922],                # Positive Electrode Material (dry) yield
                                                [0.902],                # Positive Current Collector yield
                                              )

anode_default_cost() = struct_electrode_costs(
                                              [12.5, 0.95],                # Cost of Negative Active Material                       # USD/kg
                                              [6.6, 1.0],                  # Cost of Negative Conductive                            # USD/kg
                                              [10.0, 1.0],                 # Cost of Negative Binder                                # USD/kg
                                              ########### Negative binder solvent cost - water
                                              [0.0, 0.9],                  # !!!!!!!!!!! Check !!!!!!!!!!!!!!!!!
                                              [6564082.00851682],          # Baseline annual negative active material               # kg/year
                                              [8291472.01075809],          # Baseline annual negative binder solvent evaporated     # kg/year
                                              [96.0 / 4.0],                # Multiplier for binder solvent requirement
                                              [1.2, 1.0],                  # Cost of Negative Current Collector                     # USD/m2
                                              [0.15, 0.8],                 # Cost of Negative Terminal Assembly                     # USD/cell
                                              [0.922],                     # Negative Electrode Material (dry) yield
                                              [0.902],                     # Negative Current Collector yield
                                            )

cell_costs_default() = struct_cell_costs(
                                          [0.22, 0.8],           # Cost of Cell Canister                                  # USD/cell
                                          [1.1, 1.0],            # Cost of Separator                                      # USD/m2
                                          0.98,                  # Separator yield
                                          [15.0, 1.0],           # Cost of Electrolyte                                    # USD/L
                                          0.94,                  # Electrolyte yield
                                          0.95,                  # Cell yield
                                          cathode_default_cost(),
                                          anode_default_cost(),
                                        )

baseline_cell_assembly_data_default() = struct_manufacturing(
                                                              # Receiving
                                                              [
                                                                14400.0 0.4           # Direct Labor
                                                                3.6 0.6               # Capital Equipment
                                                                900.0 0.5             # Plant Area
                                                                1.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # pos_mat_prep
                                                              [
                                                                14400.0 0.5           # Direct Labor
                                                                2.0 0.7               # Capital Equipment
                                                                600.0 0.6             # Plant Area
                                                                2.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # neg_mat_prep
                                                              [
                                                                14400.0 0.5           # Direct Labor
                                                                2.0 0.7               # Capital Equipment
                                                                600.0 0.6             # Plant Area
                                                                3.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # pos_electrode_coat
                                                              [
                                                                28800.0 0.5           # Direct Labor
                                                                8.0 0.8               # Capital Equipment
                                                                750.0 0.8             # Plant Area
                                                                4.0 1.0               # [volume ratio, special case]
                                                              ],

                                                              # neg_electrode_coat
                                                              [
                                                                28800.0 0.5           # Direct Labor
                                                                8.0 0.8               # Capital Equipment
                                                                750.0 0.8             # Plant Area
                                                                4.0 1.0               # [volume ratio, special case]
                                                              ],

                                                              # bind_slvnt_recov
                                                              [
                                                                14400.0 0.4           # Direct Labor
                                                                3.0 0.6               # Capital Equipment
                                                                225.0 0.6             # Plant Area
                                                                5.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # pos_calendr
                                                              [
                                                                14400.0 0.5           # Direct Labor
                                                                1.0 0.7               # Capital Equipment
                                                                225.0 0.6             # Plant Area
                                                                4.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # neg_calendr
                                                              [
                                                                7200.0 0.5            # Direct Labor
                                                                1.0 0.7               # Capital Equipment
                                                                225.0 0.6             # Plant Area
                                                                4.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # intr_process_mat_handling
                                                              [
                                                                28800.0 0.7           # Direct Labor
                                                                1.5 0.7               # Capital Equipment
                                                                900.0 0.6             # Plant Area
                                                                4.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # electrode_slitting
                                                              [
                                                                28800.0 0.5           # Direct Labor
                                                                2.0 0.7               # Capital Equipment
                                                                300.0 0.6             # Plant Area
                                                                4.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # vacuum_drying
                                                              [
                                                                14400.0 0.5           # Direct Labor
                                                                1.6 0.7               # Capital Equipment
                                                                300.0 0.6             # Plant Area
                                                                4.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # control_lab
                                                              [
                                                                28800.0 0.5           # Direct Labor
                                                                1.5 0.7               # Capital Equipment
                                                                300.0 0.6             # Plant Area
                                                                1.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # cell_winding
                                                              [
                                                                36000.0 0.7           # Direct Labor
                                                                4.0 0.8               # Capital Equipment
                                                                600.0 0.8             # Plant Area
                                                                6.0 2.0               # [volume ratio, special case]
                                                              ],

                                                              # CC_welding
                                                              [
                                                                36000.0 0.7           # Direct Labor
                                                                4.0 0.8               # Capital Equipment
                                                                600.0 0.8             # Plant Area
                                                                6.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # cell_insertion_container
                                                              [
                                                                21600.0 0.5           # Direct Labor
                                                                3.0 0.7               # Capital Equipment
                                                                600.0 0.6             # Plant Area
                                                                6.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # electrolyte_filling_and_cell_sealing
                                                              [
                                                                36000.0 0.5           # Direct Labor
                                                                5.0 0.7               # Capital Equipment
                                                                900.0 0.6             # Plant Area
                                                                6.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # dry_room_control
                                                              [
                                                                14400.0 0.4           # Direct Labor
                                                                20.0 0.6              # Capital Equipment
                                                                100.0 0.4             # Plant Area
                                                                7.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # formation_cycling
                                                              [
                                                                57600.0 0.7           # Direct Labor
                                                                30.0 0.8              # Capital Equipment
                                                                2200.0 0.8            # Plant Area
                                                                6.0 2.0               # [volume ratio, special case]
                                                              ],

                                                              # cell_sealing
                                                              [
                                                                14400.0 0.5           # Direct Labor
                                                                2.0 0.7               # Capital Equipment
                                                                450.0 0.6             # Plant Area
                                                                6.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # charge_retention_testing
                                                              [
                                                                21600.0 0.4           # Direct Labor
                                                                4.75 0.7              # Capital Equipment
                                                                900.0 0.6             # Plant Area
                                                                6.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # rejected_cell_scrap_recycle_assembly
                                                              [
                                                                36000.0 0.7           # Direct Labor
                                                                2.5 0.7               # Capital Equipment
                                                                600.0 0.6             # Plant Area
                                                                6.0 0.0               # [volume ratio, special case]
                                                              ],

                                                              # Shipping
                                                              [
                                                                28800.0 0.5           # Direct Labor
                                                                5.0 0.7               # Capital Equipment
                                                                900.0 0.6             # Plant Area
                                                                1.0 0.0               # [volume ratio, special case]
                                                              ],
                                                            )

costs_data_default() = struct_general_costs(
                                            715.0,                       # Pack Integration Cost                                  # USD/Pack
                                            3000.0,                      # Building cost                                          #  USD/m2
                                            18.0,                        # Labor cost                                             # $/hr
                                            150.0,                       # No of units mfg per energy_kwh_per_year                Million
                                            0.995,                       # Positive Binder Solvent Recovery percentage
                                            baseline_cell_assembly_data_default(),
                                            1.1,                         # formation_cycling_mult
                                            1.1,                         # cell_stacking_mult
                                            0.2,                         # pos_electrode_coating_solvent_SR
                                            0.2,                         # neg_electrode_coating_solvent_SR
                                          )

baseline_data_default() = struct_baseline(
                                          6000000.0,               # Baseline annual energy                                   # kWh/year
                                          25263158.0,              # Baseline annual cells adjusted for yield                 # cells/year
                                          67669329.6866343,        # Baseline annual electrode area                           # m2/year
                                          6100.0,
                                        )

battery_pack_data_default() = struct_pack_level(240.0,)

ovhd_rate_data_default() = struct_ovhd_rate(
                                            5.0 / 100.0,                      # Launch cost percent direct annual materials
                                            10.0 / 100.0,                     # Launch cost percent direct labor plus variable overhead
                                            16.666667 / 100.0,                # Depreciation rate capital equipment                          # 6-year rate
                                            5.0 / 100.0,                      # Depreciation rate building investment                        # 20 year rate
                                            25.0 / 100.0,                     # General, Sales and Administration rate direct labor plus variable overhead
                                            25.0 / 100.0,                     # General, Sales and Administration rate depreciation
                                            40.0 / 100.0,                     # R & D rate depreciation
                                            40.0 / 100.0,                     # Variable overhead rate direct labor
                                            20.0 / 100.0,                     # Variable overhead rate depreciation
                                            5.0 / 100.0,                      # Profit rate after taxes                                      # % of investment
                                            15.0 / 100.0,                     # Working capital rate                                         # % of annual variable cost
                                            5.6 / 100.0,                      # Warranty cost added to the price rate
                                          )

cost_default() = struct_cost(
  cell_costs_default(),
  baseline_cell_assembly_data_default(),
  costs_data_default(),
  baseline_data_default(),
  ovhd_rate_data_default(),
)


"""
`module  JAC.BascisGenerate`
    ... a submodel of JAC that contains methods that support tasks related to spectroscopic computation.
"""
module BascisGenerate

    using Printf, ..AngularMomentum, ..Basics, ..Continuum, ..Defaults, ..Einstein, ..ManyElectron,
                  ..PhotoEmission, ..Radial

    export generate

    """
    `Basics.generate("condensed multiplet: by single weight", multiplet::Multiplet)`
        ... to condense/reduce the number of CSF in the basis of the given multiplet due to a single 'weight';
            a multiplet::Multiplet is returned.  **Not yet implemented !**
    """
    function Basics.generate(sa::String, multiplet::Multiplet)
        !(sa == "condensed multiplet: by single weight")   &&   error("Unsupported keystring = $sa")
        error("Not yet implemented !")
    end

    """
    `Basics.generate("configuration list: NR, from basis", basis::Basis)`
        ... to (re-) generate the list of NR configurations from the given basis; a confList::Array{Configuration,1} is returned.
    """
    function Basics.generate(sa::String, basis::Basis)
        !(sa == "configuration list: NR, from basis")   &&   error("Unsupported keystring = $sa")
        confList    = Configuration[]
        NoElectrons = sum( basis.csfs[1].occupation )
        for  csf in basis.csfs
            shellList = Dict{Shell,Int64}[]
            for  i = 1:length(csf.occupation)
                n = csf.subshell[i].n;    l = Basics.subshell_l( csf.subshell[i] );    sh = Shell(n, l)
                if     haskey(shellList, sh)    shellList(sh) = shellList(sh) + csf.occupation[i]
                else   shellList = merge( shellList, Dict(sh => csf.occupation[i]) )
                end
            end
            confNew = Configuration( shellList, NoElectrons )
            push!( confList, confNew )
        end
        confListNew = ConfigurationsExcludeDoubles(confList)

        return( confListNew )
    end

    """
    `Basics.generate("configuration list: NR, single-configuration", refConf::Configuration, NoExcitations::Int64, fromShells::Array{Shell,1},`
                     toShells::Array{Shell,1})
        ... to generate a non-relativistic configuration list, including the given reference configuration (refConf) and with
            all configurations that differ by NoExcitations from the fromShells into the toShells; an Array{Configuration,1}
            is returned.
    """
    function Basics.generate(sa::String, refConf::Configuration, NoExcitations::Int64, fromShells::Array{Shell,1}, toShells::Array{Shell,1})
        !(sa == "configuration list: NR, single-configuration")   &&   error("Unsupported keystring = $sa")
        confList = [refConf]
        # First prepare a proper reference configuration that also includes all fromShells and toShells with zero occupation
        shellDict = deepcopy(refConf.shells)
        allShells = deepcopy(fromShells)
        for  sh in toShells
            if     sh in allShells
            else   append!(allShells, [sh] )
            end
        end

        for  sh in allShells
        if  !haskey(shellDict, sh )    shellDict = merge( shellDict, Dict( sh => 0 ) )    end
        end

        rconf = Configuration(shellDict, refConf.NoElectrons)
        println("aa", rconf)

        # Add single excitations
        if      NoExcitations == 0
        elseif  NoExcitations == 1
            for  fromsh in fromShells
                for  tosh in toShells
                    confnew = deepcopy(rconf)
                    confnew.shells[fromsh] = confnew.shells[fromsh] - 1
                    confnew.shells[tosh]   = confnew.shells[tosh]   + 1
                    println("bb", confnew)

                    # Check that this is valid configuration and append, if appropriate
                    add = true
                    for (k,v) in  confnew.shells
                        if  v <= -1  ||  Parity(rconf) != Parity(confnew)    add = false    end
                    end
                    if  add
                        for (k,v) in confnew.shells
                        if  v == 0   delete!(confnew.shells, k)   end
                        end
                        push!(confList, confnew);   println("fromsh = $fromsh  tosh = $tosh")
                    end
                end
            end

        # Add double excitations
        elseif  NoExcitations == 2
            for  fromsha in fromShells
            for  fromshb in fromShells
                for  tosha in toShells
                for  toshb in toShells
                    confnew = deepcopy(rconf)
                    confnew.shells[fromsha] = confnew.shells[fromsha] - 1
                    confnew.shells[fromshb] = confnew.shells[fromshb] - 1
                    confnew.shells[tosha]   = confnew.shells[tosha]   + 1
                    confnew.shells[toshb]   = confnew.shells[toshb]   + 1
                    println("bb", confnew)

                    # Check that this is valid configuration and append, if appropriate
                    add = true
                    for (k,v) in  confnew.shells
                        if  v <= -1  ||  Parity(rconf) != Parity(confnew)    add = false    end
                    end
                    if  add
                        for (k,v) in confnew.shells
                        if  v == 0   delete!(confnew.shells, k)   end
                        end
                        push!(confList, confnew)
                    end
                end
                end
            end
            end

        else
            error("stop a")
        end

        # Exclude all configurations that appear twice (and more) in the list
        confList = Basics.ConfigurationsExcludeDoubles(confList)

        return( confList )
    end

    """
    `Basics.generate("configuration list: relativistic", conf::Configuration)`
        ... to split/decompose a non-relativistic configuration into an list of relativistic ConfigurationR[]. The proper
            occupuation of the relativistic subshells is taken into account.
    """
    function Basics.generate(sa::String, conf::Configuration)
        subshellList = Subshell[]
        confList     = ConfigurationR[]

        !(sa == "configuration list: relativistic")  &&   error("Unsupported keystring = $sa")

        initialize = true;    NoElectrons = 0
        for (k, occ)  in conf.shells
            NoElectrons     = NoElectrons + occ
            wa              = Basics.shellSplitOccupation(k, occ)
            subshellListNew = Dict{Subshell,Int64}[]

            if  initialize
                subshellListNew = wa;    initialize = false
            else
                for  s in 1:length(subshellList)
                    for  a in 1:length(wa)
                        wb = Base.merge( subshellList[s], wa[a] )
                        push!(subshellListNew, wb)
                    end
                end
            end
            subshellList = deepcopy(subshellListNew)
        end

        for subsh in subshellList
        wa = ConfigurationR(subsh, NoElectrons)
        push!(confList, wa)
        end

        return( confList )
    end

    """
    `Basics.generate("CSF list: from single ConfigurationR", conf::ConfigurationR, subshellList::Array{Subshell,1})`
        ... to construct from a given (relativistic) configuration all possible CSF with regard to the subshell order as specified
            by subshellList; a list::Array{CsfR,1} is returned.
    """
    function Basics.generate(sa::String, conf::ConfigurationR, subshellList::Array{Subshell,1})
        parity  = Basics.determineParity(conf)
        csfList = CsfR[];   useStandardSubshells = true;    first = true;    previousCsfs = CsfR[]
        # subhshellList = Subshell[];
        for  subsh in subshellList
            if   subsh in keys(conf.subshells)    occ = conf.subshells[subsh]    else    occ = 0    end
            if   first
                stateList   = ManyElectron.provideSubshellStates(subsh, occ)
                currentCsfs = CsfR[]
                for  state in stateList
                    push!( currentCsfs, CsfR( true, AngularJ64(state.Jsub2//2), parity, [state.occ], [state.nu],
                                                [AngularJ64(state.Jsub2//2)], [AngularJ64(state.Jsub2//2)], Subshell[]) )
                end
                previousCsfs = copy(currentCsfs)
                first        = false
            else
                # Now support also all couplings of the subshell states with the CSFs that were built-up so far
                stateList   = ManyElectron.provideSubshellStates(subsh, occ)
                currentCsfs = CsfR[]
                ##x for  i = 1:length(previousCsfs)    println("generate-aa: ", previousCsfs[i])    end
                for  csf in  previousCsfs
                    for  state in stateList
                        occupation = deepcopy(csf.occupation);    seniority = deepcopy(csf.seniority);
                        subshellJ  = deepcopy(csf.subshellJ);     subshells = deepcopy(csf.subshells)
                        push!(occupation, state.occ);   push!(seniority, state.nu);   push!(subshellJ, AngularJ64(state.Jsub2//2) )
                        push!(subshells, subsh)
                        newXList = oplus( csf.subshellX[end], AngularJ64(state.Jsub2//2) )
                        for  newX in newXList
                            subshellX = deepcopy(csf.subshellX);   push!(subshellX, newX)
                            push!( currentCsfs, CsfR( true, subshellX[end], parity, occupation, seniority, subshellJ, subshellX, Subshell[]) )
                        end
                    end
                end
                previousCsfs = copy(currentCsfs)
            end
        end

        return( previousCsfs )
    end

    """
    `Basics.generate("shells: ordered list for NR configurations", confs::Array{Configuration,1})`
        ... to generate for confs, i.e. all the given (non-relativistic) configurations, a common and ordered shell list;
            a list::Array{Shell,1} is returned.
    """
    function Basics.generate(sa::String, confs::Array{Configuration,1})
        shells = Shell[]

        !(sa == "shells: ordered list for NR configurations")  &&   error("Unsupported keystring = $sa")

        wa = Defaults.getDefaults("ordered shell list: non-relativistic", 7)
        for  a in wa
            for  cf in 1:length(confs)
                ks = keys(confs[cf].shells)
                if  a in ks   push!(shells, a);    break    end
            end
        end

        return( shells )
    end

    """
    `Basics.generate("subshells: ordered list for relativistic configurations", confs::Array{ConfigurationR,1})`
        ... to generate for confs, i.e. all the given (relativistic) configurations, common and ordered subshell list;
            a list::Array{Subshell,1} is returned.
    """
    function Basics.generate(sa::String, confs::Array{ConfigurationR,1})
        subshells = Subshell[]

        !(sa == "subshells: ordered list for relativistic configurations")  &&   error("Unsupported keystring = $sa")

        wa = Defaults.getDefaults("ordered subshell list: relativistic", 7)
        for  a in wa
            for  cf in 1:length(confs)
                ks = keys(confs[cf].subshells)
                ## Do include 'empty' subshells into the subshell list if they are specified by the given configurations
                ## if  a in ks   &&   confs[cf].subshells[a]  !=  0     push!(subshells, a);    break    end
                if  a in ks     push!(subshells, a);    break    end
            end
        end
        ##x println("***subshells = $subshells")
        return( subshells )
    end

    """
    `Basics.generate("subshells: ordered list for two bases", basisA::Basis,  basisB::Basis)`
        ... to generate common and ordered subshell list for the two basis A and B; a list::Array{Subshell,1} is returned.
    """
    function Basics.generate(sa::String, basisA::Basis,  basisB::Basis)
        function areEqual(nn::Int64, sha::Array{Subshell,1}, shb::Array{Subshell,1})
            # Determines whether the first nn subshells are equal in sha and shb (true) or not (false)
            for  i = 1:nx
                if    sha[i] != shb[i]    return( false )   end
            end
            return( true )
        end

        subshells = Subshell[]

        !(sa == "subshells: ordered list for two bases")  &&   error("Unsupported keystring = $sa")

        nx = min(length(basisA.subshells), length(basisB.subshells))
        if  areEqual(nx, basisA.subshells, basisB.subshells)
            # If subshell order is equal, any subshell order is accepted for those subshells that occur in both basis
            for  i = 1:nx
                if    basisA.subshells[i] != basisB.subshells[i]   error("Inconsistent subshells of two bases.")
                else  push!( subshells, basisA.subshells[i])
                end
            end
            #
            if       length(basisA.subshells) > nx
                for  i = nx+1:length(basisA.subshells)    push!(subshells, basisA.subshells[i])    end
            elseif   length(basisB.subshells) > nx
                for  i = nx+1:length(basisB.subshells)    push!(subshells, basisB.subshells[i])    end
            end
        else
            println("basisA.subshells = $(basisA.subshells)")
            println("basisB.subshells = $(basisB.subshells)")
            # If subshell order is NOT equal, all subshell in basisA and basisB must follow standard order
            standardList = Defaults.getDefaults("ordered subshell list: relativistic", 7)
            na = 0;  nb = 0
            for  sh in standardList
                if  sh in basisA.subshells  ||   sh in basisB.subshells   push!( subshells, sh)    end
            end
            nn = 0;
            for sh in basisA.subshells
                if    !(sh in subshells) push!( subshells, sh)
                else  wb = findall(x->x==sh, subshells);    if wb[1] <= nn   error("stop a")   else   nn = wb[1]  end
                end
            end
            nn = 0;
            for sh in basisB.subshells
                if    !(sh in subshells) push!( subshells, sh)
                else  wb = findall(x->x==sh, subshells);    if wb[1] <= nn   error("stop a")   else   nn = wb[1]  end
                end
            end
            println("*** extended subshells from two basis = $subshells")
        end
        ##x println("subshells from two basis = $subshells")

        return( subshells )
    end

    """
    `Basics.generate("single-electron spectrum: STO", N::Int64, potential::Radial.Potential, grid::Radial.Grid; N_0::Int64=30, alpha_0::Float64=1.0,
                     beta_0::Float64=1.1)`
        ... to generate a complete one-electron spectrum with N positive and N negative states, and by using even-tempered Slater-type
            orbitals (STO) with parameters ``\alpha_i = \alpha_0 \beta_0^i``; a spectrum::SingleElecSpectrum is returned where just
            N0 positive and N_0 negative are kept for later use.  **Not yet implemented !**

    `Basics.generate("single-electron spectrum: STO, positive", N::Int64, potential::Radial.Potential, grid::Radial.Grid; N_0::Int64=30, alpha_0::Float64=1.0,
                     beta_0::Float64=1.1)`
        ... to generate the same but to return only the N_0 positive states.  **Not yet implemented !**
    """
    function Basics.generate(sa::String, N::Int64, potential::Radial.Potential, grid::Radial.Grid; N_0::Int64=30, alpha_0::Float64=1.0, beta_0::Float64=1.1)
        !(sa == "single-electron spectrum: STO"  &&  sa == "single-electron spectrum: STO, positive")   &&   error("Unsupported keystring = $sa")
        error("Not yet implemented !")

        #= * define N normalized states in stoplus::Array{Vector{Float64},1} and stominus::Array{Vector{Float64},1}
            on the given grid and with given parameters
        * calculate one-electron (NxN) Hamiltonian matrix with potential h_pq and overlap S_pq, p,q = 1..N
        * solve generalized eigenvalue problem h c_vec = epsilon S c_vec
        * set spectrum::SingleElecSpectrum with just N0 functions
        * check normalization and orthogonality of the functions   =#

        return( nothing )
    end

    """
    `Basics.generateConfigurationsForExcitationScheme(confs::Array{Configuration,1}, exScheme::Basics.NoExcitationScheme,
                                                    nMax::Int64, lValues::Array{Int64,1})`
        ... generates a list of non-relativistic configurations for the given (reference) confs and the excitation scheme.
            All orbitals in standard order are considered up to maxShell = (n_max, l_max).
    """
    function Basics.generateConfigurationsForExcitationScheme(confs::Array{Configuration,1}, exScheme::Basics.NoExcitationScheme,
                                                            nMax::Int64, lValues::Array{Int64,1})
        @warn(  "No excitations are included if exScheme::NoExcitationScheme. ")
        println("No excitations are included if exScheme::NoExcitationScheme. ")
        #
        newConfList = confs
        return( newConfList )
    end

    """
    `Basics.generateConfigurationsForExcitationScheme(confs::Array{Configuration,1}, exScheme::Basics.DeExciteSingleElectron,
                                                    nMax::Int64, lValues::Array{Int64,1})`
        ... generates a list of non-relativistic configurations for the given (reference) confs and the excitation scheme.
            All orbitals in standard order are considered up to maxShell = (n_max, l_max).
    """
    function Basics.generateConfigurationsForExcitationScheme(confs::Array{Configuration,1}, exScheme::Basics.DeExciteSingleElectron,
                                                            nMax::Int64, lValues::Array{Int64,1})
        confList = Configuration[];    NoElectrons = confs[1].NoElectrons
        shellList   = Basics.generateShellList(confs, nMax, lValues)

        # Create all configurations with single excitations from the given list of configurations
        for  conf in confs
            # Take one electron fromShell and add toShell
            for  (fromShell,occ)  in  conf.shells
                for  toShell  in  shellList
                    newShells = deepcopy( conf.shells )
                    ##x print("newShells = $newShells   fromShell = $fromShell    toShell = $toShell")
                    if      fromShell == toShell
                    elseif  haskey(conf.shells, toShell )   fromOcc = occ;   toOcc = conf.shells[toShell]
                            if  fromOcc - 1 < 0                        println("");    continue    end
                            if  toOcc   + 1 > 2*(2*toShell.l + 1)      println("");    continue    end
                            newShells[fromShell] = newShells[fromShell] - 1
                            newShells[toShell]   = newShells[toShell] + 1
                    else    fromOcc = occ
                            if  fromOcc - 1 < 0                        println("");    continue    end
                            newShells[fromShell] = newShells[fromShell] - 1
                            newShells = Base.merge( newShells, Dict( toShell => 1))
                    end
                    # Add a new configuration
                    push!( confList, Configuration( newShells, NoElectrons))
                    println("  newconf = $(Configuration( newShells, NoElectrons))")
                end
            end
        end

        nbefore     = length(confList)
        newConfList = Configuration[]
        for  conf  in  confList
            if  conf in newConfList    continue;    else    push!( newConfList,  conf)    end
        end
        nafter      = length(newConfList)
        println("Basics.generateConfigurationsForExcitationScheme():: Number of configs is $nbefore (before) and $nafter (after).")

        #
        ##x push!( newConfList, confs[1])
        ##x push!( newConfList, Configuration("[Ne] 3s 3p^6 4s") )
        ##x push!( newConfList, Configuration("[Ne] 3s 3p^6 5s") )
        return( newConfList )
    end

    """
    `Basics.generateLevelWithExtraElectron(newOrbital, symt::LevelSymmetry, level::Level)`
        ... generates a (new) level with one extra electron in subshell newOrbital.subshell and with overall symmetry symt. The function
            assumes that all CSF in the basis of level have the same symmetry as the level itself, that the new subshell is not yet part
            of the basis and that CSF with the total symmetry can be constructed; it terminates with an error if one of this assumptions
            is violated. A newLevel::Level with the same number of CSF and the same representation (eigenvector) as the given level is
            returned. From a physics viewpoint, newLevel refers to a antisymmetrized product state of [level x orbital(sh, energy)] J^P
            (symt) and with an total energy = level.energy + newOrbital.energy.
    """
    function Basics.generateLevelWithExtraElectron(newOrbital, symt::LevelSymmetry, level::Level)
        basis = level.basis;    newSubshells = copy(basis.subshells);    newCsfs = CsfR[];   J = level.J;   parity = level.parity

        push!(newSubshells, newOrbital.subshell)
        Defaults.setDefaults("relativistic subshell list", newSubshells; printout=false)
        newOrbitals = Base.merge( copy(basis.orbitals), Dict( newOrbital.subshell => newOrbital ))

        for  i = 1:length(basis.csfs)
            stateList   = ManyElectron.provideSubshellStates(newOrbital.subshell, 1)
            if  basis.csfs[i].J != J   ||   basis.csfs[i].parity != parity    error("Improper symmetry of CSF.")             end
            if  length(stateList) != 1                                        error("Improper number of subshell states.")   end
            if  AngularMomentum.triangularDelta(J, Basics.subshell_j(newOrbital.subshell), symt.J) != 1
                                                                            error("Improper coupling of (new) subshell & total J.")   end
            substate   = stateList[1]
            occupation = copy(basis.csfs[i].occupation);    push!(occupation, substate.occ)
            seniority  = copy(basis.csfs[i].seniority);     push!(seniority,  substate.nu)
            subshellJ  = copy(basis.csfs[i].subshellJ);     push!(subshellJ,  AngularJ64(substate.Jsub2//2) )
            subshellX  = copy(basis.csfs[i].subshellX);     push!(subshellX,  symt.J)
            push!(newCsfs, CsfR(true, symt.J, symt.parity, occupation, seniority, subshellJ, subshellX, Subshell[] ) )
        end

        newBasis = Basis(true, basis.NoElectrons+1, newSubshells, newCsfs, copy(basis.coreSubshells), newOrbitals)
        newLevel = Level(symt.J, AngularM64(symt.J), symt.parity, level.index, level.energy+newOrbital.energy, level.relativeOcc,
                        true, newBasis, level.mc)
        return( newLevel )
    end

    """
    `Basics.generateLevelWithExtraSubshell(sh::Subshell, level::Level)`
        ... generates a (new) level with one extra subshell sh but with the same overall symmetry as before. The function assumes that the
            new subshell is not yet part of the basis; it terminates with an error if one of this assumptions is violated.
            A newLevel::Level with the same symmetry, number of CSF, energy and the same representation (eigenvector) as the given
            level is returned.
    """
    function Basics.generateLevelWithExtraSubshell(sh::Subshell, level::Level)
        basis = level.basis;    newSubshells = copy(basis.subshells);    newCsfs = CsfR[];   J = level.J;   parity = level.parity
        newOrbitals = copy(basis.orbitals)

        push!(newSubshells, sh)
        Defaults.setDefaults("relativistic subshell list", newSubshells; printout=false)

        for  i = 1:length(basis.csfs)
            stateList   = ManyElectron.provideSubshellStates(sh, 0)
            if  length(stateList) != 1                                        error("Improper number of subshell states.")   end
            substate   = stateList[1]
            occupation = copy(basis.csfs[i].occupation);    push!(occupation, substate.occ)
            seniority  = copy(basis.csfs[i].seniority);     push!(seniority,  substate.nu)
            subshellJ  = copy(basis.csfs[i].subshellJ);     push!(subshellJ,  AngularJ64(substate.Jsub2//2) )
            subshellX  = copy(basis.csfs[i].subshellX);     push!(subshellX,  subshellX[end])
            push!(newCsfs, CsfR(true, basis.csfs[i].J, basis.csfs[i].parity, occupation, seniority, subshellJ, subshellX, Subshell[] ) )
        end

        newBasis = Basis(true, basis.NoElectrons, newSubshells, newCsfs, copy(basis.coreSubshells), newOrbitals)
        newLevel = Level(level.J, level.M, level.parity, level.index, level.energy, level.relativeOcc, true, newBasis, level.mc)
        return( newLevel )
    end

    """
    `Basics.generateLevelWithSymmetryReducedBasis(level::Level)`
        ... generates a level with a new basis and representation that only includes the CSF with the same symmetry as the level
            itself. Both, the underlying CSF basis and the eigenvectors are adopted accordingly. A (new) level::levelR is returned.
    """
    function Basics.generateLevelWithSymmetryReducedBasis(level::Level)
        basis = level.basis;    newCsfs = CsfR[];    newMc = Float64[]

        for  i = 1:length(basis.csfs)
            if  basis.csfs[i].J == level.J  &&   basis.csfs[i].parity == level.parity
                push!(newCsfs, basis.csfs[i]);    push!(newMc, level.mc[i])
            end
        end
        newBasis = Basis(true, basis.NoElectrons, basis.subshells, newCsfs, basis.coreSubshells, basis.orbitals)
        newLevel = Level(level.J, level.M, level.parity, level.index, level.energy, level.relativeOcc, true, newBasis, newMc)
        return( newLevel )
    end

    """
    `Basics.generateShellList(confs::Array{Configuration,1}, nMax::Int64, lValues::Array{Int64,1})`
        ... generates a list of (non-relativistic) shells in standard order that contains all shells from the given list of
            configurations as well as the shells up to the principal quantum number nMax and the list of orbital quantum
            numbers lValues. A shellList::Array{Shell,1} is returned.
    """
    function Basics.generateShellList(confs::Array{Configuration,1}, nMax::Int64, lValues::Array{Int64,1})
        shellList = Shell[]
        # Add all shells from the given configurations
        for  conf in confs
            for (k,v) in conf.shells    push!( shellList, k)    end
        end
        # Add all shells for the given quantum numbers
        for  n = 1:nMax
            for  ll in  lValues
                if  n >= ll + 1     push!( shellList, Shell(n,ll))      end
            end
        end
        ##x println("shellList = $shellList ")

        # Now bring the shells in standard order
        newShellList = Shell[]
        for  n = 1:nMax
            for  l = 0:30
                if  n >= l + 1  && Shell(n,l) in shellList    push!( newShellList, Shell(n,l))     end
            end
        end
        println("newShellList = $newShellList ")

        return( newShellList )
    end

end # module


"""
    propagate(F::judiPropagator{T, mode}, q)

Base propagation interfaces that calls the devito `mode` propagator (forward/adjoint/..)
with `q` as a source. The return type is infered from `F`.
"""
function propagate(F::judiPropagator{T, O}, q::AbstractArray{T}) where {T, O}
    srcGeometry, srcData, recGeometry, recData, dm = make_input(F, q)
    return time_modeling(F.model, srcGeometry, srcData, recGeometry, recData, dm, O, F.options)
end

function propagate(F::judiPropagator{T, :adjoint}, q::AbstractArray{T}) where {T}
    srcGeometry, srcData, recGeometry, recData, dm = make_input(F, q)
    return time_modeling(F.model, recGeometry, recData, srcGeometry, srcData, dm, :adjoint, F.options)
end

propagate(t::Tuple{judiPropagator, AbstractArray}) = propagate(t[1], t[2])

"""
    run_and_reduce(func, pool, nsrc, arg_func)

Runs the function `func` for indices `1:nsrc` within arguments `func(arg_func(i))`. If the
the pool is empty, a standard loop and accumulation is ran. If the pool is a julia WorkerPool or
any custom Distributed pool, the loop is distributed via `remotecall` followed by are binary tree remote reduction.
"""
function run_and_reduce(func, pool, nsrc, arg_func::Function)
    res = Vector{_TFuture}(undef, nsrc)
    for i = 1:nsrc
        args_loc = arg_func(i)
        res[i] = remotecall(func, pool, args_loc...)
    end
    res = reduce!(res)
    return res
end

run_and_reduce(func, ::Nothing, nsrc, arg_func::Function) = mapreduce(i -> func(arg_func(i)...), single_reduce!, 1:nsrc)

src_i(::judiAbstractJacobian{T, :born, FT}, q::dmType{T}, ::Integer) where {T<:Number, FT} = q
src_i(::judiPropagator{T, O}, q::judiMultiSourceVector{T}, i::Integer) where {T, O} = q[i]
src_i(::judiPropagator{T, O}, q::Vector{<:Array{T}}, i::Integer) where {T, O} = q[i]

get_nsrc(::judiPropagator, q::judiMultiSourceVector) = q.nsrc
get_nsrc(::judiPropagator, q::Vector{<:Array}) = length(q)
get_nsrc(J::judiAbstractJacobian, ::dmType{T}) where T<:Number = J.q.nsrc

"""
    multi_src_propagate(F::judiPropagator{T, O}, q::AbstractVector)

Propagates the source `q` with the `F` propagator. The return type is infered from `F` and the
propagation kernel is defined by `O` (forward, adjoint, born or adjoint_born).
"""
function multi_src_propagate(F::judiPropagator{T, O}, q::AbstractVector{T}) where {T<:Number, O}
    q = process_input_data(F, q)
    # Number of sources and init result
    nsrc = get_nsrc(F, q)
    pool = _worker_pool()
    arg_func = i -> (F[i], src_i(F, q, i))
    # Distribute source
    res = run_and_reduce(propagate, pool, nsrc, arg_func)
    return as_vec(res, Val(F.options.return_array))
end

"""
    multi_src_fg!(G, model, q, dobs, dm; options=Options(), nlind=false, lin=false)

This is the main multi-source wrapper function for `fwi_objective` and `lsrtm_objective`.
Computes the misifit and gradient (LSRTM if `lin` else FWI) for the given `q` source and `dobs` and
perturbation `dm`.
"""
function multi_src_fg!(G, model, q, dobs, dm; options=Options(), kw...)
    check_non_indexable(kw)
    # Number of sources and init result
    nsrc = try q.nsrc catch; dobs.nsrc end
    pool = _worker_pool()
    # Distribute source
    arg_func = i -> (model, q[i], dobs[i], dm, options[i], values(kw)...)
    # Distribute source
    res = run_and_reduce(multi_src_fg, pool, nsrc, arg_func)
    f, g = as_vec(res, Val(options.return_array))
    G .= g
    return f
end

check_non_indexable(d::Dict) = for (k, v) in d check_non_indexable(v) end
check_non_indexable(x) = false
check_non_indexable(::Bool) = true
check_non_indexable(::Number) = true
check_non_indexable(::judiMultiSourceVector) = throw(ArgumentError("Keyword arguments must not be source dependent"))
function check_non_indexable(::AbstractArray)
    @warn "keyword arguement Array considered source independent and copied to all workers"
    true
end


########################### Thermal Generation Models ######################################
abstract type AbstractThermalFormulation <: AbstractDeviceFormulation end
abstract type AbstractThermalDispatchFormulation <: AbstractThermalFormulation end
abstract type AbstractThermalUnitCommitment <: AbstractThermalFormulation end
struct ThermalBasicUnitCommitment <: AbstractThermalUnitCommitment end
struct ThermalStandardUnitCommitment <: AbstractThermalUnitCommitment end
struct ThermalDispatch <: AbstractThermalDispatchFormulation end
struct ThermalRampLimited <: AbstractThermalDispatchFormulation end
struct ThermalDispatchNoMin <: AbstractThermalDispatchFormulation end

########################### Active Dispatch Variables ######################################
"""
This function add the variables for power generation output to the model
"""
function activepower_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
) where {T <: PSY.ThermalGen}
    if get_warm_start(psi_container.settings)
        initial_value = d -> PSY.get_activepower(d)
    else
        initial_value = nothing
    end
    add_variable(
        psi_container,
        devices,
        variable_name(ACTIVE_POWER, T),
        false,
        :nodal_balance_active;
        ub_value = d -> PSY.get_activepowerlimits(d).max,
        lb_value = d -> PSY.get_activepowerlimits(d).min,
        init_value = initial_value,
    )
    return
end

"""
This function add the variables for power generation output to the model
"""
function reactivepower_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
) where {T <: PSY.ThermalGen}
    if get_warm_start(psi_container.settings)
        initial_value = d -> PSY.get_activepower(d)
    else
        initial_value = nothing
    end
    add_variable(
        psi_container,
        devices,
        variable_name(REACTIVE_POWER, T),
        false,
        :nodal_balance_reactive;
        ub_value = d -> PSY.get_reactivepowerlimits(d).max,
        lb_value = d -> PSY.get_reactivepowerlimits(d).min,
        init_value = initial_value,
    )
    return
end

"""
This function add the variables for power generation commitment to the model
"""
function commitment_variables!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
) where {T <: PSY.ThermalGen}
    time_steps = model_time_steps(psi_container)
    if get_warm_start(psi_container.settings)
        initial_value = d -> (PSY.get_activepower(d) > 0 ? 1.0 : 0.0)
    else
        initial_value = nothing
    end

    add_variable(psi_container, devices, variable_name(ON, T), true)
    var_names = (variable_name(START, T), variable_name(STOP, T))
    for v in var_names
        add_variable(psi_container, devices, v, true; init_value = initial_value)
    end

    return
end

activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{<:PSY.ThermalGen},
    model::DeviceModel{<:PSY.ThermalGen, <:AbstractThermalFormulation},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::SemiContinuousFF,
) = nothing

activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{<:PSY.ThermalGen},
    model::DeviceModel{<:PSY.ThermalGen, ThermalDispatchNoMin},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::SemiContinuousFF,
) = nothing

"""
This function adds the active power limits of generators when there are no CommitmentVariables
"""
function activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, <:AbstractThermalDispatchFormulation},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Nothing,
) where {T <: PSY.ThermalGen}
    constraint_data = Vector{DeviceRange}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        name = PSY.get_name(d)
        limits = PSY.get_activepowerlimits(d)
        range_data = DeviceRange(name, limits)
        add_device_services!(range_data, d, model)
        constraint_data[ix] = range_data
    end
    device_range(
        psi_container,
        constraint_data,
        constraint_name(ACTIVE_RANGE, T),
        variable_name(ACTIVE_POWER, T),
    )
    return
end

"""
This function adds the active power limits of generators when there are CommitmentVariables
"""
function activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, <:AbstractThermalFormulation},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Nothing,
) where {T <: PSY.ThermalGen}
    constraint_data = Vector{DeviceRange}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        limits = PSY.get_activepowerlimits(d)
        name = PSY.get_name(d)
        range_data = DeviceRange(name, limits)
        add_device_services!(range_data, d, model)
        constraint_data[ix] = range_data
    end
    device_semicontinuousrange(
        psi_container,
        constraint_data,
        constraint_name(ACTIVE_RANGE, T),
        variable_name(ACTIVE_POWER, T),
        variable_name(ON, T),
    )
    return
end

"""
This function adds the active power limits of generators when there are
    no CommitmentVariables
"""
function activepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, ThermalDispatchNoMin},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Nothing,
) where {T <: PSY.ThermalGen}
    constraint_data = Vector{DeviceRange}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        limits = (min = 0.0, max = PSY.get_activepowerlimits(d).max)
        name = PSY.get_name(d)
        range_data = DeviceRange(name, limits)
        add_device_services!(range_data, d, model)
        constraint_data[ix] = range_data
    end

    var_key = variable_name(ACTIVE_POWER, T)
    variable = get_variable(psi_container, var_key)
    # If the variable was a lower bound != 0, not removing the LB can cause infeasibilities
    for v in variable
        if JuMP.has_lower_bound(v)
            JuMP.set_lower_bound(v, 0.0)
        end
    end

    device_range(
        psi_container,
        constraint_data,
        constraint_name(ACTIVE_RANGE, T),
        variable_name(ACTIVE_POWER, T),
    )
    return
end

"""
This function adds the reactive  power limits of generators when there are CommitmentVariables
"""
function reactivepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, <:AbstractThermalDispatchFormulation},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen}
    constraint_data = Vector{DeviceRange}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        name = PSY.get_name(d)
        limits = PSY.get_reactivepowerlimits(d)
        range_data = DeviceRange(name, limits)
        #add_device_services!(range_data, d, model)
        # Uncomment when we implement reactive power services
        constraint_data[ix] = range_data
    end

    device_range(
        psi_container,
        constraint_data,
        constraint_name(REACTIVE_RANGE, T),
        variable_name(REACTIVE_POWER, T),
    )
    return
end

"""
This function adds the reactive power limits of generators when there CommitmentVariables
"""
function reactivepower_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, <:AbstractThermalFormulation},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen}
    constraint_data = Vector{DeviceRange}(undef, length(devices))
    for (ix, d) in enumerate(devices)
        limits = PSY.get_reactivepowerlimits(d)
        name = PSY.get_name(d)
        range_data = DeviceRange(name, limits)
        #add_device_services!(range_data, d, model)
        # Uncomment when we implement reactive power services
        constraint_data[ix] = range_data
    end

    device_semicontinuousrange(
        psi_container,
        constraint_data,
        constraint_name(REACTIVE_RANGE, T),
        variable_name(REACTIVE_POWER, T),
        variable_name(ON, T),
    )
    return
end

### Constraints for Thermal Generation without commitment variables ####
"""
This function adds the Commitment Status constraint when there are CommitmentVariables
"""
function commitment_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, D},
    ::Type{S},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen, D <: AbstractThermalFormulation, S <: PM.AbstractPowerModel}
    device_commitment(
        psi_container,
        get_initial_conditions(psi_container, ICKey(DeviceStatus, T)),
        constraint_name(COMMITMENT, T),
        (variable_name(START, T), variable_name(STOP, T), variable_name(ON, T)),
    )
    return
end

########################## Make initial Conditions for a Model #############################
function initial_conditions!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    device_formulation::Type{D},
) where {T <: PSY.ThermalGen, D <: AbstractThermalUnitCommitment}
    status_init(psi_container.initial_conditions, devices)
    output_init(psi_container.initial_conditions, devices)
    duration_init(psi_container.initial_conditions, devices)
    return
end

function initial_conditions!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    device_formulation::Type{ThermalBasicUnitCommitment},
) where {T <: PSY.ThermalGen}
    status_init(psi_container.initial_conditions, devices)
    output_init(psi_container.initial_conditions, devices)
    return
end

function initial_conditions!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    device_formulation::Type{D},
) where {T <: PSY.ThermalGen, D <: AbstractThermalDispatchFormulation}
    output_init(psi_container.initial_conditions, devices)
    return
end

########################### Ramp/Rate of Change constraints ################################
"""
This function gets the data for the generators
"""
function _get_data_for_rocc(
    initial_conditions::Vector{InitialCondition},
    resolution::Dates.TimePeriod,
)
    if resolution > Dates.Minute(1)
        minutes_per_period = Dates.value(Dates.Minute(resolution))
    else
        minutes_per_period = Dates.value(Dates.Second(resolution)) / 60
    end
    lenght_devices = length(initial_conditions)
    ini_conds = Vector{InitialCondition}(undef, lenght_devices)
    ramp_params = Vector{UpDown}(undef, lenght_devices)
    minmax_params = Vector{MinMax}(undef, lenght_devices)
    idx = 0
    for ic in initial_conditions
        g = ic.device
        name = PSY.get_name(g)
        non_binding_up = false
        non_binding_down = false
        ramplimits = PSY.get_ramplimits(g)
        rating = PSY.get_rating(g)
        if !isnothing(ramplimits)
            p_lims = PSY.get_activepowerlimits(g)
            max_rate = abs(p_lims.min - p_lims.max) / minutes_per_period
            if (ramplimits.up * rating >= max_rate) & (ramplimits.down * rating >= max_rate)
                @debug "Generator $(name) has a nonbinding ramp limits. Constraints Skipped"
                continue
            else
                idx += 1
            end
            ini_conds[idx] = ic
            ramp_params[idx] = (
                up = ramplimits.up * rating * minutes_per_period,
                down = ramplimits.down * rating * minutes_per_period,
            )
            minmax_params[idx] = p_lims
        end
    end
    if idx < lenght_devices
        deleteat!(ini_conds, (idx + 1):lenght_devices)
        deleteat!(ramp_params, (idx + 1):lenght_devices)
        deleteat!(minmax_params, (idx + 1):lenght_devices)
    end
    return ini_conds, ramp_params, minmax_params
end

"""
This function adds the ramping limits of generators when there are CommitmentVariables
"""
function ramp_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, D},
    system_formulation::Type{S},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen, D <: AbstractThermalFormulation, S <: PM.AbstractPowerModel}
    time_steps = model_time_steps(psi_container)
    resolution = model_resolution(psi_container)
    key = ICKey(DevicePower, T)
    initial_conditions = get_initial_conditions(psi_container, key)
    rate_data = _get_data_for_rocc(initial_conditions, resolution)
    ini_conds, ramp_params, minmax_params =
        _get_data_for_rocc(initial_conditions, resolution)
    if !isempty(ini_conds)
        # Here goes the reactive power ramp limits when versions for AC and DC are added
        device_mixedinteger_rateofchange(
            psi_container,
            (ramp_params, minmax_params),
            ini_conds,
            constraint_name(RAMP, T),
            (
                variable_name(ACTIVE_POWER, T),
                variable_name(START, T),
                variable_name(STOP, T),
            ),
        )
    else
        @warn "Data doesn't contain generators with ramp limits, consider adjusting your formulation"
    end
    return
end

function ramp_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, D},
    system_formulation::Type{S},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {
    T <: PSY.ThermalGen,
    D <: AbstractThermalDispatchFormulation,
    S <: PM.AbstractPowerModel,
}
    time_steps = model_time_steps(psi_container)
    resolution = model_resolution(psi_container)
    initial_conditions = get_initial_conditions(psi_container, ICKey(DevicePower, T))
    rate_data = _get_data_for_rocc(initial_conditions, resolution)
    ini_conds, ramp_params, minmax_params =
        _get_data_for_rocc(initial_conditions, resolution)
    if !isempty(ini_conds)
        # Here goes the reactive power ramp limits when versions for AC and DC are added
        device_linear_rateofchange(
            psi_container,
            ramp_params,
            ini_conds,
            constraint_name(RAMP, T),
            variable_name(ACTIVE_POWER, T),
        )
    else
        @warn "Data doesn't contain generators with ramp limits, consider adjusting your formulation"
    end
    return
end

########################### time duration constraints ######################################
"""
If the fraction of hours that a generator has a duration constraint is less than
the fraction of hours that a single time_step represents then it is not binding.
"""
function _get_data_for_tdc(
    initial_conditions_on::Vector{InitialCondition},
    initial_conditions_off::Vector{InitialCondition},
    resolution::Dates.TimePeriod,
)

    steps_per_hour = 60 / Dates.value(Dates.Minute(resolution))
    fraction_of_hour = 1 / steps_per_hour
    lenght_devices_on = length(initial_conditions_on)
    lenght_devices_off = length(initial_conditions_off)
    @assert lenght_devices_off == lenght_devices_on
    time_params = Vector{UpDown}(undef, lenght_devices_on)
    ini_conds = Matrix{InitialCondition}(undef, lenght_devices_on, 2)
    idx = 0
    for (ix, ic) in enumerate(initial_conditions_on)
        g = ic.device
        @assert g == initial_conditions_off[ix].device
        non_binding_up = false
        non_binding_down = false
        timelimits = PSY.get_timelimits(g)
        name = PSY.get_name(g)
        if !isnothing(timelimits)
            if (timelimits.up <= fraction_of_hour) & (timelimits.down <= fraction_of_hour)
                @debug "Generator $(name) has a nonbinding time limits. Constraints Skipped"
            else
                idx += 1
            end
            ini_conds[idx, 1] = ic
            ini_conds[idx, 2] = initial_conditions_off[ix]
            up_val = round(timelimits.up * steps_per_hour, RoundUp)
            down_val = round(timelimits.down * steps_per_hour, RoundUp)
            time_params[idx] = time_params[idx] = (up = up_val, down = down_val)
        end
    end
    if idx < lenght_devices_on
        ini_conds = ini_conds[1:idx, :]
        deleteat!(time_params, (idx + 1):lenght_devices_on)
    end
    return ini_conds, time_params
end

function time_constraints!(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, D},
    system_formulation::Type{S},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen, D <: AbstractThermalFormulation, S <: PM.AbstractPowerModel}
    parameters = model_has_parameters(psi_container)
    resolution = model_resolution(psi_container)
    initial_conditions_on = get_initial_conditions(psi_container, ICKey(TimeDurationON, T))
    initial_conditions_off =
        get_initial_conditions(psi_container, ICKey(TimeDurationOFF, T))
    ini_conds, time_params =
        _get_data_for_tdc(initial_conditions_on, initial_conditions_off, resolution)
    if !(isempty(ini_conds))
        if parameters
            device_duration_parameters(
                psi_container,
                time_params,
                ini_conds,
                constraint_name(DURATION, T),
                (variable_name(ON, T), variable_name(START, T), variable_name(STOP, T)),
            )
        else
            device_duration_retrospective(
                psi_container,
                time_params,
                ini_conds,
                constraint_name(DURATION, T),
                (variable_name(ON, T), variable_name(START, T), variable_name(STOP, T)),
            )
        end
    else
        @warn "Data doesn't contain generators with time-up/down limits, consider adjusting your formulation"
    end
    return
end

########################### Cost Function Calls#############################################
function cost_function(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    ::Type{<:AbstractThermalDispatchFormulation},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen}
    add_to_cost(psi_container, devices, variable_name(ACTIVE_POWER, T), :variable)
    return
end

function cost_function(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    ::Type{ThermalDispatchNoMin},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen}
    resolution = model_resolution(psi_container)
    dt = Dates.value(Dates.Minute(resolution)) / 60
    variable = get_variable(psi_container, variable_name(ACTIVE_POWER, T))

    # uses the same cost function whenever there is NO PWL
    function _ps_cost(d::PSY.ThermalGen, cost_component::PSY.VariableCost)
        return ps_cost(
            psi_container,
            variable_name(ACTIVE_POWER, T),
            PSY.get_name(d),
            cost_component,
            dt,
            1.0,
        )
    end

    # This function modified the PWL cost data when present
    function _ps_cost(
        d::PSY.ThermalGen,
        cost_component::PSY.VariableCost{Vector{NTuple{2, Float64}}},
    )
        gen_cost = JuMP.GenericAffExpr{Float64, _variable_type(psi_container)}()
        if !haskey(psi_container.variables, :PWL_cost_vars)
            time_steps = model_time_steps(psi_container)
            container = add_var_container!(
                psi_container,
                :PWL_cost_vars,
                PSY.get_name(d),
                time_steps,
                1:length(cost_component);
                sparse = true,
            )
        else
            container = get_variable(psi_container, :PWL_cost_vars)
        end
        for (t, var) in enumerate(variable[PSY.get_name(d), :])
            pwlvars = JuMP.@variable(
                psi_container.JuMPmodel,
                [i = 1:length(cost_component)],
                base_name = "{$(variable)}_{pwl}",
                start = 0.0,
                lower_bound = 0.0,
                upper_bound = PSY.get_breakpoint_upperbounds(cost_component)[i]
            )
            slopes = PSY.get_slopes(cost_component)
            first_pair = cost_component.cost[1]
            if slopes[1] != 0.0
                slopes[1] =
                    (
                        first_pair[1]^2 - slopes[1] * first_pair[2] +
                        COST_EPSILON * first_pair[2]
                    ) / (first_pair[1] * first_pair[2])
            end
            if slopes[1] < 0 || slopes[1] <= slopes[2]
                throw(IS.ConflictingInputsError("The PWL cost data provided for generator $(PSY.get_name(d)) is not compatible with a No Min Cost."))
            end
            for (ix, pwlvar) in enumerate(pwlvars)
                JuMP.add_to_expression!(gen_cost, slopes[ix] * pwlvar)
                container[(PSY.get_name(d), t, ix)] = pwlvar
            end

            c = JuMP.@constraint(
                psi_container.JuMPmodel,
                variable == sum([pwlvar for (ix, pwlvar) in enumerate(pwlvars)])
            )
            JuMP.add_to_expression!(gen_cost, c)
        end
        return sign * gen_cost * d
    end

    for d in devices
        cost_component = PSY.get_variable(PSY.get_op_cost(d))
        cost_expression = _ps_cost(d, cost_component)
        T_ce = typeof(cost_expression)
        T_cf = typeof(psi_container.cost_function)
        if T_cf <: JuMP.GenericAffExpr && T_ce <: JuMP.GenericQuadExpr
            psi_container.cost_function += cost_expression
        else
            JuMP.add_to_expression!(psi_container.cost_function, cost_expression)
        end
    end

    return
end

function cost_function(
    psi_container::PSIContainer,
    devices::IS.FlattenIteratorWrapper{T},
    ::Type{<:AbstractThermalFormulation},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, AbstractAffectFeedForward},
) where {T <: PSY.ThermalGen}
    #Variable Cost component
    add_to_cost(psi_container, devices, variable_name(ACTIVE_POWER, T), :variable)
    #Commitment Cost Components
    add_to_cost(psi_container, devices, variable_name(START, T), :startup)
    add_to_cost(psi_container, devices, variable_name(STOP, T), :shutdn)
    add_to_cost(psi_container, devices, variable_name(ON, T), :fixed)
    return
end


struct ScatteredArray{E,M,T,N,Np1} <: AbstractScatteredArray{E,M,T,N,Np1}
    data::Array{E,Np1}
end
struct VectorizedScatteredArray{E,M,T,N,Np1} <: AbstractScatteredArray{E,M,T,N,Np1}
    ptr::Ptr{E}
    size::NTuple{Np1,Int}
end
struct ScatteredArrayView{E,M,T,new_N,N,Np1,V} <: AbstractScatteredArray{E,M,T,new_N,Np1}
    ptr::Ptr{E}
    size::NTuple{new_N,Int}
    full_size::NTuple{Np1,Int}
    view::V
end

"""
Data axis for a chunked_array of mats are:
W x (chunked_array[2:N]) x mat x chunked_array[1] ÷ W
"""
struct ChunkedArray{E,M,T,N,Np2,U<:Unsigned} <: AbstractScatteredArray{E,M,T,N,Np2}
    data::Array{E,Np2}
    size::NTuple{N,Int}
    mask::U
end
struct VectorizedChunkedArray{E,M,T,N,Np2} <: AbstractScatteredArray{E,M,T,N,Np2}
    ptr::Ptr{E}
    fullsize::NTuple{Np2,Int}
end
struct ChunkedArrayView{E,M,T,new_N,N,Np2,V,U<:Unsigned} <: AbstractScatteredArray{E,M,T,new_N,Np2}
    ptr::Ptr{E}
    size::NTuple{new_N,Int}
    full_size::NTuple{Np2,Int}
    view::V
    mask::U
end

const AbstractVectorizedScatteredArray{E,M,T,N,Np1} = Union{VectorizedScatteredArray{E,M,T,N,Np1},VectorizedChunkedArray{E,M,T,N,Np1}}
const AbstractScatteredArrayView{E,M,T,new_N,N,Np1,V} = Union{ScatteredArrayView{E,M,T,new_N,N,Np1,V},ChunkedArrayView{E,M,T,new_N,N,Np1,V}}

@inline Base.pointer(ScA::ScatteredArray) = pointer(ScA.data)
@inline Base.pointer(ScA::Union{AbstractVectorizedScatteredArray,AbstractScatteredArrayView}) = ScA.ptr
@inline Base.unsafe_convert(::Type{Ptr{T}}, ScA::AbstractScatteredArray{T}) where {T} = pointer(ScA)
@inline function Base.unsafe_convert(::Type{Ptr{T}}, ScA::AbstractScatteredArray) where {T}
    Base.unsafe_convert(Ptr{T}, pointer(ScA))
end
@inline Base.convert(::Type{Ptr{T}}, ScA::AbstractScatteredArray) where T = Base.unsafe_convert(Ptr{T}, ScA)

@generated function Base.length(vsa::AbstractVectorizedScatteredArray{E,M,T,N}) where {E,M,T,N}
    quote
        $(Expr(:meta,:inline))
        $(Expr(:call, :*, [:(vsa.size[$n]) for n ∈ 1:N]...))
    end
end
@inline Base.size(ScA::AbstractScatteredArrayView) = ScA.size

"""
Expression for constructing an instance of T from the arguments.
The default constructor is `Expr(:call, T, args...)`.

For example,

T = SVector{4,Float64};
args = [:a,:b,:c,:d];
Expr(:call, T, args...)
# :((SArray{Tuple{4},Float64,1,4})(a, b, c, d))

defines how to construct the type. When extending this to custom types, either ensure
that the default constuctor is supported, or define your own `construct_expr` method.
For example, when `T <: Tuple`, `construct_expr` dispatches to:
    Expr(:tuple, args...)
instead.

"""
function construct_expr(::Type{T}, args) where T
    Expr(:call, T, args...)
end
function construct_expr(::Type{T}, args) where {T <: Tuple}
    Expr(:tuple, args...)
end
# function construct_expr(::Type{T}, args) where {S,T <: SArray{S}}
#     Expr(:call, :(SArray{$S}), args...)
# end
function construct_expr(::Type{T}, args) where {S,T <: ConstantFixedSizeArray{S}}
    Expr(:call, :(ConstantFixedSizeArray{$S}), Expr(:tuple, args...))
end


function AlongTrackAiming(arg0::OneAxisEllipsoid, arg1::Orbit, arg2::jboolean)
    return AlongTrackAiming((OneAxisEllipsoid, Orbit, jboolean), arg0, arg1, arg2)
end

function along_tile_direction(obj::AlongTrackAiming, arg0::Vector3D, arg1::GeodeticPoint)
    return jcall(obj, "alongTileDirection", Vector3D, (Vector3D, GeodeticPoint), arg0, arg1)
end

function equals(obj::Object, arg0::Object)
    return jcall(obj, "equals", jboolean, (Object,), arg0)
end

function get_class(obj::Object)
    return jcall(obj, "getClass", Class, ())
end

function get_singular_points(obj::AlongTrackAiming)
    return jcall(obj, "getSingularPoints", List, ())
end

function hash_code(obj::Object)
    return jcall(obj, "hashCode", jint, ())
end

function notify(obj::Object)
    return jcall(obj, "notify", void, ())
end

function notify_all(obj::Object)
    return jcall(obj, "notifyAll", void, ())
end

function to_string(obj::Object)
    return jcall(obj, "toString", JString, ())
end

function wait(obj::Object)
    return jcall(obj, "wait", void, ())
end

function wait(obj::Object, arg0::jlong)
    return jcall(obj, "wait", void, (jlong,), arg0)
end

function wait(obj::Object, arg0::jlong, arg1::jint)
    return jcall(obj, "wait", void, (jlong, jint), arg0, arg1)
end


module Colorsys

export rgb2yiq, yiq2rgb, rgb2hls, hls2rgb, rgb2hsv, hsv2rgb, yiq2hls, yiq2hsv, hls2yiq, hls2hsv, hsv2yiq, hsv2hls

include("CS.jl")

end # module


"""
`session_info`
"""

function session_info(filepath::String)
    sessionREGEX = match(r"[a-zA-Z]{2}\d+_\d{6}[a-z]{1}",filepath); #the result is a regex object with several info
    if isempty(sessionREGEX.match)
        sessionREGEX = match(r"[a-zA-Z]{1}\d+_\d{6}",filepath);
        if isempty(sessionREGEX.match)
            println(filepath)
            return missing, missing, missing
        else
            session = sessionREGEX.match;
        end
    else
        session = sessionREGEX.match;
    end
    mouse, giorno = split(session,"_")[1:2]
    day = "20"*giorno[1:end-1]
    # String and string are not the same thing don't change them
    mouse = String(mouse)
    day = string(day)
    daily_session = string(giorno[end])
    session = String(session)
    return mouse, day, daily_session, session
end

"""
`get_protocollo(df)`

Python preprocessing creates a Boolean vector to distinguish between 2 Protocollo
relative Gamma and Probability are infered by the respective GammaVec and ProbVec
"""
function get_protocollo(df)
    ProtName = String[]
    for i in 1:size(df,1)
        #in the original data protocol are labeled either 0 or 1
        #this function collapse the respective gamma and prob in a string
        if df[i,:Protocollo] == 0
            curr_prot = string(df[i,:ProbVec0],"/",df[i,:GamVec0])
            push!(ProtName,curr_prot)
        elseif df[i,:Protocollo] == 1
            curr_prot = string(df[i,:ProbVec1],"/",df[i,:GamVec1])
            push!(ProtName,curr_prot)
        end
    end
    #ProtName = categorical(ProtName)
    return ProtName
end

"""
`get_hierarchy`
Return a vector indicating how many consecutive falses or trues
preceded an event
"""

function nextcount(count::T, rewarded) where {T <: Number}
    if rewarded
        count > 0 ? count + 1 : one(T)
    else
        count < 0 ? count - 1 : -one(T)
    end
end
signedcount(v::AbstractArray{Bool}) = accumulate(nextcount, v;init=0.0)
get_hierarchy(v) = lag(signedcount(v), default = NaN)

"""
`create_exp_calendar`
Method1: recount days of experiment
Method2: recount days according to changes in a manipulation
days is a Symbol  for the column that stores the actual days
"""
function create_exp_calendar(df::AbstractDataFrame,days::Symbol)
    lista = sort!(union(df[:,days]))
    Exp_day = Array{Int64,1}()
    Day = []
    for (n,d) in enumerate(lista)
        push!(Exp_day,Int64(n))
        push!(Day,d)
    end
    x = DataFrame(Exp_Day = Exp_day, Day = Day)
    #x[!,days] = Day
    return x
end

function create_exp_calendar(df::AbstractDataFrame,days::Symbol,manipulation::Symbol)
    x = by(df,days) do dd
        DataFrame(Manipulation = join(unique(dd[:, :Protocol]),"*"),
        Flexi = length(union(dd[:,:Protocol])) == 2)
    end
    what = string(manipulation)
    new_name = Symbol(what*"_Day")
    reordered = sortperm(x,(days))
    x = x[reordered,:]
    x[!,new_name] = Flipping.count_series(x[:,:Manipulation])
    return x
end
"""
`add_exp_calendar`
"""
function add_exp_calendar(df,exp_calendar,protocol_calendar)
    df = join(df, exp_calendar, on = [:MouseID,:Day], kind = :inner,makeunique=true);
    df = join(df, protocol_calendar, on = [:MouseID,:Day], kind = :inner,makeunique=true);
    mask = occursin.(String.(names(df)),"_1")
    for x in[names(df)[mask]]
        DataFrames.select!(df,DataFrames.Not(x))
        #deletecols!(pokes, x)
    end
    return df
end


module BrainDecoder

export BData,
       Features,
       save_array,
       load_array

using HDF5
using MAT

# BData ######################################################################

struct BData
    select
    get_labels
    get_label  # For compatibility to bdpy
end

function BData(_file::String)

    # Load Bdata from the given file
    h5 = h5open(_file, "r")
    dataset = read(h5, "dataset")'
    metadata = MetaData(read(h5, "metadata/key"),
                        read(h5, "metadata/value")')
    vmap = read(h5, "vmap")

    close(h5)

    function select(query::String)
        # TODO: add vectorize option
        md_idx = Bool[ query == metadata.key[i] for i = 1:size(metadata.key, 1) ]
        ds_col_idx = Bool[ [1.0] == metadata.value[md_idx, i] for i = 1:size(metadata.value[md_idx, :], 2) ]  # ???
        return dataset[:, ds_col_idx]
    end

    function get_labels(key::String)
        return [vmap[key][string(v)] for v in vec(select(key))]
    end

    function get_label(key::String)
        return get_labels(key)
    end

    return BData(select, get_labels, get_label)
end

struct MetaData
    key::Array{String,1}
    value::Array{AbstractFloat,2}
end

# Features ###################################################################

struct Features
    layers
    labels
    get_features
    get_labels
end

function Features(_dir::String)

    feature_key = "feat"

    # Get layers
    layers = [basename(d) for d in readdir(_dir, join=true, sort=true) if isdir(d)]

    # Get labels
    labels = [splitext(basename(f))[1] for f in readdir(joinpath(_dir, layers[1]), join=true, sort=true) if isfile(f)]

    function get_features(layer::String)
        features = zeros(0)
        for lb in labels
            features = cat(features, _load_features(joinpath(_dir, layer, lb * ".mat"), feature_key), dims=1)
        end
        return features
    end

    function get_labels()
        return labels
    end

    function _load_features(fpath::String, key::String)
        mat = matopen(fpath)
        feat = read(mat, key)
        close(mat)
        return feat
    end

    return Features(layers, labels, get_features, get_labels)
end

# Bdpy sparse array

function save_array(fname, name, array; sparse=false)

    function _transpose(a)
        return permutedims(a, reverse(collect(1:ndims(a))))
    end

    # Conver to row-major order (Python compatilibity)
    shape_array = collect(size(array))
    background = 0
    h5 = h5open(fname, "w")
    if sparse
        index = findall(x -> x != 0, array)
        value = array[index]

        index_fixed = hcat([collect(Tuple(idx)) for idx in index]...)

        index_fixed = _transpose(index_fixed)
        index_fixed = index_fixed .- 1  # Convert to zero-based indexing for Python compatibility

        create_group(h5, name)
        write(h5[name], "__bdpy_sparse_arrray", 1)
        write(h5[name], "shape", shape_array)
        write(h5[name], "background", background)
        write(h5[name], "index", index_fixed)
        write(h5[name], "value", value)
    else
        array = _transpose(array)
        write(h5, name, array)
    end
    close(h5)

end

function load_array(fname, name)

    function _transpose(a)
        return permutedims(a, reverse(collect(1:ndims(a))))
    end

    function _load(obj::HDF5.Dataset)
        return _transpose(read(obj))
    end

    function _load(obj::HDF5.Group)
        is_sparse = read(obj["__bdpy_sparse_arrray"])
        if is_sparse != 1
            # Unsupported
        end

        ary_shape = read(obj["shape"])
        background = read(obj["background"])
        index = read(obj["index"]) .+ 1
        value = read(obj["value"])

        ary = ones(ary_shape...) .* background

        for i = 1:size(index, 1)
            ary[index[i, :]...] = value[i]
        end
        return ary
    end

    h5 = h5open(fname)
    ary = _load(h5[name])
    close(h5)

    return ary
end

end # module BrainDecoder