"""
    function upgrade_branches(
       base_network::String,
       output_location::String,
       fbase;
       indices=[],
       output_type="input"
       )
Creates a folder of network data for the network `base_network` with one line converted to LFAC, one for each index in `indices`, or if `indices` is empty, for every non-transformer branch in the network.
"""
function upgrade_branches(
      base_network::String,
      output_location::String,
      fbase;
      subnet_params=Dict(),
      interface_params=Dict(),
      indices=[],
      output_type="input",
      noloss=false
   )

   PowerModels.logger_config!("error")

   filename = split(base_network, "/")[end]
   println("$filename")
   filetype = split(lowercase(filename), '.')[end]
   base_name = filename[1:(end-length(filetype)-1)]
   (base_name, filetype) = splitext(filename)
   filetype = Unicode.normalize(filetype, casefold=true)
   fileext = filetype == ".m" ? filetype : ".json"
   if lowercase(output_type) == ".json"
      fileext = ".json"
   end
   # println(fileext)
   base_dir = base_network[1:end-length(filename)]

   # println("base_network: $base_network")
   # println("base_dir: $base_dir")

   base_subnet_df = CSV.read("$base_dir/subnetworks.csv", DataFrame, copycols=true)

   # Subnet parameters
   if !("var_f" in keys(subnet_params))
      subnet_params["var_f"] = false
   end
   if !("fmin" in keys(subnet_params))
      subnet_params["fmin"] = fbase
   end
   if !("fmax" in keys(subnet_params))
      subnet_params["fmax"] = fbase
   end
   if !("var_f_lfac" in keys(subnet_params))
      subnet_params["var_f_lfac"] = true
   end
   if !("fbase_lfac" in keys(subnet_params))
      subnet_params["fbase_lfac"] = fbase
   end
   if !("fmin_lfac" in keys(subnet_params))
      subnet_params["fmin_lfac"] = 0.1
   end
   if !("fmax_lfac" in keys(subnet_params))
      subnet_params["fmax_lfac"] = fbase
   end
   if "contingency_file" in names(base_subnet_df)
      ctg_filename = base_subnet_df.contingency_file[1]
      subnet_df = DataFrame(
         index=[1,2],
         file=["$(base_name)$(fileext)", "lfac_$(base_name)$(fileext)"],
         variable_f=[subnet_params["var_f"], subnet_params["var_f_lfac"]],
         f_base=[fbase, subnet_params["fbase_lfac"]],
         f_min=[subnet_params["fmin"], subnet_params["fmin_lfac"]],
         f_max=[subnet_params["fmax"], subnet_params["fmax_lfac"]],
         contingency_file=[ctg_filename, "lfac_$(ctg_filename)"],
         interfaces_file="interfaces_file" in names(base_subnet_df) ? [base_subnet_df.interfaces_file[1], base_subnet_df.interfaces_file[1]] : ["interfaces.csv","interfaces.csv"]
      )
   else
      subnet_df = DataFrame(
         index=[1,2],
         file=["$(base_name)$(fileext)", "lfac_$(base_name)$(fileext)"],
         variable_f=[subnet_params["var_f"], subnet_params["var_f_lfac"]],
         f_base=[fbase, subnet_params["fbase_lfac"]],
         f_min=[subnet_params["fmin"], subnet_params["fmin_lfac"]],
         f_max=[subnet_params["fmax"], subnet_params["fmax_lfac"]],
         interfaces_file="interfaces_file" in names(base_subnet_df) ? [base_subnet_df.interfaces_file[1], base_subnet_df.interfaces_file[1]] : ["interfaces.csv","interfaces.csv"]
      )
   end

   data = PowerModels.parse_file(base_network)

   ctg_br_dict = Dict{Int64, Any}()

   if "contingency_file" in names(subnet_df)
      ctg_list = parse_con_file(base_dir*"/"*subnet_df.contingency_file[1])

      for (ctg_idx,ctg) in enumerate(ctg_list)
         if ctg["component"] == "branch"
            circuit = parse(Int64, ctg["ckt"])

            br_idx = parse(Int64, sort(
               collect(keys(
                  filter(p->(
                     (last(p)["f_bus"]==ctg["i"] && last(p)["t_bus"]==ctg["j"])
                     || (last(p)["f_bus"]==ctg["j"] && last(p)["t_bus"]==ctg["i"])),
                     data["branch"]
                     )
                  )
               )
            )[circuit])

            # println("br_idx: $(br_idx)")
            ctg_br_dict[br_idx] = ctg

         end
      end
   end

   # Converter limits and loss parameters in p.u.
   if !("imax" in keys(interface_params))
      interface_params["imax"] = 40.0
   end
   if !("vmax" in keys(interface_params))
      interface_params["vmax"] = 1.1
   end
   if !("c1" in keys(interface_params))
      interface_params["c1"] = noloss ? 0 : 1e-4
   end
   if !("c2" in keys(interface_params))
      interface_params["c2"] = noloss ? 0 : 1e-4
   end
   if !("c3" in keys(interface_params))
      interface_params["c3"] = noloss ? 0 : 1e-4
   end
   if !("sw1" in keys(interface_params))
      interface_params["sw1"] = noloss ? 0 : 1e-4
   end
   if !("sw2" in keys(interface_params))
      interface_params["sw2"] = noloss ? 0 : 1e-4
   end
   if !("sw3" in keys(interface_params))
      interface_params["sw3"] = noloss ? 0 : 1e-4
   end
   if !("M" in keys(interface_params))
      interface_params["M"] = 0.9
   end
   if !("smax" in keys(interface_params))
      interface_params["smax"] = 50.0
   end
   if !("R" in keys(interface_params))
      interface_params["R"] = noloss ? 0 : 1.0e-5
   end
   if !("X" in keys(interface_params))
      interface_params["X"] = noloss ? 0 : 1.0e-3
   end
   if !("G" in keys(interface_params))
      interface_params["G"] = 0.0
   end
   if !("B" in keys(interface_params))
      interface_params["B"] = noloss ? 0 : 1.0e-4
   end
   if !("transformer" in keys(interface_params))
      interface_params["transformer"] = false
   end
   if !("tap" in keys(interface_params))
      interface_params["tap"] = 1.0
   end
   if !("shift" in keys(interface_params))
      interface_params["shift"] = 0.0
   end
   s_max = 100
   p_max = 0.0
   q_max = 0.0
   loss_param = 0.0
   lfac_vmin = 0.9
   lfac_vmax = 1.1
   # println(indices)
   for (i,(br_idx,branch)) in enumerate(data["branch"])
      lfac_ctg_br_dict = Dict{Int64, Any}()
      ctg_br_dict_tmp = deepcopy(ctg_br_dict)
      br_specified = ((length(indices) == 0) || (parse(Int64, br_idx) in indices)) # Defines whether or not the branch was specified as an input. Include all if empty set was specified.
      if (branch["br_status"]==1) && (!branch["transformer"]) && br_specified
         println("LFAC upgrade on branch $(branch["index"])")

         # iface_keys = ["index", "subnet_index", "bus"]
         # iface_types = [Int64, Int64, Int64]
         # # Add all other parameters in the interface_params dictionary
         # for (key,val) in keys(interface_params)
         #    push!(iface_keys, key)
         #    push!(iface_types, typeof(val))
         # end
         iface_df = DataFrame(index=Int64[],
            subnet_index=Int64[],
            bus=Int64[],
            imax=Float64[],
            vmax=Float64[],
            c1=Float64[],
            c2=Float64[],
            c3=Float64[],
            sw1=Float64[],
            sw2=Float64[],
            sw3=Float64[],
            M=Float64[],
            smax=Float64[],
            R=Float64[],
            X=Float64[],
            G=Float64[],
            B=Float64[],
            transformer=Bool[],
            tap=Float64[],
            shift=Float64[]
            )
         # println("sourceid: $(branch["source_id"])")
         if (length(branch["source_id"])>3) && (Unicode.normalize(branch["source_id"][4], casefold=true) == "sc") # is a series compensation branch
            seriescomp_bus = branch["sc_bus"]
            # println("seriescomp_bus: $seriescomp_bus")
            # Set two interface buses for base network
            subnet_index = 1
            base_f_bus = nothing
            base_t_bus = nothing
            next_f_bus = nothing
            next_t_bus = nothing
            lfac_next_branch = nothing
            if branch["f_bus"] == seriescomp_bus
               sc_f_bus = seriescomp_bus
               sc_t_bus = 2 # series compensation branch goes from seriescomp_bus to the t_bus in the LFAC network (always bus 2)
               base_t_bus = copy(branch["t_bus"])
               for (next_br_idx,next_branch) in data["branch"]
                  if next_branch["index"] != branch["index"] # exclude series compensation branch
                     if next_branch["f_bus"] == seriescomp_bus
                        base_f_bus = copy(next_branch["t_bus"])
                        lfac_next_branch = next_branch
                        next_f_bus = seriescomp_bus
                        next_t_bus = 1 # existing branch goes from seriescomp_bus to the f_bus in the LFAC network (bus 1)
                        println("Series compensation. Including connected branch $(lfac_next_branch["index"]) in LFAC network.")
                        break
                     elseif next_branch["t_bus"] == seriescomp_bus
                        base_f_bus = copy(next_branch["f_bus"])
                        lfac_next_branch = next_branch
                        next_t_bus = seriescomp_bus
                        next_f_bus = 1 # existing branch goes to seriescomp_bus from the f_bus in the LFAC network (bus 1)
                        println("Series compensation. Including connected branch $(lfac_next_branch["index"]) in LFAC network.")
                        break
                     end
                  end
               end
            elseif branch["t_bus"] == seriescomp_bus
               sc_t_bus = seriescomp_bus
               sc_f_bus = 1 # series compensation branch goes to seriescomp_bus from the f_bus in the LFAC network (always bus 1)
               base_f_bus = copy(branch["f_bus"])
               for (next_br_idx,next_branch) in data["branch"]
                  if next_branch["index"] != branch["index"] # exclude series compensation branch
                     if next_branch["f_bus"] == seriescomp_bus
                        base_t_bus = copy(next_branch["t_bus"])
                        lfac_next_branch = next_branch
                        next_f_bus = seriescomp_bus
                        next_t_bus = 2 # existing branch goes from seriescomp_bus to the t_bus in the LFAC network (bus 2)
                        println("Series compensation. Including connected branch $(lfac_next_branch["index"]) in LFAC network.")
                        break
                     elseif next_branch["t_bus"] == seriescomp_bus
                        base_t_bus = copy(next_branch["f_bus"])
                        lfac_next_branch = next_branch
                        next_t_bus = seriescomp_bus
                        next_f_bus = 2 # existing branch goes to seriescomp_bus from the t_bus in the LFAC network (bus 2)
                        println("Series compensation. Including connected branch $(lfac_next_branch["index"]) in LFAC network.")
                        break
                     end
                  end
               end
            else
               throw(ArgumentError("The series compensation bus $seriescomp_bus specified for branch $(branch["index"]) is not connected to the branch. f_bus=$f_bus and t_bus=$t_bus"))
            end

            lfac_buslookup = Dict(
               base_f_bus=>1,
               base_t_bus=>2,
               seriescomp_bus=>seriescomp_bus
            )

            # create output directory
            output_dir = "$output_location/br$(br_idx)_$(lfac_next_branch["index"])"
            if !isdir(output_dir)
               mkpath(output_dir)
            end

            # LFAC "from bus" interface
            iface_index = 1
            bus = base_f_bus
            interface_params["index"] = iface_index
            interface_params["subnet_index"] = subnet_index
            interface_params["bus"] = bus
            push!(iface_df, interface_params)
            f_area = copy(data["bus"]["$bus"]["area"])
            f_zone = copy(data["bus"]["$bus"]["zone"])
            f_kv_base = copy(data["bus"]["$bus"]["base_kv"])

            # LFAC "to bus" interface
            iface_index = 2
            bus = base_t_bus
            interface_params["index"] = iface_index
            interface_params["subnet_index"] = subnet_index
            interface_params["bus"] = bus
            push!(iface_df, interface_params)
            t_area = copy(data["bus"]["$bus"]["area"])
            t_zone = copy(data["bus"]["$bus"]["zone"])
            t_kv_base = copy(data["bus"]["$bus"]["base_kv"])

            for (bus_idx,bus) in data["bus"]
               bus["name"] = split(bus["name"], ";")[1]
            end
            println("series comp branch: $(branch["index"])")
            println("lfac next branch: $(lfac_next_branch["index"])")
            println("saving network at $output_dir/$(base_name)$(fileext)")
            # Disable branches and save network
            # Disabling branches is the only change to the base network.
            branch["br_status"] = 0
            lfac_next_branch["br_status"] = 0
            if fileext == ".m"
               io = open("$output_dir/$(base_name).m", "w");
               export_matpower(io, data)
               close(io)
            else
               stringnet = JSON.json(data)
               open("$output_dir/$(base_name)$(fileext)", "w") do f
                  write(f, stringnet)
               end
            end
            # Restore branch status
            branch["br_status"] = 1
            lfac_next_branch["br_status"] = 1

            # Create and save LFAC subnetwork
            lfac_case = Dict{String,Any}()
            lfac_case["name"] = data["name"]
            lfac_case["source_version"] = data["source_version"]
            lfac_case["baseMVA"] = data["baseMVA"]
            lfac_case["per_unit"] = true
            # Bus data
            lfac_case["bus"] = Dict{String, Any}(
               "1"=>Dict{String, Any}(
                  "bus_type"=>3,
                  "vmin"=>lfac_vmin,
                  "vmax"=>lfac_vmax,
                  "bus_i"=>1,
                  "index"=>1,
                  "va"=>0.0,
                  "vm"=>1.0,
                  "name"=>"Bus 1\tLF",
                  "area"=>f_area,
                  "zone"=>f_zone,
                  "base_kv"=>f_kv_base
               ),
               "2"=>Dict{String, Any}(
                  "bus_type"=>1,
                  "vmin"=>lfac_vmin,
                  "vmax"=>lfac_vmax,
                  "bus_i"=>2,
                  "index"=>2,
                  "va"=>0.0,
                  "vm"=>1.0,
                  "name"=>"Bus 2\tLF",
                  "area"=>t_area,
                  "zone"=>t_zone,
                  "base_kv"=>t_kv_base
               ),
               "$seriescomp_bus"=>Dict{String, Any}(
                  "bus_type"=>1,
                  "vmin"=>lfac_vmin,
                  "vmax"=>lfac_vmax,
                  "bus_i"=>seriescomp_bus,
                  "index"=>seriescomp_bus,
                  "va"=>0.0,
                  "vm"=>1.0,
                  "name"=>"SC Bus\tLF",
                  "area"=>data["bus"]["$seriescomp_bus"]["area"],
                  "zone"=>data["bus"]["$seriescomp_bus"]["zone"],
                  "base_kv"=>data["bus"]["$seriescomp_bus"]["base_kv"]
               )
            )
            # if "apply_ang_lim" in keys(data["bus"]["$seriescomp_bus"])
            #    lfac_case["bus"]["$seriescomp_bus"]["apply_ang_lim"] = data["bus"]["$seriescomp_bus"]["apply_ang_lim"]
            # end
            # Branch Data
            lfac_branch = deepcopy(branch)
            lfac_branch["f_bus"] = sc_f_bus
            lfac_branch["t_bus"] = sc_t_bus

            lfac_next_branch_copy = deepcopy(lfac_next_branch)
            lfac_next_branch_copy["f_bus"] = next_f_bus
            lfac_next_branch_copy["t_bus"] = next_t_bus
            # Change buses for alt angle limits to corresponding LFAC buses
            if "alt_ang_lim" in keys(lfac_branch)
               lfac_branch["alt_ang_lim"] = [lfac_buslookup[bus] for bus in lfac_branch["alt_ang_lim"]]
            end
            if "alt_ang_lim" in keys(lfac_next_branch_copy)
               lfac_next_branch_copy["alt_ang_lim"] = [lfac_buslookup[bus] for bus in lfac_next_branch_copy["alt_ang_lim"]]
            end

            lfac_case["branch"] = Dict{String, Any}(
               "$(lfac_branch["index"])"=>lfac_branch,
               "$(lfac_next_branch_copy["index"])"=>lfac_next_branch_copy
            )
            # Gen data
            lfac_case["gen"] = Dict{String, Any}(
               "1"=>Dict{String, Any}(
                  "gen_status"=>0,
                  "index"=>1,
                  "gen_bus"=>1,
                  "vg"=>1.0
               )
            )
            # Other data
            lfac_case["storage"] = Dict{String, Any}()
            lfac_case["switch"] = Dict{String, Any}()
            lfac_case["shunt"] = Dict{String, Any}()
            lfac_case["load"] = Dict{String, Any}()
            lfac_case["dcline"] = Dict{String, Any}()

            if fileext == ".m"
               io = open("$output_dir/lfac_$(base_name).m", "w");
               export_matpower(io, lfac_case)
               close(io)
            else
               stringnet = JSON.json(lfac_case)
               open("$output_dir/lfac_$(base_name)$(fileext)", "w") do f
                  write(f, stringnet)
               end
            end

         else # is not a series compensation branch

            # Check if branch is connected to a series compensation bus.
            # If so, skip the branch so that series compensated branches are not
            # split up in different subnetworks.
            sc_connected = false
            for (other_br_idx,other_branch) in data["branch"]
               # println("sourceid: $(other_branch["source_id"])")
               if (length(other_branch["source_id"])>3) && (Unicode.normalize(other_branch["source_id"][1], casefold=true) == "branch") && (Unicode.normalize(other_branch["source_id"][4], casefold=true) == "sc") # is a series compensation branch
                  # println("series compensation branch $(other_branch["index"])")
                  # println("other_branch: $other_branch")
                  if "sc_bus" ∉ keys(other_branch)
                     other_branch["sc_bus"] = other_branch["source_id"][2]
                  end
                  seriescomp_bus = other_branch["sc_bus"]
                  if (branch["f_bus"] == seriescomp_bus) || (branch["t_bus"] == seriescomp_bus)
                     sc_connected = true
                     break
                  end
               end
            end
            if sc_connected
               println("skipped branch $(branch["index"]) with series compensation")
               continue
            end
            # create output directory
            output_dir = "$output_location/br$(br_idx)"
            if !isdir(output_dir)
               mkpath(output_dir)
            end
            # Set two interface buses for base network
            subnet_index = 1
            iface_index = 1
            base_f_bus = branch["f_bus"]
            interface_params["index"] = iface_index
            interface_params["subnet_index"] = subnet_index
            interface_params["bus"] = base_f_bus
            push!(iface_df, interface_params)

            f_area = copy(data["bus"]["$base_f_bus"]["area"])
            f_zone = copy(data["bus"]["$base_f_bus"]["zone"])
            f_kv_base = copy(data["bus"]["$base_f_bus"]["base_kv"])
            iface_index = 2
            base_t_bus = branch["t_bus"]
            interface_params["index"] = iface_index
            interface_params["subnet_index"] = subnet_index
            interface_params["bus"] = base_t_bus
            push!(iface_df, interface_params)
            t_area = copy(data["bus"]["$base_t_bus"]["area"])
            t_zone = copy(data["bus"]["$base_t_bus"]["zone"])
            t_kv_base = copy(data["bus"]["$base_t_bus"]["base_kv"])
            for (bus_idx,bus) in data["bus"]
               if "name" in keys(bus)
                  bus["name"] = split(bus["name"], ";")[1]
               end
            end
            # Disable branch and save network
            branch["br_status"] = 0
            if fileext == ".m"
               io = open("$output_dir/$(base_name).m", "w");
               export_matpower(io, data)
               close(io)
            else
               stringnet = JSON.json(data)
               open("$output_dir/$(base_name)$(fileext)", "w") do f
                  write(f, stringnet)
               end
            end
            # Restore branch status
            branch["br_status"] = 1
            # Create and save LFAC subnetwork
            lfac_case = Dict{String,Any}()
            lfac_case["name"] = data["name"]
            lfac_case["source_version"] = data["source_version"]
            lfac_case["baseMVA"] = data["baseMVA"]
            lfac_case["per_unit"] = true
            # Bus data
            lfac_case["bus"] = Dict{String, Any}(
               "1"=>Dict{String, Any}(
                  "bus_type"=>3,
                  "vmin"=>lfac_vmin,
                  "vmax"=>lfac_vmax,
                  "bus_i"=>1,
                  "index"=>1,
                  "va"=>0.0,
                  "vm"=>1.0,
                  "name"=>"Bus 1\tLF",
                  "area"=>f_area,
                  "zone"=>f_zone,
                  "base_kv"=>f_kv_base
               ),
               "2"=>Dict{String, Any}(
                  "bus_type"=>1,
                  "vmin"=>lfac_vmin,
                  "vmax"=>lfac_vmax,
                  "bus_i"=>2,
                  "index"=>2,
                  "va"=>0.0,
                  "vm"=>1.0,
                  "name"=>"Bus 2\tLF",
                  "area"=>t_area,
                  "zone"=>t_zone,
                  "base_kv"=>t_kv_base
               )
            )
            # Branch Data
            lfac_branch = copy(branch)
            lfac_branch["f_bus"] = 1
            lfac_branch["t_bus"] = 2
            lfac_case["branch"] = Dict{String, Any}(
               "$(lfac_branch["index"])"=>lfac_branch
            )
            # Gen data
            lfac_case["gen"] = Dict{String, Any}(
               "1"=>Dict{String, Any}(
                  "gen_status"=>0,
                  "index"=>1,
                  "gen_bus"=>1,
                  "vg"=>1.0
               )
            )
            # Other data
            lfac_case["storage"] = Dict{String, Any}()
            lfac_case["switch"] = Dict{String, Any}()
            lfac_case["shunt"] = Dict{String, Any}()
            lfac_case["load"] = Dict{String, Any}()
            lfac_case["dcline"] = Dict{String, Any}()
            if fileext == ".m"
               io = open("$output_dir/lfac_$(base_name).m", "w");
               export_matpower(io, lfac_case)
               close(io)
            else
               stringnet = JSON.json(lfac_case)
               open("$output_dir/lfac_$(base_name)$(fileext)", "w") do f
                  write(f, stringnet)
               end
            end

            lfac_buslookup = Dict(
               base_f_bus=>1,
               base_t_bus=>2
            )

         end

         br_idx_int = parse(Int64,br_idx)
         if br_idx_int in keys(ctg_br_dict_tmp)
            bus_i = lfac_buslookup[ctg_br_dict_tmp[br_idx_int]["i"]]
            bus_j = lfac_buslookup[ctg_br_dict_tmp[br_idx_int]["j"]]
            lfac_ctg_br_dict[br_idx_int] = pop!(ctg_br_dict_tmp, br_idx_int)
            lfac_ctg_br_dict[br_idx_int]["i"] = bus_i
            lfac_ctg_br_dict[br_idx_int]["j"] = bus_j
            lfac_ctg_br_dict[br_idx_int]["ckt"] = 1
         end

         # Set two interface buses for LFAC line
         subnet_index = 2
         # LFAC "from bus" inerface
         iface_index = 1
         bus = 1
         interface_params["index"] = iface_index
         interface_params["subnet_index"] = subnet_index
         interface_params["bus"] = bus
         push!(iface_df, interface_params)
         # LFAC "to bus" inerface
         iface_index = 2
         bus = 2
         interface_params["index"] = iface_index
         interface_params["subnet_index"] = subnet_index
         interface_params["bus"] = bus
         push!(iface_df, interface_params)
         # Save interfaces file
         interfaces_filename = "interfaces_file" in names(base_subnet_df) ? base_subnet_df.interfaces_file[1] : "interfaces.csv"
         CSV.write("$output_dir/$interfaces_filename", iface_df)
         # Save subnetworks file
         CSV.write("$output_dir/subnetworks.csv", subnet_df)

         # println("output_dir: $output_dir")
         if "contingency_file" in names(subnet_df)
            write_branch_ctg_file(ctg_br_dict_tmp, output_dir*"/"*ctg_filename)
            write_branch_ctg_file(lfac_ctg_br_dict, output_dir*"/"*"lfac_"*ctg_filename)
         end
      end
   end
end

function enumerate_branches(
      base_network::String,
      output_location::String,
      fbase;
      subnet_params=Dict(),
      interface_params=Dict(),
      indices=[],
      output_type="input",
      noloss=false
   )
   println("The function enumerate_branches is deprecated. Please use upgrade_branches instead.")
   upgrade_branches(
      base_network,
      output_location,
      fbase;
      subnet_params,
      interface_params,
      indices,
      output_type,
      noloss
   )
end

function write_branch_ctg_file(ctg_dict, filename)
   open(filename, "w") do io
      for (idx,ctg) in ctg_dict
         output_str = "CONTINGENCY $(ctg["label"])\n$(uppercase(ctg["action"])) $(uppercase(ctg["component"])) FROM BUS $(ctg["i"]) TO BUS $(ctg["j"]) CIRCUIT $(ctg["ckt"])\nEND\n"
         write(io, output_str)
      end
      write(io,"END\n")
   end
end

function change_angle_lims(base_network, output_location, angle_lim_deg, indices=[]; output_type=".json")
   filename = split(base_network, "/")[end]
   prefix = base_network[1:(end-length(filename))]
   println("$filename")

   filetype = split(lowercase(filename), '.')[end]
   base_name = filename[1:(end-length(filetype)-1)]
   (base_name, filetype) = splitext(filename)
   filetype = Unicode.normalize(filetype, casefold=true)
   fileext = filetype == ".m" ? filetype : ".json"
   if lowercase(output_type) == ".json"
      fileext = ".json"
   end

   output_dir = "$output_location/"
   if !isdir(output_dir)
      mkpath(output_dir)
   end
   cp("$prefix/interfaces.csv", "$output_dir/interfaces.csv", force=true)
   cp("$prefix/subnetworks.csv", "$output_dir/subnetworks.csv", force=true)
   if isfile("$prefix/contingencies.con")
      cp("$prefix/contingencies.con", "$output_dir/contingencies.con", force=true)
   end

   data = PowerModels.parse_file(base_network)

   for (i,(br_idx,branch)) in enumerate(data["branch"])
      if ((length(indices) == 0) || (parse(Int64, br_idx) in indices))
         branch["angmin"] = -angle_lim_deg * pi/180.0
         branch["angmax"] = angle_lim_deg * pi/180.0
      end
   end
   if fileext == ".m"
      io = open("$output_dir/$(base_name).m", "w");
      export_matpower(io, data)
      close(io)
   else
      stringnet = JSON.json(data)
      open("$output_dir/$(base_name)$(fileext)", "w") do f
         write(f, stringnet)
      end
   end

   if isfile("$prefix/lfac_$filename")
      data = PowerModels.parse_file("$prefix/lfac_$filename")
      for (i,(br_idx,branch)) in enumerate(data["branch"])
         if ((length(indices) == 0) || (parse(Int64, br_idx) in indices))
            branch["angmin"] = -angle_lim_deg * pi/180.0
            branch["angmax"] = angle_lim_deg * pi/180.0
         end
      end
      if fileext == ".m"
         io = open("$output_dir/$(base_name).m", "w");
         export_matpower(io, data)
         close(io)
      else
         stringnet = JSON.json(data)
         open("$output_dir/$(base_name)$(fileext)", "w") do f
            write(f, stringnet)
         end
      end
   end
end

function change_lims(base_network, output_location, angle_lim_deg, indices=[])
   filename = split(base_network, "/")[end]
   prefix = base_network[1:(end-length(filename))]
   println("$filename")

   output_dir = "$output_location/"
   if !isdir(output_dir)
      mkpath(output_dir)
   end
   cp("$prefix/interfaces.csv", "$output_dir/interfaces.csv", force=true)
   cp("$prefix/subnetworks.csv", "$output_dir/subnetworks.csv", force=true)

   data = PowerModels.parse_file(base_network)

   for (i,(br_idx,branch)) in enumerate(data["branch"])
      if ((length(indices) == 0) || (parse(Int64, br_idx) in indices))
         branch["angmin"] = -angle_lim_deg * pi/180.0
         branch["angmax"] = angle_lim_deg * pi/180.0
      end
   end
   io = open("$output_dir/$filename", "w");
   export_matpower(io, data)
   close(io)

   if isfile("$prefix/lfac_$filename")
      data = PowerModels.parse_file("$prefix/lfac_$filename")
      for (i,(br_idx,branch)) in enumerate(data["branch"])
         if ((length(indices) == 0) || (parse(Int64, br_idx) in indices))
            branch["angmin"] = -angle_lim_deg * pi/180.0
            branch["angmax"] = angle_lim_deg * pi/180.0
         end
      end
      io = open("$output_dir/lfac_$filename", "w");
      export_matpower(io, data)
      close(io)
   end
end

function scale_load(
      scaling,
      areas,
      directory,
      objective,
      x_axis,
      y_axis;
      gen_areas=[],
      area_transfer=[],
      gen_zones=[],
      zone_transfer=[],
      plot_vert_line::Tuple=([],""),
      plot_horiz_line::Tuple=([],""),
      xlimits::Array{Any,1}=[],
      ylimits::Array{Any,1}=[],
      output_plot_label::Tuple{String,String}=("",""),
      scopf::Bool=false,
      contingency::Int64=0
   )

   scale_all_areas = false
   if length(areas) == 0
      scale_all_areas = true
   end
   # get load pd and qd from areas of interest and put them in dict_filt
   sn_data = read_sn_data(directory)
   dict_filt = Dict()
   for (subnet_idx,sn_subnet) in sn_data["sn"]
      println("number of loads: $(length(sn_subnet["load"]))")
      for (load_idx,load) in sn_subnet["load"]
         if (sn_subnet["bus"]["$(load["load_bus"])"]["area"] in areas) || (scale_all_areas)
            pd_keys = ["sn",subnet_idx,"load",load_idx,"pd"]
            pd = load["pd"]
            set_nested!(dict_filt, pd_keys, pd)
            qd_keys = ["sn",subnet_idx,"load",load_idx,"qd"]
            qd = load["qd"]
            set_nested!(dict_filt, qd_keys, qd)
         end
      end
   end

   # get the nested keys values
   (keys,values) = traverse_nested(dict_filt)

   # apply scaling
   # println("keys: $keys")
   # println("values: $values")
   values_scaled = collect(eachrow(values .* scaling'))
   # println("values_scaled: $values_scaled")
   params = (keys,values_scaled)
   println("number of scaled loads: $(length(values_scaled))")
   (results_dict, output_plot) = run_multiple_params(
      directory,
      objective,
      x_axis,
      y_axis,
      params,
      gen_areas,
      area_transfer,
      gen_zones,
      zone_transfer,
      plot_vert_line,
      plot_horiz_line,
      xlimits,
      ylimits,
      output_plot_label,
      scopf,
      contingency
   )
   return (results_dict, output_plot)
end


#!/usr/bin/env julia

function sam(genome_path, bam_paths...; alt_reads=3, alt_frac=0.1,
	min_mapq=10, region="", max_depth=1_000_000)
	lines = []
	genome_path = expanduser(genome_path)
	bam_paths = map(expanduser, bam_paths)

	samples = map(s -> replace(s, r".bam$", ""), bam_paths)
	S = length(samples)
	println("CHROM\tPOSITION\tREF\tALT\t", join(samples, '\t'))

	#spileup = expanduser("~/tools/pypette/compiled/spileup")
	spileup = expanduser("/home/gnanavel/Work/spileup")
	mpileup_options = ""
	if region != ""; mpileup_options = "-r$(region)"; end

	max_alleles = 100
	push!(lines, pipeline(`samtools mpileup -d $(max_depth) -A -x -R -sB -q0 $(mpileup_options) -f $(genome_path) $(bam_paths)`, `$(spileup) $(alt_reads) $(min_mapq)`))

	return lines
end

function format(lines)
	for line in lines
		cols = split(rstrip(line), '\t')
		assert(length(cols) >= 3)
		if cols[3] == "N"; continue; end

		total_reads = zeros(Int32, length(samples))
		allele_reads = zeros(Int32, length(samples), max_alleles)
		alleles = Array{ASCIIString}(0)

		for (s, pileup) in enumerate(cols[4:end])
			if pileup == ""; continue; end
			t = split(pileup, ' ')
			for a in 1:3:length(t)
				count = int(t[a+1])
				total_reads[s] += count
				if t[a] == "."; continue; end
				idx = findfirst(allele -> allele == t[a], alleles)
				if idx == 0 && length(alleles) >= max_alleles; continue; end
				if idx == 0; push!(alleles, t[a]); idx = length(alleles); end
				allele_reads[s, idx] = count
			end
		end

		for a in 1:length(alleles)
			frac = allele_reads[:, a] ./ total_reads
			alt_gt = (allele_reads[:, a] .>= alt_reads) & (frac .>= alt_frac)
			if !any(alt_gt); continue; end

			# Reformat indels in VCF4 format
			ref = cols[3]; alt = alleles[a]
			if length(alt) >= 2
				if alt[2] == '+'    # Insertion
					alt = string(alt[1] == '.' ? ref : alt[1], alt[3:end])
				elseif alt[2] == '-'  # Deletion
					ref = string(ref, alt[3:end])
					alt = alt[1] == '.' ? ref[1] : alt[1]
				end
			end

			print("$(cols[1])\t$(cols[2])\t$(uppercase(ref))\t$(uppercase(alt))")
			for s in 1:length(samples)
				print("\t$(allele_reads[s, a]):$(total_reads[s])")
				if alt_gt[s]; print(":*"); end
			end
			println()
		end
	end
end

function call(genome_path, bam_paths...; alt_reads=3, alt_frac=0.1,
	min_mapq=10, region="", max_depth=1_000_000)

	genome_path = expanduser(genome_path)
	bam_paths = map(expanduser, bam_paths)

	samples = map(s -> replace(s, r".bam$", ""), bam_paths)
	S = length(samples)
	println("CHROM\tPOSITION\tREF\tALT\t", join(samples, '\t'))

	#spileup = expanduser("~/tools/pypette/compiled/spileup")
	spileup = expanduser("/home/gnanavel/Work/spileup")
	mpileup_options = ""
	if region != ""; mpileup_options = "-r$(region)"; end

	max_alleles = 100
	for line in eachline(pipeline(`samtools mpileup -d $(max_depth) -A -x -R -sB -q0 $(mpileup_options) -f $(genome_path) $(bam_paths)`, `$(spileup) $(alt_reads) $(min_mapq)`))
		cols = split(rstrip(line), '\t')
		assert(length(cols) >= 3)
		if cols[3] == "N"; continue; end

		total_reads = zeros(Int32, length(samples))
		allele_reads = zeros(Int32, length(samples), max_alleles)
		alleles = Array{ASCIIString}(0)

		for (s, pileup) in enumerate(cols[4:end])
			if pileup == ""; continue; end
			t = split(pileup, ' ')
			for a in 1:3:length(t)
				count = int(t[a+1])
				total_reads[s] += count
				if t[a] == "."; continue; end
				idx = findfirst(allele -> allele == t[a], alleles)
				if idx == 0 && length(alleles) >= max_alleles; continue; end
				if idx == 0; push!(alleles, t[a]); idx = length(alleles); end
				allele_reads[s, idx] = count
			end
		end

		for a in 1:length(alleles)
			frac = allele_reads[:, a] ./ total_reads
			alt_gt = (allele_reads[:, a] .>= alt_reads) & (frac .>= alt_frac)
			if !any(alt_gt); continue; end

			# Reformat indels in VCF4 format
			ref = cols[3]; alt = alleles[a]
			if length(alt) >= 2
				if alt[2] == '+'    # Insertion
					alt = string(alt[1] == '.' ? ref : alt[1], alt[3:end])
				elseif alt[2] == '-'  # Deletion
					ref = string(ref, alt[3:end])
					alt = alt[1] == '.' ? ref[1] : alt[1]
				end
			end

			print("$(cols[1])\t$(cols[2])\t$(uppercase(ref))\t$(uppercase(alt))")
			for s in 1:length(samples)
				print("\t$(allele_reads[s, a]):$(total_reads[s])")
				if alt_gt[s]; print(":*"); end
			end
			println()
		end
	end
end


using Metalhead
using Metalhead: classify

vgg = VGG19()
img = load("goma.jpeg")

@show classify(vgg, img)


using Test
using MinimalRLCore
using Random

mutable struct TestEnv <: MinimalRLCore.AbstractEnvironment
    name::String
    size::Int
    state::Float64
    rew::Float64
    TestEnv(name, size) = new(name, size, 0.0, 0.0)
end

name(t::TestEnv) = t.name

MinimalRLCore.get_state(env::TestEnv) = env.state
MinimalRLCore.get_reward(env::TestEnv) = env.rew
MinimalRLCore.get_actions(env::TestEnv) = (1, 2)
MinimalRLCore.is_terminal(env::TestEnv) = false

MinimalRLCore.reset!(env::TestEnv, rng::Random.AbstractRNG=Random.GLOBAL_RNG) =
    env.state = rand(rng) * env.size

MinimalRLCore.reset!(env::TestEnv, state::Number) =
    env.state = state

MinimalRLCore.environment_step!(env::TestEnv, action, rng::AbstractRNG=Random.GLOBAL_RNG) =
    env.rew = action*env.state

function test_env()
    Random.seed!(10)
    expected_state = 0.5629122239323647
    @testset "TestEnv Global RNG" begin
        env = TestEnv("Matt", 5)
        @test start!(env) ≈ expected_state
        @test all(step!(env, 1) .≈ (expected_state, 1*expected_state, false))
        @test all(step!(env, 2) .≈ (expected_state, 2*expected_state, false))
        @test all(MinimalRLCore.get_actions(env) .== (1,2))
    end

    rng = MersenneTwister(10)
    @testset "TestEnv Managed RNG" begin
        env = TestEnv("Matt", 5)
        @test start!(env, rng) ≈ expected_state
        @test all(step!(env, 1, rng) .≈ (expected_state, 1*expected_state, false))
        @test all(step!(env, 2, rng) .≈ (expected_state, 2*expected_state, false))
        @test all(MinimalRLCore.get_actions(env) .== (1,2))
    end

    Random.seed!(10)
    expected_state = rand()
    @testset "TestEnv deterministic state restart" begin
        env = TestEnv("Matt", 5)
        @test start!(env, expected_state) ≈ expected_state
        @test all(step!(env, 1, rng) .≈ (expected_state, 1*expected_state, false))
        @test all(step!(env, 2, rng) .≈ (expected_state, 2*expected_state, false))
        @test all(MinimalRLCore.get_actions(env) .== (1,2))
    end
end


mutable struct Packet
    seq :: Int
    src :: UInt16
    dst :: UInt16
    ttl :: UInt8

    time_start :: Int
    time_done :: Int
    done :: Bool
end

Base.@kwdef mutable struct Node <: AbstractAgent
    id::Int

    # (x, y) coordinates
    pos :: Tuple{Int, Int}

    # role = :relay | :sink
    role :: Symbol

    # channel = 37 | 38 | 39
    channel :: UInt8 = 37

    # state = :scanning | :advertising | :sleeping
    state :: Symbol = :scanning

    # the time of the event's (current state) begging
    event_start :: Int = rand(0:50)

    # whether the device is currently sending data (this removes the need for redundant allocations)
    transmitting :: Bool = false

    # the length of the interval between transmissions on different channels (ms)
    t_interpdu :: UInt = 10

    # the length of the interval between scanning on different channels (ms)
    t_scan_interval :: UInt = 20

    # the length of the interval before advertising the received packet
    t_back_off_delay :: UInt = 50

    # the number of extra retransmissions of the received packet
    n_retx_transmit_count :: UInt = 0

    # the length of the delay between bonus transmissions of the received packet (ms)
    t_retx_transmit_delay :: UInt = 20

    # additional random component of delay between retransmissions (min:max ms)
    rt_retx_random_delay :: UnitRange{Int} = 20:50

    # the current number of extra transmissions left
    n_transmit_left :: UInt = 0

    # buffer containing seq of the received packets
    received_packets :: CircularBuffer{Int} = CircularBuffer{Int}(20)

    # transmitting power
    tx_power :: Float64 = 1e-3

    # the reference to the current holding packet
    packet_seq :: Int = 0

    # own ttl for the current packet
    packet_ttl :: UInt8 = 0

    pocket = CircularBuffer{Tuple{Int, Int, UInt8}}(20)
end

Base.@kwdef mutable struct Source <: AbstractAgent
    id :: Int
    pos :: Tuple{Int, Int}

    role :: Symbol = :source
    state :: Symbol = :advertising

    channel :: UInt8 = 37
    event_start :: UInt = 0
    transmitting :: Bool = false

    t_interpdu :: UInt = 1

    # the number of extra transmissions for a packet originating from the source
    n_og_transmit_count :: UInt = 2

    # the length of the delay between additional transmissions of the original packet (ms)
    t_og_transmit_delay :: UInt = 10

    # the range of the additional random component to delay between additional original transmissions
    rt_og_random_delay :: UnitRange{Int} = 0:20

    # the current number of extra transmissions left
    n_transmit_left :: UInt = 0

    # transmitting power
    tx_power :: Float64 = 1e-3

    packet_seq :: Int = 0
    packet_ttl :: UInt8 = 0
end


Base.isequal(model1::CoreModel, model2::CoreModel) =
    isequal(model1.S, model2.S) &&
    isequal(model1.b, model2.b) &&
    isequal(model1.c, model2.c) &&
    isequal(model1.xl, model2.xl) &&
    isequal(model1.xu, model2.xu) &&
    isequal(model1.rxns, model2.rxns) &&
    isequal(model1.mets, model2.mets)

Base.copy(model::CoreModel) =
    CoreModel(model.S, model.b, model.c, model.xl, model.xu, model.rxns, model.mets)

Base.isequal(model1::CoreModelCoupled, model2::CoreModelCoupled) =
    isequal(model1.lm, model2.lm) &&
    isequal(model1.C, model2.C) &&
    isequal(model1.cl, model2.cl) &&
    isequal(model1.cu, model2.cu)

Base.copy(model::CoreModelCoupled) = CoreModelCoupled(model.lm, model.C, model.cl, model.cu)


export mkchans

function mkchans(chans::ChanSpec, S::GphysData; f::Symbol=:x, keepempty::Bool=false, keepirr::Bool=true)
  chan_list = (if chans == Int64[]
    Int64.(collect(1:S.n))
  elseif typeof(chans) <: UnitRange
    Int64.(collect(chans))
  elseif typeof(chans) <: Integer
    [Int64(chans)]
  else
    chans
  end)

  # added 2020-02-12; prevents processing an empty channel
  k = trues(length(chan_list))
  if keepempty == false
    F = getfield(S, f)
    for (j,i) in enumerate(chan_list)
      if isempty(F[i])
        k[j] = false
      end
    end
  end

  # added 2020-03-08: option to auto-delete irregular channels
  if keepirr == false
    for (j,i) in enumerate(chan_list)
      if S.fs[i] == 0.0
        k[j] = false
      end
    end
  end
  return chan_list[k]
end


const GENS =  64 # number of conventional generators
const PEAKERS =  80 # number of conventional generators
const STORG_UNITS = 8 # number of storage units
const N_ZONES = 2
const M_ZONES = 2
const BLOCKS = 7

const INITIAL_DAY = 1
const FINAL_DAY = 60
# Running time for the 1st Week-Ahead UC run.
# INIT_HR_FUCR=0 means the FUCR's optimal outcomes are ready at 00:00
const INIT_HR_FUCR = 6
# Running time for the 2nd WA unit commitment run.
# INIT_HR_SUCR=17 means the SUCR's optimal outcomes are ready at 17:00
const INIT_HR_SUCR = 17
# HRS_FUCR = 168-INIT_HR_FUCR, and INIT_HR_FUCR runs from 0 to 23;
# INIT_HR_FUCR=6 ==> HRS_FUCR =162
const HRS_FUCR = 162
# HRS_SUCR = 168-INIT_HR_SUCR, and INIT_HR_SUCR runs from 17 to 23;
# INIT_HR_SUCR=17 ==> HRS_FUCR =168
const HRS_SUCR = 151

const LOAD_SHED_PENALTY = 3000; # Load-Shedding Penalty
const OVERGEN_PENALTY = 4000; # Over-generation Penalty
const LOAD_SHED_MAX = 100; # Load-Shedding Penalty
const OVERGEN_MAX = 100; # Over-generation Penalty
const VIOLATION_PENALTY = 500;
const VIOLATION_MAX = 10;
#Solver's optimality gap that serves as the optimization termination criteria
const SOLVER_EPGAP =0.005;

const FILE_ACCESS_OVER = "w+";
const FILE_ACCESS_APPEND = "a+";


# classification
building(w::Way) = haskey(w.tags, "building")
highway(w::Way) = haskey(w.tags, "highway")

# predicates
visible{T <: OSMElement}(obj::T) = (get(obj.tags, "visible", "") != "false")
services(w::Way) = (get(w.tags,"highway", "") == "services")
reverse(w::Way) = (get(w.tags,"oneway", "") == "-1")

function haslanes(w::Way)
    v = get(w.tags, "lanes", "")
    length(v)==1 && '1' <= v[1] <= '9'
end

function oneway(w::Way)
    v = get(w.tags,"oneway", "")

    if v == "false" || v == "no" || v == "0"
        return false
    elseif v == "-1" || v == "true" || v == "yes" || v == "1"
        return true
    end

    highway = get(w.tags,"highway", "")
    junction = get(w.tags,"junction", "")

    return (highway == "motorway" ||
            highway == "motorway_link" ||
            junction == "roundabout")
end

# attributes
lanes(w::Way) = parse(Int, w.tags["lanes"])

# feature class
roadway(w::Way) = highway(w) * get(ROAD_CLASSES, w.tags["highway"], 0)

function walkway(way::Way)
    sidewalk = get(way.tags, "sidewalk", "")
    if sidewalk != "no"
        if haskey(PED_CLASSES, "sidewalk:$(sidewalk)")
            return PED_CLASSES["sidewalk:$(sidewalk)"]
        elseif haskey(PED_CLASSES, way.tags["highway"])
            return PED_CLASSES[way.tags["highway"]]
        end
    end
    return 0
end

function cycleway(w::Way)
    bicycle = get(w.tags, "bicycle", "")
    cycleway = get(w.tags, "cycleway", "")
    highway = get(w.tags, "highway", "")

    cycleclass = "cycleway:$(cycleway)"
    bikeclass = "bicycle:$(bicycle)"

    if bicycle != "no"
        if haskey(CYCLE_CLASSES, cycleclass)
            return CYCLE_CLASSES[cycleclass]
        elseif haskey(CYCLE_CLASSES, bikeclass)
            return CYCLE_CLASSES[bikeclass]
        elseif haskey(CYCLE_CLASSES, highway)
            return CYCLE_CLASSES[highway]
        end
    end
    return 0
end


using BenchmarkTools
using BSON
using Benchmarks

# Benchmark parameters

N = 512
n_threads = min.(2 .^ (0:10), Sys.CPU_THREADS) |> unique

# Run and collect benchmarks

print_system_info()

for t in n_threads
    @info "Benchmarking multithreading (N=$N, threads=$t)..."
    julia = Base.julia_cmd()
    run(`$julia -t $t --project benchmark_multithreading_single.jl $N`)
end

suite = BenchmarkGroup(["size", "threads"])
for t in n_threads
    suite[(N, t)] = BSON.load("multithreading_benchmark_$t.bson")[:trial]
end

# Summarize benchmarks

df = benchmarks_dataframe(suite)
sort!(df, :threads)
benchmarks_pretty_table(df, title="Multithreading benchmarks")

suite_Δ = speedups_suite(suite, base_case=(N, 1))
df_Δ = speedups_dataframe(suite_Δ)
sort!(df_Δ, :threads)
benchmarks_pretty_table(df_Δ, title="Multithreading speedup")


using GeometryTypes, GLVisualize, GLAbstraction, Reactive, GLWindow, GLFW, Colors

const description = """
You can drag the edges of this graph,
or remove them with a right click.
Camera is fixed in this example.
"""

if !isdefined(:runtests)
    window = glscreen()
end
window.color = RGBA(0.04f0, 0.02f0, 0.05f0, 1f0)

const record_interactive = true

n = 200
n_connections = 200
w, h = widths(window)
a = map(x->x.*Point2f0(w, h), rand(Point2f0, n))
indices = Signal(rand(1:n, n_connections))
# for the points, we only need unique indices!
unique_indices = map(unique, indices)
scales = map(Vec2f0, rand(5f0:15f0, n))
colors = zeros(RGBA{Float32}, length(a))

# when scale and position is given, its enough to pass the Circle type instead of
# a concrete instance
points = visualize(
    (Circle, a),
    indices=unique_indices,
    scale=scales, offset=map(x-> -x./2f0, scales),
    color=RGBA(1f0,1f0,1f0,1f0),
    stroke_width=3f0, stroke_color=RGBA(1f0,1f0,1f0,0.4f0),
    glow_width=4f0, glow_color=colors
)

const point_robj = points.children[] # temporary way of getting the render object. Shouldn't stay like this
 # best way to get the gpu object. One could also start by creating a gpu array oneself.
 # this is a bit tricky, since not there are three different types.
 # for points and lines you need a GLBuffer. e.g gpu_position = GLBuffer(rand(Point2f0, 50)*1000f0)
const gpu_position = point_robj[:position]
const gpu_scales = point_robj[:scale]
const gpu_colors = point_robj[:glow_color]
const gpu_offset = point_robj[:offset]

# GPUVector adds functionality like push! and splice!
# now the lines and points share the same gpu object
# for linesegments, you can pass indices, which needs to be of some 32bit int type
cmap = map(RGBA{Float32}, colormap("Blues", 50))
lines = visualize(
    gpu_position, :linesegment,
    indices=indices, color=cmap[rand(1:50, n)], # sample random colors from gradient
    thickness=0.5f0
)

# current tuple of renderobject id and index into the gpu array
const m2id = GLWindow.mouse2id(window)
isoverpoint = const_lift(is_same_id, m2id, point_robj)
v0 = (0, scales, colors)
time = bounce(linspace(0f0, 1f0, 20)) # bounces between 0 - 1, for the animation
preserve(foldp(v0, isoverpoint, time) do v0, overpoint, t
    index0, scale0, color0 = v0
    if overpoint
        id, index = value(m2id)
        if id==point_robj.id && length(a) >= index
            new_colors = zeros(RGBA{Float32}, length(a))
            gpu_colors[index] = RGBA{Float32}(0.95, 0.56, t, 0.7)
            newscale = scale0[index] + t*7f0
            gpu_scales[index] = newscale
            gpu_offset[index] = -newscale./2f0
            return index, scale0, color0
        end
    else
        if index0 > 0
            gpu_colors[index0] = color0[index0]
            gpu_scales[index0] = scale0[index0]
            gpu_offset[index0] = -scale0[index0]./2f0
        end
    end
    return v0
end)

# inputs are a dict, materialize gets the keys out of it (equivalent to mouseposition = window.inputs[:mouseposition])
@materialize mouse_buttons_pressed, mouseposition = window.inputs

# single left mousekey pressed (while no other mouse key is pressed)
key_pressed = const_lift(GLAbstraction.singlepressed, mouse_buttons_pressed, GLFW.MOUSE_BUTTON_LEFT)
# righ
right_pressed = const_lift(GLAbstraction.singlepressed, mouse_buttons_pressed, GLFW.MOUSE_BUTTON_RIGHT)
# dragg while key_pressed. Drag only starts if isoverpoint is true
mousedragg = GLAbstraction.dragged(mouseposition, key_pressed, isoverpoint)

# use mousedrag and mouseid + index to actually change the gpu array with the positions
preserve(foldp((value(m2id)..., Point2f0(0)), mousedragg) do v0, dragg
    if dragg == Vec2f0(0) # if drag just started. Not the best way, maybe dragged should return a tuple of (draggvalue, started)
        id, index = value(m2id)
        if id==point_robj.id && length(gpu_position) >= index
            p0 = gpu_position[index]
        else
            p0 = v0[3]
        end
    else
        id, index, p0 = v0
        if id==point_robj.id && length(gpu_position) >= index
            gpu_position[index] = Point2f0(p0) + Point2f0(dragg)
        end
    end
    return id, index, p0
end)
# On right click remove nodes!
s = map(right_pressed) do rp
    id, index = value(m2id)
    if rp && id==point_robj.id && length(gpu_position) >= index
        new_indices = Int[]
        indice_val = value(indices)
        for i=1:2:length(indice_val) #filter out indices
            a, b = indice_val[i], indice_val[i+1]
            if a != index && b != index
                push!(new_indices, a, b)
            end
        end
        push!(indices, new_indices) # update indices!
    end
end
preserve(s)
# _view it!
_view(lines, window, camera=:fixed_pixel)
_view(points, window, camera=:fixed_pixel)

if !isdefined(:runtests)
    renderloop(window)
end


# Bucket all tables with the same column counts together in anticipation of get index
# overloading. This will minimize the number of overloaded functions added to the system.
abstract AbstractDataTable{N, R<:NTuple{N}, C<:NTuple{N, AbstractVector}}

# Tuple unroller: Un-tuple-ize a vector of tuples into a tuple of vectors
@generated function Base.convert{R<:Tuple}(::Type{Tuple{[Vector{r} for r in R.parameters]...}}, vs::AbstractVector{R})
	x = :(t = (); for i = 1:length(vs) end; t;)
	x.args[1].args[2].args = [:(Vector{$(r)}(length(vs))) for r in R.parameters][:]
	x.args[2].args[1].args[2].args = [:(@inbounds t[$(r)][i] = vs[i][$(r)]) for r = 1:length(R.parameters)][:]
	x
end

# Vector unroller: index a tuple of vectors into a tuple
@generated function Base.getindex{N}(D::AbstractDataTable{N}, r::Int64)
	x = :(())
	prepend!(x.args, [:(Base.getindex(getcolumns(D)[$(c)], r)) for c = 1:N])
	x
end

# Unroll value setting
@generated function Base.setindex!{N, R, C}(D::AbstractDataTable{N, R, C}, v::R, r::Int64)
	x = quote v end
	prepend!(x.args, [:(Base.setindex!(getcolumns(D)[$(c)], v[$(c)], r)) for c = 1:N])
	x
end

# Unroll vector emptying
@generate function Base.empty!{N}(D::AbstractDataTable{N})
	x = quote D end
	prepend!(x.args, [:(Base.empty!(getcolumns(D)[$(c)])) for c = 1:N])
	x
end

# All concrete types must implement getcolumns that returns NTuple{N, AbstractVector}
# getcolumns()

# Courtesy method to check that the table has been constructed properly
Base.isvalid{N, R, C}(D::AbstractDataTable{N, R, C}) =
	length(getcolumns(D)) == N &&
	length(R.parameters) == N &&
	length(C.parameters) == N &&
	minimum([length(getcolumns(D)[c]) for c = 1:N]) == maximum([length(getcolumns(D)[c]) for c = 1:N]) &&
	typeof(getcolumns(D)) == C &&
	all([(R.parameters[i] == C.parameters[i].parameters[1])::Bool for i = 1:N]) &&
	isvalid(Val{DataView}, D)

# Iterable methods
Base.start(::AbstractDataTable) = 1
Base.next(D::AbstractDataTable, r::Int64) = (D[r], r + 1)
Base.done(D::AbstractDataTable, r::Int64) = r > length(D)
Base.eltype{N, R, C}(::Type{AbstractDataTable{N, R, C}}) = R
Base.length{N}(D::AbstractDataTable{N}) = minimum([length(getcolumns(D)[c]) for c = 1:N])

# Collection methods
Base.isempty(D::AbstractDataTable) = length(D) < 1
Base.endof(D::AbstractDataTable) = length(D)

# Indexable methods
# A single column returns a vector; type unstable
# A single row and column pair returns an element; type unstable
Base.getindex{N, R, C}(D::AbstractDataTable{N, R, C}, c::Symbol) = getcolumns(D)[getcolumnindex(D, c)]::C.parameters[c]
Base.getindex{N, R, C}(D::AbstractDataTable{N, R, C}, c::AbstractString) = D[Symbol(c)]::C.parameters[c]
Base.getindex{N, R, C}(D::AbstractDataTable{N, R, C}, r::Int64, c::Int64) = getcolumns(D)[c][r]::R.parameters[c]
Base.getindex{N, R, C}(D::AbstractDataTable{N, R, C}, r::Int64, c::Symbol) = D[r, getcolumnindex(D, c)]::R.parameters[c]
Base.getindex{N, R, C}(D::AbstractDataTable{N, R, C}, r::Int64, c::AbstractString) = D[r, Symbol(c)]::R.parameters[c]

# Array methods
Base.ndims(::AbstractDataTable) = 2
Base.size{N}(D::AbstractDataTable{N}) = (length(D), N)
Base.linearindexing(::Type{AbstractDataTable}) = Base.LinearFast()


using TimeSeries

function generate_header_ta_func_with_timearray()
    s_code = generate_header()
    s_code *= "\n"
    s_code *= "\n"
    s_code *= "using TimeSeries"
    s_code
end

function generate_ta_func_with_timearray(d::OrderedDict{Symbol,Any}, symb_func::Symbol)

    d_var_symb = Dict{String,Symbol}(
        "inReal" => :price,
        "Open" => _OPEN,
        "High" => _HIGH,
        "Low" => _LOW,
        "Close" => _CLOSE,
        "Volume" => _VOLUME,
    )

    function replace_var_to_symbol(s)
        get(d_var_symb, s, s)
    end

    func_info = d[symb_func]

    s_doc_RequiredInputArguments = ""
    s_doc_OptionalInputArguments = ""
    s_doc_OutputArguments = ""

    params_lv1_with_values = ""

    params_lv1 = String[]
    params_lv2 = String[]
    params_RequiredInputArguments = String[]
    params_OptionalInputArguments = String[]
    params_OutputArguments = String[]

    lst_args_name = AbstractString[]
    for arg = func_info["RequiredInputArguments"]
        push!(lst_args_name, arg["Name"])
    end
    N = length(lst_args_name)
    arg_typ = "TimeSeries.TimeArray"
    ret_typ = "TimeSeries.TimeArray"
    d_colnames = OrderedDict()
    if lst_args_name==["inReal0", "inReal1"]
        Nta_input = 2
        args = Symbol[:ta, :ta2]
        d_colnames = OrderedDict(
            :ta=>[:price],
            :ta2=>[:price]
        )
    else
        Nta_input = 1
        args = Symbol[:ta]
        for arg = func_info["RequiredInputArguments"]
            colname = string(replace_var_to_symbol(arg["Name"]))
            push!(params_lv1, colname)
            push!(params_RequiredInputArguments, colname)
            #s_doc_RequiredInputArguments *= "\n            - $colname"
        end
        d_colnames[:ta] = params_lv1
    end

    params_lv2_with_types = ""

    index = 1
    for (arg, params) in d_colnames
        s_doc_RequiredInputArguments *= "\n        - $arg::$arg_typ with:"
        if index != 1
            params_lv1_with_values *= ", "
            params_lv2_with_types *= ", "
        end

        for (j, colname) in enumerate(params)
            if j != 1
                params_lv1_with_values *= ", "
            end
            if string(colname) == "price"
                params_lv1_with_values *= "$arg[$colname].values"
            else
                params_lv1_with_values *= "$arg[\"$colname\"].values"
            end
            s_doc_RequiredInputArguments *= "\n            - $colname"
        end

        params_lv2_with_types *=  "$arg::$arg_typ"
        index += 1
    end

    if length(params_lv1_with_values) > 1
        params_lv1_with_values *= ", "
    end

    for arg = func_info["OptionalInputArguments"]
        varname = fix_varname(arg["Name"])
        vartyp = d_typ_to_jl[arg["Type"]]
        def_val = arg["DefaultValue"]
        push!(params_lv1, varname)
        push!(params_OptionalInputArguments, varname)
        s_doc_OptionalInputArguments *= "\n        - $varname=$vartyp($def_val)"
    end

    s_doc_OutputArguments *= "\n        - $ret_typ with:"
    for arg = func_info["OutputArguments"]
        varname = cleanup_prefix(arg["Name"])
        push!(params_OutputArguments, varname)
        s_doc_OutputArguments *= "\n            - $varname"
    end

    #params_lv1 = join(params_lv1, ", ")

    for (index, arg) in enumerate(func_info["RequiredInputArguments"])
        #if index != 1
        #    params_lv2_with_types *= ", "
        #end
        varname = string(replace_var_to_symbol(arg["Name"]))
        vartyp = d_typ_to_jl[arg["Type"]]
    end
    for (index, arg) in enumerate(func_info["OptionalInputArguments"])
        if index != 1
            params_lv2_with_types *= ", "
            params_lv1_with_values *= ", "
        else
            params_lv2_with_types *= "; "
        end
        varname = fix_varname(arg["Name"])
        vartyp = d_typ_to_jl[arg["Type"]]
        def_val = arg["DefaultValue"]
        params_lv2_with_types *= "$varname=$vartyp($def_val)"
        params_lv1_with_values *= "$varname=$varname"
    end

    funcname = string(symb_func)
    first_req_input_array = string(replace_var_to_symbol(func_info["RequiredInputArguments"][1]["Name"]))

    s = "

\"\"\"
    $funcname($params_lv2_with_types, price=:$(_PRICE))

$(func_info["ShortDescription"]) ($(func_info["CamelCaseName"]))

    $(func_info["GroupId"])

    Level: 2 - DataFrame

Arguments:

    RequiredInputArguments:$(s_doc_RequiredInputArguments)

    OptionalInputArguments:$(s_doc_OptionalInputArguments)

Returns:$(s_doc_OutputArguments)

\"\"\"
function $funcname($params_lv2_with_types, price=:$(_PRICE))
    price = string(price)
    result = $funcname($params_lv1_with_values)
    dt = ta.timestamp
    out = TimeArray(dt, result, $params_OutputArguments)
    out
end
"

s

end


module MLJ

## METHOD IMPORT

# from the Standard Library:
import Distributed: @distributed, nworkers, pmap
import Pkg
import Pkg.TOML

# from the MLJ universe:
using MLJBase
import MLJBase.save
using MLJEnsembles
using MLJTuning
using MLJModels
using OpenML
using MLJSerialization
import MLJSerialization.save # but not re-exported
using MLJIteration
import MLJIteration.IterationControl

using Tables, CategoricalArrays
import Distributions
import Distributions: pdf, mode
import Statistics, StatsBase, LinearAlgebra, Random
import Random: AbstractRNG, MersenneTwister
using ProgressMeter
using ComputationalResources
using ComputationalResources: CPUProcesses

# to be extended:
import MLJBase: fit, update, clean!, fit!, predict, fitted_params,
                show_as_constructed, ==
import MLJModels: models
import ScientificTypes

## METHOD EXPORT

export MLJ_VERSION

## METHOD RE-EXPORT

# re-export from Random, Statistics, Distributions, CategoricalArrays:
export pdf, logpdf, mode, median, mean, shuffle!, categorical, shuffle,
    levels, levels!, std, support, sampler

# re-exports from (MLJ)ScientificTypesBase via MLJBase
export Scientific, Found, Unknown, Known, Finite, Infinite,
    OrderedFactor, Multiclass, Count, Continuous, Textual,
    Binary, ColorImage, GrayImage, Image, Table
export scitype, scitype_union, elscitype, nonmissing, trait
export coerce, coerce!, autotype, schema, info

# re-export from MLJBase:
export nrows, color_off, color_on,
    selectrows, selectcols, restrict, corestrict, complement,
    target_scitype, input_scitype, output_scitype, load_path, training_losses,
    predict, predict_mean, predict_median, predict_mode, predict_joint,
    transform, inverse_transform, evaluate, fitted_params, params,
    @constant, @more, HANDLE_GIVEN_ID, UnivariateFinite,
    classes, table, report, rebind!,
    partition, unpack,
    default_measure, measures,
    @load_boston, @load_ames, @load_iris, @load_reduced_ames, @load_crabs,
    load_boston, load_ames, load_iris, load_reduced_ames, load_crabs,
    Machine, machine, AbstractNode, @node,
    source, node, fit!, freeze!, thaw!, Node, sources, origins,
    machines, sources, anonymize!, @from_network, fitresults,
    @pipeline, Stack,
    ResamplingStrategy, Holdout, CV, TimeSeriesCV,
    StratifiedCV, evaluate!, Resampler, iterator,
    default_resource, pretty,
    make_blobs, make_moons, make_circles, make_regression,
    fit_only!, return!, int, decoder

# MLJBase/composition/abstract_types.jl:
for T in vcat(MLJBase.MLJModelInterface.ABSTRACT_MODEL_SUBTYPES,
    MLJBase.COMPOSITE_TYPES,
    MLJBase.SURROGATE_TYPES)
    @eval(export $T)
end
export Surrogate, Composite
export DeterministicNetwork, ProbabilisticNetwork, UnsupervisedNetwork

# MLJBase/measure/measures.jl:
export orientation, reports_each_observation,
    is_feature_dependent, aggregation,
    aggregate, default_measure, value,
    supports_class_weights, prediction_type, human_name

# MLJBase/measures/continuous.jl:
export mav, mae, mape, rms, rmsl, rmslp1, rmsp, l1, l2, log_cosh,
    MAV, MAE, MeanAbsoluteError, mean_absolute_error, mean_absolute_value,
    LPLoss, RootMeanSquaredProportionalError, RMSP,
    RMS, rmse, RootMeanSquaredError, root_mean_squared_error,
    RootMeanSquaredLogError, RMSL, root_mean_squared_log_error, rmsl, rmsle,
    RootMeanSquaredLogProportionalError, rmsl1, RMSLP,
    MAPE, MeanAbsoluteProportionalError, log_cosh_loss, LogCosh, LogCoshLoss

# MLJBase/measures/confusion_matrix.jl:
export confusion_matrix, confmat, ConfusionMatrix

# MLJBase/measures/finite.jl:
export cross_entropy, BrierScore, brier_score,
    BrierLoss, brier_loss,
    LogLoss, log_loss,
    misclassification_rate, mcr, accuracy,
    balanced_accuracy, bacc, bac, BalancedAccuracy,
    matthews_correlation, mcc, MCC, AUC, AreaUnderCurve,
    MisclassificationRate, Accuracy, MCR, BACC, BAC,
    MatthewsCorrelation

# MLJBase/measures/finite.jl -- Multiclass{2} (order independent):
export auc, area_under_curve, roc_curve, roc

# MLJBase/measures/finite.jl -- OrderedFactor{2} (order dependent):
export TruePositive, TrueNegative, FalsePositive, FalseNegative,
    TruePositiveRate, TrueNegativeRate, FalsePositiveRate,
    FalseNegativeRate, FalseDiscoveryRate, Precision, NPV, FScore,
    NegativePredictiveValue,
    # standard synonyms
    TPR, TNR, FPR, FNR, FDR, PPV,
    Recall, Specificity, BACC,
    # instances and their synonyms
    truepositive, truenegative, falsepositive, falsenegative,
    true_positive, true_negative, false_positive, false_negative,
    truepositive_rate, truenegative_rate, falsepositive_rate,
    true_positive_rate, true_negative_rate, false_positive_rate,
    falsenegative_rate, negativepredictive_value,
    false_negative_rate, negative_predictive_value,
    positivepredictive_value, positive_predictive_value,
    tpr, tnr, fpr, fnr,
    falsediscovery_rate, false_discovery_rate, fdr, npv, ppv,
    recall, sensitivity, hit_rate, miss_rate,
    specificity, selectivity, f1score, fallout

# MLJBase/measures/finite.jl -- Finite{N} - multiclass generalizations of
# above OrderedFactor{2} measures (but order independent):
export MulticlassTruePositive, MulticlassTrueNegative, MulticlassFalsePositive,
      MulticlassFalseNegative, MulticlassTruePositiveRate,
      MulticlassTrueNegativeRate, MulticlassFalsePositiveRate,
      MulticlassFalseNegativeRate, MulticlassFalseDiscoveryRate,
      MulticlassPrecision, MulticlassNegativePredictiveValue, MulticlassFScore,
      # standard synonyms
      MTPR, MTNR, MFPR, MFNR, MFDR, MPPV,
      MulticlassRecall, MulticlassSpecificity,
      # instances and their synonyms
      multiclass_truepositive, multiclass_truenegative,
      multiclass_falsepositive,
      multiclass_falsenegative, multiclass_true_positive,
      multiclass_true_negative, multiclass_false_positive,
      multiclass_false_negative, multiclass_truepositive_rate,
      multiclass_truenegative_rate, multiclass_falsepositive_rate,
      multiclass_true_positive_rate, multiclass_true_negative_rate,
      multiclass_false_positive_rate, multiclass_falsenegative_rate,
      multiclass_negativepredictive_value, multiclass_false_negative_rate,
      multiclass_negative_predictive_value, multiclass_positivepredictive_value,
      multiclass_positive_predictive_value, multiclass_tpr, multiclass_tnr,
      multiclass_fpr, multiclass_fnr, multiclass_falsediscovery_rate,
      multiclass_false_discovery_rate, multiclass_fdr, multiclass_npv,
      multiclass_ppv, multiclass_recall, multiclass_sensitivity,
      multiclass_hit_rate, multiclass_miss_rate, multiclass_specificity,
      multiclass_selectivity, macro_f1score, micro_f1score,
      multiclass_f1score, multiclass_fallout, multiclass_precision,
      # averaging modes
      no_avg, macro_avg, micro_avg

# MLJBase/measures/loss_functions_interface.jl
export dwd_margin_loss, exp_loss, l1_hinge_loss, l2_hinge_loss, l2_margin_loss,
    logit_margin_loss, modified_huber_loss, perceptron_loss, sigmoid_loss,
    smoothed_l1_hinge_loss, zero_one_loss, huber_loss, l1_epsilon_ins_loss,
    l2_epsilon_ins_loss, lp_dist_loss, logit_dist_loss, periodic_loss,
    quantile_loss

# MLJBase/measures/loss_functions_interface.jl
export DWDMarginLoss, ExpLoss, L1HingeLoss, L2HingeLoss, L2MarginLoss,
    LogitMarginLoss, ModifiedHuberLoss, PerceptronLoss, SigmoidLoss,
    SmoothedL1HingeLoss, ZeroOneLoss, HuberLoss, L1EpsilonInsLoss,
    L2EpsilonInsLoss, LPDistLoss, LogitDistLoss, PeriodicLoss,
    QuantileLoss

# re-export from MLJEnsembles:
export EnsembleModel

# re-export from MLJTuning:
export Grid, RandomSearch, Explicit, TunedModel, LatinHypercube,
    learning_curve!, learning_curve

# re-export from MLJModels:
export models, localmodels, @load, @iload, load, info,
    ConstantRegressor, ConstantClassifier,     # builtins/Constant.jl
    FeatureSelector, UnivariateStandardizer,   # builtins/Transformers.jl
    Standardizer, UnivariateBoxCoxTransformer,
    OneHotEncoder, ContinuousEncoder, UnivariateDiscretizer,
    FillImputer, matching, BinaryThresholdPredictor

# re-export from MLJIteration:
export MLJIteration
export IteratedModel
for control in MLJIteration.CONTROLS
    eval(:(export $control))
end
export IterationControl

# re-export from MLJOpenML
const MLJOpenML = OpenML

export OpenML, MLJOpenML

# re-export from MLJSerialization:
export Save # control, not method

# re-export from ComputaionalResources:
export CPU1, CPUProcesses, CPUThreads

## CONSTANTS

const srcdir = dirname(@__FILE__)

## INCLUDE FILES

include("version.jl")       # defines MLJ_VERSION constant
include("scitypes.jl")      # extensions to ScientificTypesBase.scitype

end # module


module Atmos

export AtmosModel,
       AtmosAcousticLinearModel, AtmosAcousticGravityLinearModel,
       RemainderModel,
       AtmosLESConfiguration,
       AtmosGCMConfiguration

using LinearAlgebra, StaticArrays
using ..VariableTemplates
using ..MoistThermodynamics
using ..PlanetParameters
import ..MoistThermodynamics: internal_energy
using ..SubgridScaleParameters
using GPUifyLoops
using ..MPIStateArrays: MPIStateArray
using ..Mesh.Grids: VerticalDirection, HorizontalDirection, min_node_distance

import CLIMA.DGmethods: BalanceLaw, vars_aux, vars_state, vars_gradient,
                        vars_diffusive, flux_nondiffusive!,
                        flux_diffusive!, source!, wavespeed, boundary_state!,
                        gradvariables!, diffusive!, init_aux!, init_state!,
                        update_aux!, LocalGeometry, lengthscale,
                        resolutionmetric, DGModel, calculate_dt,
                        nodal_update_aux!, num_state,
                        num_integrals,
                        vars_integrals, vars_reverse_integrals,
                        indefinite_stack_integral!,
                        reverse_indefinite_stack_integral!,
                        integral_load_aux!, integral_set_aux!,
                        reverse_integral_load_aux!,
                        reverse_integral_set_aux!
import ..DGmethods.NumericalFluxes: boundary_state!,
                                    boundary_flux_diffusive!,
                                    NumericalFluxNonDiffusive,
                                    NumericalFluxGradient,
                                    NumericalFluxDiffusive

"""
    AtmosModel <: BalanceLaw

A `BalanceLaw` for atmosphere modeling.

# Usage

    AtmosModel(orientation, ref_state, turbulence, moisture, radiation, source,
               boundarycondition, init_state)

"""
struct AtmosModel{FT,O,RS,T,M,P,R,S,BC,IS} <: BalanceLaw
  orientation::O
  ref_state::RS
  turbulence::T
  moisture::M
  precipitation::P
  radiation::R
  source::S
  # TODO: Probably want to have different bc for state and diffusion...
  boundarycondition::BC
  init_state::IS
end

function calculate_dt(grid, model::AtmosModel, Courant_number)
    T = 290.0
    return Courant_number * min_node_distance(grid, VerticalDirection()) / soundspeed_air(T)
end

abstract type AtmosConfiguration end
struct AtmosLESConfiguration <: AtmosConfiguration end
struct AtmosGCMConfiguration <: AtmosConfiguration end

function AtmosModel{FT}(::Type{AtmosLESConfiguration};
                         orientation::O=FlatOrientation(),
                         ref_state::RS=HydrostaticState(LinearTemperatureProfile(FT(200),
                                                                                FT(280),
                                                                                FT(grav) / FT(cp_d)),
                                                                                FT(0)),
                         turbulence::T=SmagorinskyLilly{FT}(0.21),
                         moisture::M=EquilMoist(),
                         precipitation::P=NoPrecipitation(),
                         radiation::R=NoRadiation(),
                         source::S=( Gravity(),
                                     Coriolis(),
                                     GeostrophicForcing{FT}(7.62e-5, 0, 0)),
                         # TODO: Probably want to have different bc for state and diffusion...
                         boundarycondition::BC=NoFluxBC(),
                         init_state::IS=nothing) where {FT<:AbstractFloat,O,RS,T,M,P,R,S,BC,IS}
  @assert init_state ≠ nothing

  atmos = (
        orientation,
        ref_state,
        turbulence,
        moisture,
        precipitation,
        radiation,
        source,
        boundarycondition,
        init_state,
       )

  return AtmosModel{FT,typeof.(atmos)...}(atmos...)
end
function AtmosModel{FT}(::Type{AtmosGCMConfiguration};
                         orientation::O        = SphericalOrientation(),
                         ref_state::RS         = HydrostaticState(
                                                   LinearTemperatureProfile(
                                                    FT(200),
                                                    FT(280),
                                                    FT(grav) / FT(cp_d)),
                                                  FT(0)),
                         turbulence::T         = SmagorinskyLilly{FT}(0.21),
                         moisture::M           = EquilMoist(),
                         precipitation::P      = NoPrecipitation(),
                         radiation::R          = NoRadiation(),
                         source::S             = (Gravity(), Coriolis()),
                         boundarycondition::BC = NoFluxBC(),
                         init_state::IS=nothing) where {FT<:AbstractFloat,O,RS,T,M,P,R,SU,S,BC,IS}
  @assert init_state ≠ nothing
  atmos = (
        orientation,
        ref_state,
        turbulence,
        moisture,
        precipitation,
        radiation,
        source,
        boundarycondition,
        init_state,
       )

  return AtmosModel{FT,typeof.(atmos)...}(atmos...)
end

function vars_state(m::AtmosModel, FT)
  @vars begin
    ρ::FT
    ρu::SVector{3,FT}
    ρe::FT
    turbulence::vars_state(m.turbulence, FT)
    moisture::vars_state(m.moisture, FT)
    radiation::vars_state(m.radiation, FT)
  end
end
function vars_gradient(m::AtmosModel, FT)
  @vars begin
    u::SVector{3,FT}
    h_tot::FT
    turbulence::vars_gradient(m.turbulence,FT)
    moisture::vars_gradient(m.moisture,FT)
  end
end
function vars_diffusive(m::AtmosModel, FT)
  @vars begin
    ∇h_tot::SVector{3,FT}
    turbulence::vars_diffusive(m.turbulence,FT)
    moisture::vars_diffusive(m.moisture,FT)
  end
end

function vars_aux(m::AtmosModel, FT)
  @vars begin
    ∫dz::vars_integrals(m, FT)
    ∫dnz::vars_reverse_integrals(m, FT)
    coord::SVector{3,FT}
    orientation::vars_aux(m.orientation, FT)
    ref_state::vars_aux(m.ref_state,FT)
    turbulence::vars_aux(m.turbulence,FT)
    moisture::vars_aux(m.moisture,FT)
    radiation::vars_aux(m.radiation,FT)
  end
end
function vars_integrals(m::AtmosModel,FT)
  @vars begin
    radiation::vars_integrals(m.radiation,FT)
  end
end
function vars_reverse_integrals(m::AtmosModel,FT)
  @vars begin
    radiation::vars_reverse_integrals(m.radiation,FT)
  end
end

include("orientation.jl")
include("ref_state.jl")
include("turbulence.jl")
include("moisture.jl")
include("precipitation.jl")
include("radiation.jl")
include("source.jl")
include("boundaryconditions.jl")
include("linear.jl")
include("remainder.jl")

"""
    flux_nondiffusive!(m::AtmosModel, flux::Grad, state::Vars, aux::Vars,
                       t::Real)

Computes flux non-diffusive flux portion of `F` in:

```
∂Y
-- = - ∇ • (F_{adv} + F_{press} + F_{nondiff} + F_{diff}) + S(Y)
∂t
```
Where

 - `F_{adv}`      Advective flux             ; see [`flux_advective!`]@ref()
 - `F_{press}`    Pressure flux              ; see [`flux_pressure!`]@ref()
 - `F_{diff}`     Fluxes that state gradients; see [`flux_diffusive!`]@ref()
"""
@inline function flux_nondiffusive!(m::AtmosModel, flux::Grad, state::Vars,
                                    aux::Vars, t::Real)
  ρ = state.ρ
  ρinv = 1/ρ
  ρu = state.ρu
  u = ρinv * ρu

  # advective terms
  flux.ρ   = ρ * u
  flux.ρu  = ρ * u .* u'
  flux.ρe  = u * state.ρe

  # pressure terms
  p = pressure(m.moisture, m.orientation, state, aux)

  if m.ref_state isa HydrostaticState
    flux.ρu += (p-aux.ref_state.p)*I
  else
    flux.ρu += p*I
  end
  flux.ρe += u*p
  flux_radiation!(m.radiation, m, flux, state, aux, t)
  flux_moisture!(m.moisture, m, flux, state, aux, t)
end

function gradvariables!(atmos::AtmosModel, transform::Vars, state::Vars, aux::Vars, t::Real)
  ρinv = 1/state.ρ
  transform.u = ρinv * state.ρu
  transform.h_tot = total_specific_enthalpy(atmos.moisture, atmos.orientation, state, aux)

  gradvariables!(atmos.moisture, transform, state, aux, t)
  gradvariables!(atmos.turbulence, transform, state, aux, t)
end

function diffusive!(atmos::AtmosModel, diffusive::Vars, ∇transform::Grad, state::Vars, aux::Vars, t::Real)
  diffusive.∇h_tot = ∇transform.h_tot

  # diffusion terms required for SGS turbulence computations
  diffusive!(atmos.turbulence, atmos.orientation, diffusive, ∇transform, state, aux, t)
  # diffusivity of moisture components
  diffusive!(atmos.moisture, diffusive, ∇transform, state, aux, t)
end

@inline function flux_diffusive!(atmos::AtmosModel, flux::Grad, state::Vars,
                                 diffusive::Vars, aux::Vars, t::Real)
  ν, τ = turbulence_tensors(atmos.turbulence, state, diffusive, aux, t)
  D_t = (ν isa Real ? ν : diag(ν)) * inv_Pr_turb
  d_h_tot = -D_t .* diffusive.∇h_tot
  flux_diffusive!(atmos, flux, state, τ, d_h_tot)
  flux_diffusive!(atmos.moisture, flux, state, diffusive, aux, t, D_t)
end

#TODO: Consider whether to not pass ρ and ρu (not state), foc BCs reasons
@inline function flux_diffusive!(atmos::AtmosModel, flux::Grad, state::Vars,
                                 τ, d_h_tot)
  flux.ρu += τ * state.ρ
  flux.ρe += τ * state.ρu
  flux.ρe += d_h_tot * state.ρ
end

@inline function wavespeed(m::AtmosModel, nM, state::Vars, aux::Vars, t::Real)
  ρinv = 1/state.ρ
  u = ρinv * state.ρu
  return abs(dot(nM, u)) + soundspeed(m.moisture, m.orientation, state, aux)
end

function update_aux!(dg::DGModel, m::AtmosModel, Q::MPIStateArray, t::Real)
  FT = eltype(Q)
  auxstate = dg.auxstate

  if num_integrals(m, FT) > 0
    indefinite_stack_integral!(dg, m, Q, auxstate, t)
    reverse_indefinite_stack_integral!(dg, m, Q, auxstate, t)
  end

  nodal_update_aux!(atmos_nodal_update_aux!, dg, m, Q, t)

  return true
end

function atmos_nodal_update_aux!(m::AtmosModel, state::Vars, aux::Vars,
                                 t::Real)
  atmos_nodal_update_aux!(m.moisture, m, state, aux, t)
  atmos_nodal_update_aux!(m.radiation, m, state, aux, t)
  atmos_nodal_update_aux!(m.turbulence, m, state, aux, t)
end

function integral_load_aux!(m::AtmosModel, integ::Vars, state::Vars, aux::Vars)
  integral_load_aux!(m.radiation, integ, state, aux)
end

function integral_set_aux!(m::AtmosModel, aux::Vars, integ::Vars)
  integral_set_aux!(m.radiation, aux, integ)
end

function reverse_integral_load_aux!(m::AtmosModel, integ::Vars, state::Vars, aux::Vars)
  reverse_integral_load_aux!(m.radiation, integ, state, aux)
end

function reverse_integral_set_aux!(m::AtmosModel, aux::Vars, integ::Vars)
  reverse_integral_set_aux!(m.radiation, aux, integ)
end

# TODO: figure out a nice way to handle this
function init_aux!(m::AtmosModel, aux::Vars, geom::LocalGeometry)
  aux.coord = geom.coord
  atmos_init_aux!(m.orientation, m, aux, geom)
  atmos_init_aux!(m.ref_state, m, aux, geom)
  atmos_init_aux!(m.turbulence, m, aux, geom)
end

"""
    source!(m::AtmosModel, source::Vars, state::Vars, diffusive::Vars, aux::Vars, t::Real)
Computes flux `S(Y)` in:
```
∂Y
-- = - ∇ • F + S(Y)
∂t
```
"""
function source!(m::AtmosModel, source::Vars, state::Vars, diffusive::Vars, aux::Vars, t::Real)
  atmos_source!(m.source, m, source, state, diffusive, aux, t)
end

boundary_state!(nf, m::AtmosModel, x...) =
  atmos_boundary_state!(nf, m.boundarycondition, m, x...)

function init_state!(m::AtmosModel, state::Vars, aux::Vars, coords, t, args...)
  m.init_state(m, state, aux, coords, t, args...)
end

boundary_flux_diffusive!(nf::NumericalFluxDiffusive,
                         atmos::AtmosModel,
                         F,
                         state⁺, diff⁺, aux⁺, n⁻,
                         state⁻, diff⁻, aux⁻,
                         bctype, t,
                         state1⁻, diff1⁻, aux1⁻) =
  atmos_boundary_flux_diffusive!(nf, atmos.boundarycondition, atmos,
                                 F,
                                 state⁺, diff⁺, aux⁺, n⁻,
                                 state⁻, diff⁻, aux⁻,
                                 bctype, t,
                                 state1⁻, diff1⁻, aux1⁻)

end # module