verbose = true
 using Dates
 if verbose println(now()) end
 if verbose && !isdefined(Main,:RvSpectML)  println("# Loading RvSpecML")    end
 using RvSpectMLBase
 using EchelleInstruments#, EchelleInstruments.NEID
 using RvSpectML
 if verbose println("# Loading NeidSolarScripts")    end
 using NeidSolarScripts
 println("# Loading other packages 1/2")
 using ArgParse

println("# Parsing arguments...")
   lsf_width = 3.0e3
   function parse_commandline()
     s = ArgParseSettings( description = "Calculate order CCFs from NEID L1 FITS files.")
     #import_settings!(s, s_files_only, args_only=false)
     add_arg_group!(s, "Files", :argg_files)
     @add_arg_table! s begin
         "manifest"
             help = "Manifest file (CVS) containing FITS files to analyze.\nExpects columns named Filename, bjd, target, and continuum_filename."
             arg_type = String
             default = "manifest.csv"
             #required = true
         "output"
             help = "Filename for output CCFs (jld2)"
             arg_type = String
             default = "daily_ccfs.jld2"
         "--param"
             help = "Parameters file"
             arg_type = String
         "--overwrite"
            help = "Specify it's ok to overwrite the output file."
            action = :store_true
      end
      add_arg_group!(s, "CCF parameters", :argg_ccf_param)
      @add_arg_table! s begin
        "--orders_to_use"
           help = "First and last order _index_ to compute CCFs for"
           nargs = 2
           arg_type = Int64
           #default = [ min_order(NEID2D()), max_order(NEID2D()) ]
           #default = [ first(orders_to_use_default(NEID2D())), last(orders_to_use_default(NEID2D())) ]
        "--mask_scale_factor"
            help = "Specify CCF mask width scale as multiple of NEID default v width " * string(default_ccf_mask_v_width(NEID2D()))
            arg_type = Float64
            #default = round(lsf_width/default_ccf_mask_v_width(NEID2D()), sigdigits=3)
        "--line_width_50_default"
            help = "Specify default line full width half maximum"
            arg_type = Float64
            #default = 7.9e3 # m/2
        "--variable_mask_scale"
            help = "Vary width of mask to compensate for solar rotation."
            action = :store_true
        "--ccf_mid_vel"
            help = "Middle of velocity range to use for CCFs."
            arg_type = Float64
            #default = 0.0
        "--range_no_mask_change"
            help = "Avoid lines that would result in a mask change due to BC using line width times this factor (TODO: Verify definition)"
            arg_type = Float64
            #default = 5.0
        "--v_step"
            help = "Specify v step for computing CCF "
            arg_type = Float64
            #default = 155.0 # m/2
      end
      add_arg_group!(s, "Line list parameters", :argg_line_list_param)
      @add_arg_table! s begin
         "--line_list_filename"
             help = "Line list file"
             arg_type = String
             #default = joinpath(pkgdir(NeidSolarScripts),"data","solar_line_list_espresso.csv")
         "--recompute_line_weights"
             help = "Force recalculation of SNR-based line weight factors."
             action = :store_true
      end
      add_arg_group!(s, "Continuum normalization parameters", :argg_continuum_param)
      @add_arg_table! s begin
         "--sed_filename"
             help = "Filename for input SED to normalize by"
             arg_type = String
             default = "/home/eford/Code/RvSpectMLEcoSystem/NeidSolarScripts/data/neidMaster_HR_SmoothLampSED_20210101.fits"
         "--continuum_poly_half_width"
             help = "Half width for window to use for smoothing prior to polynomial fit to continuum."
             arg_type = Int64
             default = 50
         "--quantile_fit_continuum"
             help = "Quantile of rolling window to use prior to polynomial fit to continuum."
             arg_type = Float64
             default = 0.9
         "--order_poly_continuum"
             help = "Order of polynomial to fit after dividing by SED proivded."
             arg_type = Int64
             default = 4
        "--anchors_filename"
             help = "Filename for anchor locations to use in computing continuum to normalize by."
             arg_type = String
             #default = "/home/eford/Code/RvSpectMLEcoSystem/NeidSolarScripts/data/neidMaster_HR_SmoothLampSED_20210101.fits"
         "--smoothing_half_width"
             help = "Half width for window to use for smoothing prior to findind local maximum."
             arg_type = Int64
             default = 6
         "--stretch_factor"
             help = "Stretch factor to use for scaling flux in distance calculation for rolling pin continuum normalization"
             arg_type = Float64
             default = 5.0
         "--merging_threshold"
             help = "Threshold (in λ units) for merging nearby continuum anchors."
             arg_type = Float64
             default = 0.25
         "--fwhm_continuum"
            help = "Full width half-max to use for performing continuum normalization."
            arg_type = Float64
            default = Continuum.fwhm_sol/1000
         "--min_rollingpin_r"
            help = "Minimum rolling pin radius for continuum normalization in uints of FWHM."
            arg_type = Float64
            default = 100.0
         "--nu_continuum"
            help = "Exponent for rolling pin radius in continuum normalization"
            arg_type = Float64
            default = 1.0
         "--apply_continuum_normalization"
            help = "Apply continuum normalization."
            action = :store_true
        "--continuum_normalization_individually"
            help = "Calculate continuum normalization for each file rather than averaged spectra."
            action = :store_true
        "--save_continuum_normalization"
            help = "NOT IMPLEMENTED.  Save continuum normalization to file."
            action = :store_true
        "--recompute_continuum_normalization"
            help = "CURRENTLY ONLY OPTION.  Recompute continuum normalization even if continuum file exists."
            action = :store_true
      end
      add_arg_group!(s, "Filter manifest for ", :argg_filter_param)
      @add_arg_table! s begin
         "--target"
            help = "Target field"
            arg_type = String
            default = "Sun"
         "--datestr"
            help = "Filenames that contain date string."
            arg_type = String
        "--max_solar_hour_angle"
           help = "Maximum absolute value of solar hour angle."
           arg_type = Float64
           default = 6.0
         "--max_airmass"
            help = "Maximum airmass."
            arg_type = Float64
            #default = 10.0
         "--min_snr_factor"
            help = "Minimum SNR relative to max_snr."
            arg_type = Float64
            #default = 0.0
         "--max_snr"
            help = "Specify max_snr manually."
            arg_type = Float64
         "--start_time"
            help = "Specify daily start time for CCFs to be processed (HH MM)"
            nargs = 2
            arg_type = Int64
            default = [0, 0]
            #default = [18, 30]
        "--stop_time"
           help = "Specify daily stop time for CCFs to be processed (HH MM)"
           nargs = 2
           arg_type = Int64
           #default = [ min_order(NEID2D()), max_order(NEID2D()) ]
           #default = [22, 30]
           default = [23, 59]
         "--max_num_spectra"
            help = "Maximum number of spectra to process."
            arg_type = Int
            default = 250   # TODO: Increase  after finish testing
      end

     return parse_args(s)
 end
 args = parse_commandline()

if verbose   println("# Loading other packages 2/2")    end
 using CSV, DataFrames, Query, Dates
 using JLD2, FileIO, SHA
 using StatsBase, Statistics, NaNMath

 # Filename arguments
 @assert isfile(args["manifest"]) || islink(args["manifest"])
 manifest_filename = args["manifest"]
 #args["overwrite"] = true   # TODO: Comment after done testing
 if isfile(args["output"]) && !args["overwrite"]
    error("# Output file " * args["output"] * " already exists (size = " * string(filesize(args["output"])) * " ).")
 end
 @assert !isfile(args["output"]) || args["overwrite"] == true
 @assert match(r"\.jld2$",args["output"]) != nothing
 daily_ccf_filename = args["output"]
 touch(daily_ccf_filename)

 file_hashes = Dict{String,String}()
 start_processing_time = now()

 # Include parameters file if specified
 if args["param"] != nothing
     @warn "Haven't implemented parameters file for this script yet."
    if isfile(args["param"]) || islink(args["param"])
        println("# Including parameter code from ", args["param"], ".")
        include(args["param"])
        file_hashes["param.in"] = bytes2hex(args["param"])
    end
 end
 # Overide values from param file if on commandline.  If missing, set defaults
 if args["line_list_filename"] != nothing
    line_list_filename = args["line_list_filename"]
 elseif !@isdefined line_list_filename
     #line_list_filename = joinpath(pkgdir(NeidSolarScripts),"data","solar_line_list_espresso.csv")
     line_list_filename = joinpath(pkgdir(NeidSolarScripts),"data","espresso+neid_mask_97_to_108.mas")
 end
 @assert isfile(line_list_filename) || islink(line_list_filename)
 if args["sed_filename"] != nothing
    sed_filename = args["sed_filename"]
 #elseif !@isdefined sed_filename
 #     sed_filename = joinpath("/home/eford/Code/RvSpectMLEcoSystem/NeidSolarScripts/data","neidMaster_HR_SmoothLampSED_20210101.fits")
 end
 @assert !@isdefined(sed_filename) || isfile(sed_filename) || islink(sed_filename)
 @assert !@isdefined(anchors_filename) || isfile(anchors_filename) || islink(anchors_filename)

 # Extract arguments with non-standard types
 if args["orders_to_use"] != nothing && length(args["orders_to_use"]) == 2
     orders_to_use = first(args["orders_to_use"]):last(args["orders_to_use"])
 elseif !@isdefined orders_to_use
     orders_to_use = first(orders_to_use_default(NEID2D())):last(orders_to_use_default(NEID2D()))
 end
 @assert min_order(NEID2D()) <= first(orders_to_use) <= max_order(NEID2D())
 @assert min_order(NEID2D()) <= last(orders_to_use) <= max_order(NEID2D())

 if args["ccf_mid_vel"] != nothing
     ccf_mid_velocity = args["ccf_mid_vel"]
 elseif !@isdefined ccf_mid_velocity
     ccf_mid_velocity = 0.0
 end
 @assert -5e5 < ccf_mid_velocity < 5e5  # arbitrary ball park restrictions to catch obvious typeos

 if args["v_step"] != nothing
     v_step = args["v_step"]
 elseif !@isdefined v_step
     v_step = 150.0
 end
 @assert 50 <= v_step <= 2000 # m/s

 if args["mask_scale_factor"] != nothing
     mask_scale_factor = args["mask_scale_factor"]
 elseif !@isdefined mask_scale_factor
     mask_scale_factor = round(lsf_width/default_ccf_mask_v_width(NEID2D()), sigdigits=3)
 end
 @assert 1 <= mask_scale_factor <= 64 # arbitrary big number to catch obvious typeos

 if args["line_width_50_default"] != nothing
     line_width_50_default = args["line_width_50_default"]
 elseif !@isdefined line_width_50_default
     line_width_50_default = 7.9e3 # m/s
 end
 @assert 2e3 <= line_width_50_default <= 50e3   # arbitrary big number to catch obvious typeos

 if args["range_no_mask_change"] != nothing
     range_no_mask_change = args["range_no_mask_change"]
 elseif !@isdefined range_no_mask_change
     range_no_mask_change = 5.0 # m/s
 end
 @assert 0 <= range_no_mask_change <= 10
 start_time = args["start_time"] != nothing && length(args["start_time"]) == 2 ? Time(args["start_time"][1], args["start_time"][2]) : Time(0,0)
 stop_time =  args["stop_time"] != nothing && length(args["stop_time"]) == 2 ? Time(args["stop_time"][1],  args["stop_time"][2]) : Time(12,59,59)

 @assert 3 <= args["continuum_poly_half_width"] <= 200 # arbitrary limits for now
 @assert 0.5 <= args["quantile_fit_continuum"] <= 1-1/(2*args["smoothing_half_width"])
 @assert 0 <= args["order_poly_continuum"] <= 6 # arbitrary limits for now
 @assert 3 <= args["smoothing_half_width"] <= 200 # arbitrary limits for now
 @assert 1.0 <= args["stretch_factor"] <= 100    # arbitrary limits for now
 @assert 0.1 <= args["merging_threshold"] <= 2.0 # arbitrary limits for now
 @assert 1.0 <= args["fwhm_continuum"] <= 30.0   # km/s  arbitrary limits for now
 @assert 10 <= args["min_rollingpin_r"] <= 1000 # arbitrary limits for now
 @assert 0.7 <= args["nu_continuum"] <= 1.3  # Recommendations from Cretignier et al.
 #args["apply_continuum_normalization"] = true   # TODO Remove after done testing
 #args["continuum_normalization_individually"] = false #  TODO Remove after done testing
 #args["recompute_line_weights"] = true  # TODO Remove after done testing

if verbose println("# Reading manifest of files to process.")  end
  df_files  = CSV.read(manifest_filename, DataFrame)
    @assert size(df_files,1) >= 1
    @assert hasproperty(df_files,:Filename)
    @assert hasproperty(df_files,:target)
    @assert hasproperty(df_files,:bjd)
    @assert hasproperty(df_files,:ssbz)
    @assert hasproperty(df_files,:exptime)
    @assert hasproperty(df_files,:airmass)
    if args["target"] == "Sun" || args["target"] == "Solar"
      @assert hasproperty(df_files,:alt_sun)  # TODO: Compute if not avaliable?
      @assert hasproperty(df_files,:Δfwhm²)   # TODO: Compute if not avaliable?
      df_files[!,"solar_hour_angle"] = NeidSolarScripts.SolarRotation.get_solar_hour_angle(df_files.bjd,obs=:WIYN)
      if eltype(df_files[!,:order_snrs]) == String   # TODO: Compute if not avaliable?
    end
    @assert hasproperty(df_files,:order_snrs)
        df_files[!,:order_snrs] = map(i->parse.(Float64,split(df_files[i,:order_snrs][2:end-1],',')),1:size(df_files,1))
    end
    @assert eltype(df_files[!,:order_snrs]) == Vector{Float64}

  if  args["max_snr"] != nothing
      max_snr = args["max_snr"]
  elseif !@isdefined max_snr
      df_files_tmp = df_files |>
        @filter( args["target"] == nothing || _.target == args["target"] ) |> DataFrame
      max_snr = maximum(NaNMath.sum.(df_files_tmp.order_snrs))
  end
  @assert 0 < max_snr < Inf

  @assert args["max_airmass"] == nothing || 1 < args["max_airmass"] <= 10  # not tested beyond 10
  @assert args["min_snr_factor"] == nothing || 0 <= args["min_snr_factor"] < 1
  @assert args["max_solar_hour_angle"] == nothing || 0 < args["max_solar_hour_angle"] <= 6

  df_files_use = df_files |>
    @filter( args["target"] == nothing || _.target == args["target"] ) |>
    @filter( args["datestr"] == nothing || occursin(args["datestr"],_.target) ) |>
    @filter( args["max_airmass"] == nothing || _.airmass <= args["max_airmass"] ) |>
    @filter( args["max_solar_hour_angle"] == nothing || abs(_.solar_hour_angle) <= args["max_solar_hour_angle"] ) |>
    @filter( args["start_time"] == nothing || Time(julian2datetime(_.bjd)) >= start_time ) |>
    @filter( args["stop_time"] == nothing || Time(julian2datetime(_.bjd)) <= stop_time ) |> # TODO for other instruments may need to deal wtih cross end of 24 UTC
    @filter( args["min_snr_factor"] == nothing || sum(_.order_snrs) >= args["min_snr_factor"] * max_snr ) |>
    @take(args["max_num_spectra"] ) |> @orderby(_.bjd) |>
    DataFrame
  println("# Found ", size(df_files_use,1), " files of ",  size(df_files,1), " to process.")
  @assert size(df_files_use,1) >= 1

df_files_cleanest = df_files_use |>
    @filter( _.airmass <= 2.0 ) |>
    @filter( abs(_.solar_hour_angle) <= 1 ) |>
    @filter( sum(_.order_snrs) >= 0.9 * max_snr ) |>
    @take(args["max_num_spectra"] ) |>
    DataFrame
  println("# Found ", size(df_files_cleanest,1), " files considered clean.")
  #@assert size(df_files_cleanest,1) >= 1

if verbose println(now()) end

if @isdefined(sed_filename)
   sed = Continuum.read_master_sed_neid(filename=sed_filename)
   min_order_for_continuum = min_order(NEID2D()) # 12
   max_order_for_continuum = max_order(NEID2D())
   orders_to_use_for_continuum = min_order_for_continuum:max_order_for_continuum
end

pipeline_plan = PipelinePlan()
 if verbose println("# Reading data files.")  end
 # For v1.0.0
 #all_spectra = Spectra2DBasic{Float64, Float32, Float32, Matrix{Float64}, Matrix{Float32}, Matrix{Float32}, NEID2D}[]
 # For v1.1.*
 all_spectra = Spectra2DBasic{Float64, Float64, Float32, Matrix{Float64}, Matrix{Float64}, Matrix{Float32}, NEID2D}[]
 spec = NEID.read_data(first(eachrow(df_files_use)).Filename)
 mean_clean_flux = zeros(Float64,size(spec.flux));
 mean_clean_var = zeros(Float64,size(spec.flux));
 #mean_clean_weight_sum = zeros(Float64,size(spec.flux));
 mean_clean_flux_sed_normalized = zeros(Float64,size(spec.flux));
 mean_clean_var_sed_normalized = zeros(Float64,size(spec.flux));
 #mean_clean_flux_sed_normalized_weight_sum = zeros(Float64,size(spec.flux));
 mean_clean_flux_continuum_normalized = zeros(Float64,size(spec.flux));
 mean_clean_var_continuum_normalized = zeros(Float64,size(spec.flux));
 #mean_clean_flux_continuum_normalized_weight_sum = zeros(Float64,size(spec.flux));
 normalization_anchors_list = [];
 for (i,row) in enumerate(eachrow(df_files_use))
    if !(isfile(row.Filename)||islink(row.Filename))
        println("# Couldn't find input FITS file: ", row.Filename)
        continue
    end
    #= if !(isfile(row.continuum_filename)||islink(row.continuum_filename))
        println("# Couldn't find matching continuum file: ", row.continuum_filename)
        continue
    end =#
    if verbose
        println("# Reading ", row.Filename, "(",i,"/",size(df_files_use,1),")")
        flush(stdout)
    end
    local spec = NEID.read_data(row)
    file_hashes[row.Filename] = bytes2hex(sha256(row.Filename))
    weights = 1
    if row.Filename ∈ df_files_cleanest.Filename
            mean_clean_flux .+= spec.flux # .*weights
            mean_clean_var .+= spec.var # .*weights
    end

    #=
    m = match(r"(neidL1_\d+[T_]\d+)\.fits$", row.Filename)
    continuum_filename = joinpath(neid_data_path,target_subdir,output_dir,"continuum", m[1] * "_continuum=new.jld2")
    if !(isfile(continuum_filename)||islink(continuum_filename))
        println("# Couldn't find matching continuum file", continuum_filename)
        continue
    end
    continuum = load(continuum_filename, "continuum")
    =#

    if @isdefined sed
      (f_norm, var_norm) = Continuum.normalize_by_sed(spec.λ,spec.flux,spec.var, sed; poly_order = args["order_poly_continuum"], half_width = args["continuum_poly_half_width"], quantile = args["quantile_fit_continuum"], orders_to_use=orders_to_use_for_continuum)
      if row.Filename ∈ df_files_cleanest.Filename
            mean_clean_flux_sed_normalized .+= f_norm # .*weights
            mean_clean_var_sed_normalized .+= var_norm # .*weights
      end
    else
        f_norm = spec.flux
        var_norm = spec.var
    end

    if args["apply_continuum_normalization"]==true && args["continuum_normalization_individually"]==true
        local anchors, continuum, f_filtered
        if args["anchors_filename"] != nothing
            @assert isfile(args["anchors_filename"]) && filesize(args["anchors_filename"])>0
            anchors = load(args["anchors_filename"],"anchors")
            (anchors, continuum, f_filtered) = Continuum.calc_continuum(spec.λ, f_norm, var_norm, anchors;
                orders_to_use = orders_to_use_for_continuum, verbose = true )
            push!(normalization_anchors_list, anchors)
        else
            (anchors, continuum, f_filtered) = Continuum.calc_continuum(spec.λ, f_norm, var_norm; fwhm = args["fwhm_continuum"]*1000, ν = args["nu_continuum"],
                stretch_factor = args["stretch_factor"], merging_threshold = args["merging_threshold"], smoothing_half_width = args["smoothing_half_width"], min_R_factor = args["min_rollingpin_r"],
                orders_to_use = orders_to_use_for_continuum, verbose = false )
            push!(normalization_anchors_list, anchors)
        end
            #=
            continuum = load(row.continuum_filename, "continuum")
            file_hashes[row.continuum_filename] = bytes2hex(sha256(row.continuum_filename))
            =#
        f_norm ./= continuum
        var_norm ./= continuum.^2
        if row.Filename ∈ df_files_cleanest.Filename
                mean_clean_flux_continuum_normalized .+= f_norm # .*weights
                mean_clean_var_continuum_normalized .+= var_norm # .*weights
#                mean_clean_var_continuum_normalized_weight_sum .+= weights
            end
        spec.flux .= f_norm
        spec.var .= var_norm

    end
    push!(all_spectra,spec)
 end
 GC.gc()
 dont_need_to!(pipeline_plan,:read_spectra);

 #mean_clean_flux ./= mean_clean_flux_weight_sum
 #mean_clean_flux_sed_normalized ./= mean_clean_flux_continuum_normalized_weight_sum
 #mean_clean_flux_continuum_normalized ./= mean_clean_flux_continuum_normalized_weight_sum

 if args["apply_continuum_normalization"]==true && !(args["continuum_normalization_individually"] == true)
     println("# Computing continuum normalization from mean spectra.")
     local anchors, continuum, f_filtered
     if args["anchors_filename"] !=nothing
         @assert isfile(args["anchors_filename"]) && filesize(args["anchors_filename"])>0
         anchors = load(args["anchors_filename"],"anchors")
     else
         if @isdefined sed
             (anchors, continuum, f_filtered) = Continuum.calc_continuum(spec.λ, mean_clean_flux_sed_normalized, mean_clean_var_sed_normalized; fwhm = args["fwhm_continuum"]*1000, ν = args["nu_continuum"],
                stretch_factor = args["stretch_factor"], merging_threshold = args["merging_threshold"], smoothing_half_width = args["smoothing_half_width"], min_R_factor = args["min_rollingpin_r"],
                orders_to_use = orders_to_use_for_continuum, verbose = false )
        else
            (anchors, continuum, f_filtered) = Continuum.calc_continuum(spec.λ, mean_clean_flux, mean_clean_var; fwhm = args["fwhm_continuum"]*1000, ν = args["nu_continuum"],
                stretch_factor = args["stretch_factor"], merging_threshold = args["merging_threshold"], smoothing_half_width = args["smoothing_half_width"], min_R_factor = args["min_rollingpin_r"],
                orders_to_use = orders_to_use_for_continuum, verbose = false )
        end
    end
    normalization_anchors_list = anchors

    for (i,row) in enumerate(eachrow(df_files_use))
        (anchors, continuum, f_filtered) = Continuum.calc_continuum(all_spectra[i].λ, all_spectra[i].flux, all_spectra[i].var,
                    anchors, smoothing_half_width = args["smoothing_half_width"], orders_to_use=orders_to_use_for_continuum)
        all_spectra[i].flux ./= continuum
        all_spectra[i].var ./= continuum.^2
        if row.Filename ∈ df_files_cleanest.Filename
            mean_clean_flux_continuum_normalized .+= all_spectra[i].flux # .*weights
            mean_clean_var_continuum_normalized .+= all_spectra[i].var # .*weights
            #                mean_clean_var_continuum_normalized_weight_sum .+= weights
        end
    end
 end

 order_list_timeseries = extract_orders(all_spectra, pipeline_plan,  orders_to_use=orders_to_use, remove_bad_chunks=false, recalc=true )

line_width = line_width_50_default
 max_mask_scale_factor = max(mask_scale_factor, max(lsf_width,
                             line_width/sqrt(8*log(2)))/default_ccf_mask_v_width(NEID2D())) # 4.0
 max_bc = RvSpectMLBase.max_bc
 #max_bc = RvSpectMLBase.max_bc_earth_rotation
 #max_orders = min_order(NEID2D()):max_order(NEID2D())
 #good_orders = orders_to_use_default(NEID2D())
 #orders_to_use = max_orders
 if isfile(line_list_filename) && (filesize(line_list_filename)>0) && !args["recompute_line_weights"]
   println("# Reading ", line_list_filename)
   line_list_espresso = CSV.read(line_list_filename, DataFrame)
   @assert all(map(k->k ∈ names(line_list_espresso), ["lambda","weight","order"]))
   dont_need_to!(pipeline_plan,:clean_line_list_tellurics)
 else
    println("# Can't find ", line_list_filename, ".  Trying ESPRESSO line list.")
    #orders_to_use = good_orders
    #order_list_timeseries = extract_orders(all_spectra,pipeline_plan, orders_to_use=orders_to_use, recalc=true )
    touch(line_list_filename)
    #line_list_filename = "G2.espresso.mas"
    linelist_for_ccf_fn_w_path = joinpath(pkgdir(EchelleCCFs),"data","masks",line_list_filename)
    line_list_espresso = prepare_line_list(linelist_for_ccf_fn_w_path, all_spectra, pipeline_plan, v_center_to_avoid_tellurics=ccf_mid_velocity,
       Δv_to_avoid_tellurics = 2*max_bc+range_no_mask_change*line_width_50_default+max_mask_scale_factor*default_ccf_mask_v_width(NEID2D()), orders_to_use=#=orders_to_use=#56:108, recalc=true, verbose=true)
    #CSV.write(line_list_filename, line_list_espresso)
 end
 file_hashes[line_list_filename] = bytes2hex(sha256(line_list_filename))
 #outputs["line_list_espresso"] = line_list_espresso

if args["variable_mask_scale"]
     maxΔfwhm² = -0.569375
     @assert maximum(df_files_use.Δfwhm²) < maxΔfwhm²
     Δfwhm = 1000.0 .* sqrt.(maxΔfwhm².-df_files_use.Δfwhm²[1:length(all_spectra)])  # How much to increase fwhm by to acheive uniform fwhm
else
    Δfwhm = zeros(0)
end

if verbose println(now()) end
line_width_50 = line_width_50_default
  #order_list_timeseries = extract_orders(all_spectra,pipeline_plan,  orders_to_use=orders_to_use,  remove_bad_chunks=false, recalc=true )
  @time (order_ccfs, order_ccf_vars, v_grid_order_ccfs) = ccf_orders(order_list_timeseries, line_list_espresso, pipeline_plan,
    mask_type=:gaussian, Δfwhm=Δfwhm,
    mask_scale_factor=mask_scale_factor, range_no_mask_change=5*line_width_50, ccf_mid_velocity=ccf_mid_velocity, v_step=v_step,
    v_max=max(range_no_mask_change*line_width_50,2*max_bc), orders_to_use=orders_to_use, allow_nans=true, calc_ccf_var=true,
    recalc=true)

#orders_to_use2 = orders_to_use[map(i->!iszero(order_ccfs[:,i,:]),1:length(orders_to_use))]
#order_list_timeseries2 = extract_orders(all_spectra,pipeline_plan,  orders_to_use=orders_to_use2,  remove_bad_chunks=false, recalc=true )

#=
ccf_dir = joinpath(neid_data_path,target_subdir,"ccfs")
if !isdir(ccf_dir)
   mkdir(ccf_dir)
end
=#
if verbose println(now()) end
#daily_ccf_filename = joinpath(neid_data_path,target_subdir,"output","ccfs", date_str * "_ccfs=default.jld2")
#daily_ccf_filename = joinpath(neid_data_path,target_subdir,"ccfs", date_str * "_ccfs=default.jld2")
println("# Saving results to ", daily_ccf_filename, ".")
  stop_processing_time = now()
  jldopen(daily_ccf_filename, "w") do f
    f["v_grid"] = v_grid_order_ccfs
    f["order_ccfs"] = order_ccfs
    f["order_ccf_vars"] = order_ccf_vars
    f["orders_to_use"] = orders_to_use
    f["manifest"] = df_files_use
    f["calc_order_ccf_args"] = args
    f["ccf_line_list"] = line_list_espresso
    if size(df_files_cleanest,1) >= 1
      f["mean_clean_flux"] = mean_clean_flux
      f["mean_clean_var"] = mean_clean_var
      if size(mean_clean_flux_sed_normalized,1) > 0
          f["mean_clean_flux_sed_normalized"] = mean_clean_flux_sed_normalized
          f["mean_clean_var_sed_normalized"] = mean_clean_var_sed_normalized
      end
      if size(mean_clean_flux_continuum_normalized,1) > 0
          f["mean_clean_flux_continuum_normalized"] = mean_clean_flux_continuum_normalized
          f["mean_clean_var_continuum_normalized"] = mean_clean_var_continuum_normalized
      end
    end
    if args["apply_continuum_normalization"]
        f["normalization_anchors"] = normalization_anchors_list
    end
    f["start_processing_time"] = start_processing_time
    f["stop_processing_time"] = stop_processing_time
    f["daily_ccf_filename"] = daily_ccf_filename
    f["file_hashes"] = file_hashes
  end
if verbose println(now()) end

#=
for (i,row) in enumerate(eachrow(df_files_use))
    m = match(r"(neidL1_\d+[T_]\d+)\.fits$", row.Filename)
    #ccf_filename = joinpath(neid_data_path,target_subdir,"output","ccfs", m[1] * "_ccfs=default.jld2")
    ccf_filename = joinpath(neid_data_path,target_subdir,"ccfs", m[1] * "_ccfs=default.jld2")
    jldopen(ccf_filename, "w") do f
        f["v_grid"] = v_grid_order_ccfs
        f["order_ccfs"] = order_ccfs[:,:,i]
        f["order_ccf_vars"] = order_ccf_vars[:,:,i]
        f["orders_to_use"] = orders_to_use
        for k in keys(row)
            f[string(k)] = row[k]
        end
        f["ccf_filename"] = ccf_filename
    end
    if i==size(order_ccfs,3) break end
end
=#

#=

#good_orders = orders_to_use_default(NEID2D())
 #good_orders = 56:111 # orders_to_use_default(NEID2D())
 #good_orders = 57:96 # orders_to_use_default(NEID2D())
 #good_orders = 55 .+ vcat(1:14, 17:24, 27:29, 37:38,40:41)
 #good_orders = 55 .+ collect(1:14)
 #good_orders = 55 .+ collect(17:24)
 #good_orders = 55 .+ vcat(collect(17:24), collect(27:29), collect(37:38),collect(40:41))
 #good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41))
 #good_orders = [55, 56, 57]
# order_list_timeseries = extract_orders(all_spectra,pipeline_plan,  orders_to_use=good_orders, remove_bad_chunks=false, recalc=true )

outputs["times"] = order_list_timeseries.times
 alt_sun = map(i->calc_solar_alt(all_spectra[i].metadata[:bjd]),1:length(all_spectra))
 obs_idx_max_alt_sun = argmax(alt_sun)
 time_max_alt_sun = order_list_timeseries.times[obs_idx_max_alt_sun]
 outputs["time_max_alt_sun"] = time_max_alt_sun

 order_snrs = [RvSpectMLBase.calc_snr(order_list_timeseries[obsid].data[c]) for c in 1:length(order_list_timeseries[1].data),
                    obsid in 1:length(order_list_timeseries) ]
 order_sum_flux = [NaNMath.sum(order_list_timeseries[obsid].data[c].flux) for c in 1:length(order_list_timeseries[1].data),
                                        obsid in 1:length(order_list_timeseries) ]
 order_mean_flux = [NaNMath.mean(order_list_timeseries[obsid].data[c].flux) for c in 1:length(order_list_timeseries[1].data),
                                                                        obsid in 1:length(order_list_timeseries) ]

 mean_order_mean_flux = vec(mean(order_mean_flux,dims=2))

 order_weights =  (order_snrs[:,obs_idx_max_alt_sun]./order_snrs).^2
 #order_weights = (order_sum_flux[:,obs_idx_max_alt_sun]./order_sum_flux)
 outputs["order_snrs"] = order_snrs
 outputs["order_sum_flux"] = order_sum_flux
 outputs["order_mean_flux"] = order_mean_flux
 outputs["order_weights"] = order_weights

#times = order_list_timeseries.times.-time_max_alt_sun
#plot(times,order_weights', label=:none)
=#

#=
for obsid in 1:length(order_list_timeseries)
    for ch in 1:length(order_list_timeseries[1].data)
        order_list_timeseries[obsid].data[ch].flux .*= order_weights[ch,obsid] #./mean_order_mean_flux[ch]
        order_list_timeseries[obsid].data[ch].var .*= (order_weights[ch,obsid]).^2 # ./mean_order_mean_flux[ch]).^2
    end
end
=#

#=for obsid in 1:length(order_list_timeseries)
    for ch in 1:length(order_list_timeseries[1].data)
        order_list_timeseries[obsid].data[ch].flux ./= order_weights[ch,obsid] #./mean_order_mean_flux[ch]
        order_list_timeseries[obsid].data[ch].var ./= (order_weights[ch,obsid]).^2 # ./mean_order_mean_flux[ch]).^2
    end
end
=#

#=
using Plots
 plot(map(obsid->NaNMath.sum(order_list_timeseries[obsid].data[1].flux)/(NaNMath.sum(order_list_timeseries[obsid].data[end].flux)),
                    1:length(all_spectra)),label="Raw")
 #plot!(map(obsid->NaNMath.sum(order_list_timeseries[obsid].data[10].flux)*order_weights[10,obsid]/(NaNMath.sum(order_list_timeseries[obsid].data[30].flux*order_weights[30,obsid])), 1:length(all_spectra)),label="weighted")
 #plot!(map(obsid->NaNMath.sum(order_list_timeseries[obsid].data[10].flux)*order_weights[10,obsid]^2/(NaNMath.sum(order_list_timeseries[obsid].data[30].flux*order_weights[30,obsid]^2)), 1:length(all_spectra)),label="weighted^2")

# plot(order_list_timeseries[obsid].data[ch].λ,order_list_timeseries[obsid].data[ch].flux)
=#

#=
times = order_list_timeseries.times.-time_max_alt_sun
for obsid in 1:length(order_list_timeseries)
    order_snrs[:,obsid] ./= order_snrs[15,obsid]
    order_snrs_new[:,obsid] ./= order_snrs_new[15,obsid]
end

plot(times,order_snrs', label=:none)
plot!(times,order_snrs_new'./order_snrs_new[15,:]', label=:none)

plt = plot()
for obsid in 1:length(order_list_timeseries)
    for ch in 1:10:length(order_list_timeseries[1].data)
        plot(order_list_timeseries[obsid].data[ch].λ,order_list_timeseries[obsid].data[ch].flux)

    end
end
=#

#=
maxΔfwhm² = -0.569375
@assert maximum(df_files_use.Δfwhm²) < maxΔfwhm²
Δfwhm = 1000.0 .* sqrt.(maxΔfwhm².-df_files_use.Δfwhm²)  # How much to increase fwhm by to acheive uniform fwhm

#msf = 1.4; fwtf = 1.5
#msf = 6.0; fwtf = 1.5
#msf = 0.62; fwtf = 1.5  # one pixel because tophat not using NEID's default v width
#msf = 1.0; fwtf = 1.5  # using NEID's default v width
msf = lsf_width/default_ccf_mask_v_width(NEID2D()); fwtf = 0.5  # using LSF width
#msf = line_width_50_default/default_ccf_mask_v_width(NEID2D()); fwtf = 1.0  # using line width
#msf = 1.24; fwtf = 1.5   # two pixels
 println("# mask_scale_factor = ", msf, ", frac_width_to_fit =  ", fwtf, ". ")
 ((ccfs_espresso, ccf_vars_espresso), v_grid) = ccf_total(order_list_timeseries, line_list_espresso, pipeline_plan,
   mask_type=:gaussian, Δfwhm=Δfwhm,
   mask_scale_factor=msf, range_no_mask_change=5*line_width_50, ccf_mid_velocity=ccf_mid_velocity, v_step=100, #155,
   v_max=max(5*line_width_50,2*max_bc), allow_nans=true, calc_ccf_var=true, recalc=true)
  outputs["v_grid"] = v_grid
  outputs["ccfs_espresso"] = ccfs_espresso
  outputs["ccf_vars_espresso"] = ccf_vars_espresso

ccf_median = vec(median(ccfs_espresso,dims=2))
 ccf_dist_from_median = map(i->RvSpectML.earth_mover_distance(v_grid,ccf_median,ccfs_espresso[:,i]), 1:size(ccfs_espresso,2) )

fwtf = 0.5; println("# mask_scale_factor = ", msf, ", frac_width_to_fit =  ", fwtf, ". ")
 alg_fit_rv =  EchelleCCFs.MeasureRvFromCCFGaussian(frac_of_width_to_fit=fwtf)
 rvs_ccf_espresso = RvSpectML.calc_rvs_from_ccf_total(ccfs_espresso, ccf_vars_espresso, pipeline_plan, v_grid=v_grid, times = order_list_timeseries.times, recalc=true, bin_nightly=true, alg_fit_rv=alg_fit_rv)
 σ_rvs_espresso = deepcopy(read_cache(pipeline_plan,:σ_rvs_ccf_total))
 #scatter((order_list_timeseries.times.-first(order_list_timeseries.times)).*24, rvs_ccf_espresso, label=:none)
 outputs["rvs_ccf_espresso"] = rvs_ccf_espresso
 outputs["σ_rvs_espresso"] = σ_rvs_espresso

 delay_until_first_obs_used_for_fit = 1/24
 times = copy(order_list_timeseries.times)
 #Δt_fit = min(time_max_alt_sun-(first(times)+delay_until_first_obs_used_for_fit),last(times)-time_max_alt_sun)
 Δt_fit = 2.0/24
 idx_first_obs_to_fit = findfirst(t->time_max_alt_sun-t<=Δt_fit,times)
 @assert 1 <= idx_first_obs_to_fit length(times)
 idx_last_obs_to_fit = findlast(t->t-time_max_alt_sun<=Δt_fit,times)
 idx_to_fit = idx_first_obs_to_fit:idx_last_obs_to_fit
 idx_to_fit = idx_to_fit[ccf_dist_from_median[idx_to_fit] .< 10]
 (mean_rv_to_fit, std_rv_to_fit) = mean_and_std(rvs_ccf_espresso[idx_to_fit])
 idx_for_robust_fit = idx_to_fit[mean_rv_to_fit-3*std_rv_to_fit .<= rvs_ccf_espresso[idx_to_fit] .<= mean_rv_to_fit+3*std_rv_to_fit]

 times .-= time_max_alt_sun
 #times = order_list_timeseries.times.-first(order_list_timeseries.times)
 x = [times[idx_for_robust_fit]  ones(length(times[idx_for_robust_fit])) ]
 linfit = (x'*x)\x'*(rvs_ccf_espresso[idx_for_robust_fit])
 x = [times  ones(length(times)) ]
 v_fit = (x*linfit)
 v_t0 = [[0] [1]]*linfit
 idx_not_bad_distance = ccf_dist_from_median .< 10
 (times_binned, rvs_binned) = bin_times_and_rvs_max_Δt(times=times[idx_not_bad_distance], rvs=rvs_ccf_espresso[idx_not_bad_distance], Δt_threshold=5/(60*24))
 #idx_to_fit_binned = findfirst(t->t>=times[first(idx_to_fit)], times_binned):findlast(t->t<=times[last(idx_to_fit)], times_binned)
 #idx_to_fit_binned = findfirst(t->time_max_alt_sun-t.<=Δt_fit,times_binned):findlast(t->t-time_max_alt_sun.<=Δt_fit,times_binned)
 idx_to_fit_binned = findfirst(t->t>=-Δt_fit,times_binned):findlast(t->t<=Δt_fit,times_binned)
 x = [times_binned  ones(length(times_binned)) ]
 v_fit_binned = x*linfit
 println("# RMS to linear fit: ", std(rvs_ccf_espresso[idx_to_fit].-v_fit[idx_to_fit]), " m/s.  Binned: ", std(rvs_binned[idx_to_fit_binned].-v_fit_binned[idx_to_fit_binned]))
 println("# Mean RV = ", mean(rvs_binned[idx_to_fit_binned]), " m/s  Slope = ", linfit[1]/24, " m/s/hr")
 outputs["times_binned"] = times_binned
 outputs["rvs_binned"] = rvs_binned
 outputs["rv_rms"] = std(rvs_ccf_espresso[idx_to_fit].-v_fit[idx_to_fit])
 outputs["rv_0"] = v_t0
 outputs["rv_mean"] = mean(rvs_binned[idx_to_fit_binned])
 outputs["rv_slope"] = linfit[1]/24
 outputs["rv_rms_binned"] = std(rvs_binned[idx_to_fit_binned].-v_fit_binned[idx_to_fit_binned])

using Plots
 plt = scatter( (times[idx_not_bad_distance]).*24, (rvs_ccf_espresso.-v_fit)[idx_not_bad_distance],  ms=1.5, label=:none)
 scatter!(plt,(times_binned).*24, (rvs_binned.-v_fit_binned), ms=4.0, label=:none)
 vline!(plt,[-Δt_fit,Δt_fit].* 24, label=:none, color=:grey)
 vline!(plt,[0].* 24, label=:none, color=:orange)
 xlims!(plt,-3.5,3.5)
 #ylims!(plt,minimum(rvs_ccf_espresso.-v_fit)-0.5*std_rv_to_fit, maximum(rvs_ccf_espresso.-v_fit)+0.5*std_rv_to_fit)
 ylims!(plt, extrema((rvs_ccf_espresso.-v_fit)[idx_not_bad_distance]))
 xlabel!(plt,"Time (hr)")
 ylabel!(plt,"RV - linear fit (m/s)")
 title!(plt,"Sun " * date_str * " CCF ESPRESSO\n" *
        "rms_5m = " * string(round(std(rvs_binned[idx_to_fit_binned].-v_fit_binned[idx_to_fit_binned]),digits=3)) * "m/s" *
        " mean = " * string(round(mean(rvs_binned[idx_to_fit_binned]),digits=2)) *
        " Slope = " * string(round(linfit[1]/24,digits=2))  * "m/s/hr")
 display(plt)
 savefig(joinpath(output_dir,"rvs_" * date_str * ".png"))

(ccfs_norm, ccf_vars_norm) = EchelleCCFs.calc_normalized_ccfs( v_grid, ccfs_espresso, ccf_vars_espresso, v_mid=ccf_mid_velocity, dv_min=2.5*line_width_50,dv_max= 2*max_bc )
 ccf_template = EchelleCCFs.calc_ccf_template(ccfs_norm[:,idx_to_fit], ccf_vars_norm[:,idx_to_fit], assume_normalized=true)
 ccf_vars_template = EchelleCCFs.calc_ccf_template(ccf_vars_norm[:,idx_to_fit], ccf_vars_norm[:,idx_to_fit], assume_normalized=true)
 #plot(v_grid,ccfs_espresso, labels=:none)
 #plot(v_grid,ccfs_norm , labels=:none)
 #plt = plot(v_grid,(ccfs_norm.-ccf_template)[:,range(1,size(ccfs_norm,2),step=2)], labels=:none)
 #xlims!(plt,ccf_mid_velocity-2*line_width_50,ccf_mid_velocity+2*line_width_50)
 #heatmap(v_grid,1:size(ccfs_norm,2),(ccfs_norm.-ccf_template)',seriescolor=:balance)
 cols_to_fit = findlast(x->x<ccf_mid_velocity-3*line_width_50,v_grid):findfirst(x->x>ccf_mid_velocity+3*line_width_50,v_grid)
 maxabsz = maximum(abs.(extrema(view(ccfs_norm,cols_to_fit,idx_to_fit)'.-view(ccf_template,cols_to_fit,:)')))
 plt = heatmap(view(v_grid,cols_to_fit),(times[1:size(ccfs_norm,2)]).*24,view(ccfs_norm,cols_to_fit,:)'.-view(ccf_template,cols_to_fit,:)',clims=(-maxabsz,maxabsz),seriescolor=:balance)
 xlabel!(plt,"v (m/s)")
 ylabel!(plt,"Observation Time (hr)")
 title!(plt,"Sun " * date_str * ":\nCCF - <CCF>, ESPRESSO mask + SNR weights")
 #xlims!(ccf_mid_velocity-RvSpectMLBase.max_bc,ccf_mid_velocity+RvSpectMLBase.max_bc)
 xlims!(plt,ccf_mid_velocity-2*line_width_50,ccf_mid_velocity+2*line_width_50)
 hline!(plt,[-Δt_fit,Δt_fit].*24, label=:none, color=:grey)
 hline!(plt,[0.], label=:none, color=:orange)
 ylims!(plt,-3.5,3.0)
 #vline!([-2.1e4], color=4, label=:none)
 display(plt)
 #savefig(joinpath(output_dir,"ccf_heatmaps_sun_" * date_str * "_espresso_msf=" * string(round(msf,digits=1)) * ".png"))
 savefig(joinpath(output_dir,"ccf_heatmaps_sun_" * date_str * "_espresso_msf=" * string(round(msf,digits=1)) * "_new.png"))

 outputs["mean_ccf"] = ccf_template

gp_post = RvSpectML.TemporalGPInterpolation.construct_gp_posterior(v_grid,ccf_template,sigmasq_obs=ccf_vars_template,use_logx=false,smooth_factor=2)
 ccf_template_smooth = RvSpectML.TemporalGPInterpolation.predict_mean(gp_post(v_grid))
 ccf_template_deriv = RvSpectML.numerical_deriv(v_grid,RvSpectML.TemporalGPInterpolation.predict_mean(gp_post(v_grid)))
 norm = sum(abs2.(ccf_template_deriv[cols_to_fit]))
 rvs_proj = vec(sum((ccf_template_smooth[cols_to_fit].-ccfs_norm[cols_to_fit,:]).*ccf_template_deriv[cols_to_fit],dims=1)./norm)
 σ_rvs_proj = vec(sum((ccf_vars_norm[cols_to_fit]).*abs2.(ccf_template_deriv[cols_to_fit]),dims=1))./ norm
 #ccf_resid_minus_rv_proj = (ccfs_norm.-ccf_template).+(rvs_proj.*ccf_template_deriv')'
 ccf_resid_minus_rv_proj = (ccfs_norm.-ccf_template).+((v_fit.-mean(v_fit)).*ccf_template_deriv')'

 outputs["ccf_template_smooth"] = ccf_template_smooth
 outputs["rvs_proj"] = rvs_proj
 outputs["σ_rvs_proj"] = σ_rvs_proj
 outputs["ccf_resid_minus_rv_proj"] = ccf_resid_minus_rv_proj

#plt = heatmap(view(v_grid,cols_to_fit),(times[1:size(ccfs_norm,2)]).*24,ccf_resid_minus_rv_proj[cols_to_fit,:]')
maxabsz = maximum(abs.(extrema(ccf_resid_minus_rv_proj)))
climmax = 0.00030
 plt = heatmap(v_grid,(times[1:size(ccfs_norm,2)]).*24,ccf_resid_minus_rv_proj',clims=(-climmax,climmax),seriescolor=:balance)
 xlabel!(plt,"v (m/s)")
 ylabel!(plt,"Observation Time (hr)")
 title!(plt,"Sun " * date_str * ":\nCCF-<CCF>+v⋅d<CCF>/dv")
 #xlims!(ccf_mid_velocity-RvSpectMLBase.max_bc,ccf_mid_velocity+RvSpectMLBase.max_bc)
 xlims!(plt,ccf_mid_velocity-2*line_width_50,ccf_mid_velocity+2*line_width_50)
 hline!(plt,[-Δt_fit,Δt_fit].*24, label=:none, color=:grey)
 hline!(plt,[0.], label=:none, color=:orange)
 ylims!(plt,-3.5,3.0)
 display(plt)
 #savefig(joinpath(output_dir,"ccf_resid_heatmaps_sun_" * date_str * "_espresso_msf=" * string(round(msf,digits=1)) * ".png"))
 savefig(joinpath(output_dir,"ccf_resid_heatmaps_sun_" * date_str * "_espresso_msf=" * string(round(msf,digits=1)) * "_new.png"))
=#
@assert true

#=
 all_spectra = nothing
 order_list_timeseries = nothing
 GC.gc()
end
=#

#EchelleCCFs.write_each_ccf_fits(map(s->s.metadata,all_spectra), v_grid, ccfs_espresso, ccf_vars_espresso,   fits_hdr=EchelleCCFs.make_ccf_fits_header(all_spectra[1].metadata),
#            line_list_filename=line_list_filename,total_vel=rvs_ccf_espresso, sigma_total_vel=σ_rvs_espresso )

#=
##
max_orders = 56:111
  order_list_timeseries = extract_orders(all_spectra,pipeline_plan, orders_to_use=max_orders, recalc=true )

line_list_filename = "G2.espresso.mas"
 linelist_for_ccf_fn_w_path = joinpath(pkgdir(EchelleCCFs),"data","masks",line_list_filename)
 line_list_espresso = prepare_line_list(linelist_for_ccf_fn_w_path, all_spectra, pipeline_plan, v_center_to_avoid_tellurics=ccf_mid_velocity,
      Δv_to_avoid_tellurics = 1*RvSpectMLBase.max_bc+5*line_width_50+2*default_ccf_mask_v_width(EXPRES1D()),
      orders_to_use=max_orders, recalc=true, verbose=true)

##

good_orders = orders_to_use_default(NEID2D())

(order_ccfs, order_ccf_vars, v_grid_order_ccfs) = ccf_orders(order_list_timeseries, line_list_espresso, pipeline_plan, mask_scale_factor=msf, ccf_mid_velocity=ccf_mid_velocity,
 v_step=155, v_max=max(5*line_width_50,2*max_bc), orders_to_use=good_orders, allow_nans=true, calc_ccf_var=true,recalc=true)
 outputs["order_ccfs"] = order_ccfs
 outputs["order_ccf_vars"] = order_ccf_vars
 outputs["v_grid_order_ccfs"] =v_grid_order_ccfs

obs_id = idx_to_fit[1]
 plt = heatmap(v_grid_order_ccfs, 1:size(order_ccfs,2), (order_ccfs[:,:,obs_id]./mean(order_ccfs[:,:,obs_id],dims=1))')
 xlabel!(plt,"v (m/s)")
 ylabel!(plt,"Order (ignore #)")
 title!(plt,"Sun " * date_str  * " Obs# " * string(obs_id))
 #xlims!(ccf_mid_velocity-RvSpectMLBase.max_bc,ccf_mid_velocity+RvSpectMLBase.max_bc)
 xlims!(plt,ccf_mid_velocity-2.0*line_width_50,ccf_mid_velocity+2.0*line_width_50)
 display(plt)
 savefig(joinpath(output_dir,"order_ccf_heatmaps_sun_" * date_str * "_espresso_msf=" * string(round(msf,digits=1)) * ".png"))

ccf_order_template = zeros(size(order_ccfs,1),size(order_ccfs,2))
 ccf_order_template_vars = zeros(size(order_ccfs,1),size(order_ccfs,2))
 ccf_order_resid = zeros(size(order_ccfs,1),size(order_ccfs,2),size(order_ccfs,3))
 rvs_proj_order = zeros(length(times),size(order_ccfs,2))
 σ_rvs_proj_order = zeros(length(times),size(order_ccfs,2))
 ccf_order_resid_rv_from_order = zeros(size(order_ccfs,1),size(order_ccfs,2),size(order_ccfs,3))

 for ord in 1:size(order_ccfs,2)
     if !(sum(order_ccfs[:,ord,:]) > 0)  continue end
     (ccfs_order_norm, ccf_order_vars_norm) = EchelleCCFs.calc_normalized_ccfs( v_grid, order_ccfs[:,ord,:], order_ccf_vars[:,ord,:], v_mid=ccf_mid_velocity, dv_min=2.5*line_width_50,dv_max= 2*max_bc )
     ccf_order_template[:,ord] .= EchelleCCFs.calc_ccf_template(ccfs_order_norm[:,idx_to_fit], ccf_order_vars_norm[:,idx_to_fit], assume_normalized=true)
     ccf_order_template_vars[:,ord] .= EchelleCCFs.calc_ccf_template(ccf_order_vars_norm[:,idx_to_fit], ccf_order_vars_norm[:,idx_to_fit], assume_normalized=true)
     println("# ord = ", ord, " mean = ", NaNMath.mean(ccf_order_template[:,ord]), " var = ", NaNMath.mean(ccf_order_template_vars[:,ord]))
     gp_post = RvSpectML.TemporalGPInterpolation.construct_gp_posterior(v_grid,ccf_order_template[:,ord],sigmasq_obs=ccf_order_template_vars[:,ord],use_logx=false,smooth_factor=2)
     ccf_template_smooth = RvSpectML.TemporalGPInterpolation.predict_mean(gp_post(v_grid))
     ccf_template_deriv = RvSpectML.numerical_deriv(v_grid,RvSpectML.TemporalGPInterpolation.predict_mean(gp_post(v_grid)))
     ccf_order_resid[:,ord,:] .= (ccfs_order_norm.-ccf_order_template[:,ord]).+(rvs_proj.*ccf_template_deriv')'
     norm = sum(abs2.(ccf_template_deriv[cols_to_fit]))
     rvs_proj_order[:,ord] .= vec(sum((ccf_template_smooth[cols_to_fit].-ccfs_order_norm[cols_to_fit,:]).*ccf_template_deriv[cols_to_fit],dims=1))./norm
     σ_rvs_proj_order[:,ord] .= vec(sum((ccf_order_vars_norm[cols_to_fit,:]).*abs2.(ccf_template_deriv[cols_to_fit]),dims=1))./ norm
     ccf_order_resid_rv_from_order[:,ord,:] .= (ccfs_order_norm.-ccf_template).+(rvs_proj_order[:,ord].*ccf_template_deriv')'
 end
 outputs["ccf_order_template"] = ccf_order_template
 outputs["ccf_order_template_vars"] = ccf_order_template_vars
 outputs["rvs_proj_order"] = rvs_proj_order
 outputs["σ_rvs_proj_order"] = σ_rvs_proj_order
 outputs["ccf_order_resid"] = ccf_order_resid
 outputs["ccf_order_resid_rv_from_order"] = ccf_order_resid_rv_from_order
=#
#=
obs_id = idx_to_fit[6]
 #maxabsz = maximum(abs.(extrema(ccf_order_resid[:,:,obs_id])))
 #plt = heatmap(v_grid_order_ccfs, 1:size(order_ccfs,2), ccf_order_resid[:,:,obs_id]',clims=(-maxabsz,maxabsz),seriescolor=:balance)
 maxabsz = maximum(abs.(extrema(reshape(sum(ccf_order_resid,dims=3),size(ccf_order_resid)[1:2]))))
 plt = heatmap(v_grid_order_ccfs, 1:size(order_ccfs,2), reshape(sum(ccf_order_resid,dims=3),size(ccf_order_resid)[1:2])',clims=(-maxabsz,maxabsz),seriescolor=:balance)
 xlabel!(plt,"v (m/s)")
 ylabel!(plt,"Order (ignore #)")
 title!(plt,"Sun " * date_str  )
 #xlims!(plt,ccf_mid_velocity-RvSpectMLBase.max_bc,ccf_mid_velocity+RvSpectMLBase.max_bc)
 xlims!(plt,ccf_mid_velocity-2.0*line_width_50,ccf_mid_velocity+2.0*line_width_50)
 display(plt)
 #savefig(joinpath(output_dir,"order_ccf_resid_heatmaps_sun_" * date_str * "_espresso_msf=" * string(round(msf,digits=1)) * ".png"))
=#
#=
alg_fit_rv2 =  EchelleCCFs.MeasureRvFromCCFGaussian(frac_of_width_to_fit=fwtf)
order_rvs_g = zeros(length(all_spectra), size(order_ccfs,2))
order_rv_std_g = zeros(length(all_spectra), size(order_ccfs,2))
order_rvs_t = zeros(length(all_spectra), size(order_ccfs,2))
order_rv_std_t = zeros(length(all_spectra), size(order_ccfs,2))
for i in 1:size(order_ccfs,2)
   println("# Index for order: ", i)
   if sum(view(order_ccfs,:,i,1)) > 0
      rvs_order_ccf = RvSpectML.calc_rvs_from_ccf_total(view(order_ccfs,:,i,:), view(order_ccf_vars,:,i,:), pipeline_plan, v_grid=v_grid_order_ccfs, times = order_list_timeseries.times, recalc=true, bin_nightly=true, alg_fit_rv=alg_fit_rv2)
      alg_fit_rv3 =  EchelleCCFs.MeasureRvFromCCFTemplate(v_grid=v_grid_order_ccfs, frac_of_width_to_fit=fwtf, template = vec(mean(view(order_ccfs,:,i,:),dims=2)) )
      order_rvs_g[:,i] .= read_cache(pipeline_plan, :rvs_ccf_total )
      order_rv_std_g[:,i] .= read_cache(pipeline_plan, :σ_rvs_ccf_total)
      rvs_order_ccf = RvSpectML.calc_rvs_from_ccf_total(view(order_ccfs,:,i,:), view(order_ccf_vars,:,i,:), pipeline_plan, v_grid=v_grid_order_ccfs, times = order_list_timeseries.times, recalc=true, bin_nightly=true, alg_fit_rv=alg_fit_rv3)
      order_rvs_t[:,i] .= read_cache(pipeline_plan, :rvs_ccf_total )
      order_rv_std_t[:,i] .= read_cache(pipeline_plan, :σ_rvs_ccf_total)
   end
end
outputs["order_rvs_g"] = order_rvs_g
outputs["order_rvs_t"] = order_rvs_t
=#

#=
outputs[""] =
outputs[""] =
outputs[""] =
outputs[""] =

=#
#=
order_snr = mapreduce(obsid->map(ord->RvSpectMLBase.calc_snr(all_spectra[obsid],787:6214,ord)  ,1:size(first(all_spectra).flux,2)), hcat, 1:length(all_spectra))
outputs["order_snr"] = order_snr

save(outputs_filename, outputs)
=#
#=

EchelleCCFs.write_each_ccf_fits(map(s->s.metadata,all_spectra), #= 100.0 .* EXPRESS USED cm/s =# v_grid, ccfs_espresso, ccf_vars_espresso,
                  fits_hdr=EchelleCCFs.make_ccf_fits_header(all_spectra[1].metadata),
                  line_list_filename=line_list_filename,# orders=161 .-good_orders,ccf_orders=order_ccfs,ccf_orders_var=order_ccf_vars,
                  total_vel=100.0 .* rvs_ccf_espresso, sigma_total_vel= 100.0 .* σ_rvs_espresso,
                  order_vels = 100.0 .* order_rvs_g, sigma_order_vels = 100.0 .* order_rv_std_g )
=#


function calculate_initial_conditions!(sim::Simulation, inputs::SimulationInputs)
    @debug "Start state intialization routine"
    sys = get_system(inputs)
    initial_guess = sim.x0_init
    res = PSY.solve_powerflow!(sys)
    if !res
        error("Power Flow fail")
    end
    var_count = get_variable_count(inputs)

    @debug "Setting up initilization indexing"
    bus_size = get_bus_count(inputs)
    bus_vars_count = 2 * bus_size
    bus_range = 1:bus_vars_count
    injection_start = get_injection_pointer(inputs)
    injection_count = 1
    branches_start = get_branches_pointer(inputs)
    branches_count = 1

    @debug "Updating Voltage guess"
    for bus in PSY.get_components(PSY.Bus, sys)
        @debug PSY.get_name(bus)
        @debug "V_r" PSY.get_magnitude(bus) * cos(PSY.get_angle(bus))
        @debug "V_i" PSY.get_magnitude(bus) * sin(PSY.get_angle(bus))
        bus_n = PSY.get_number(bus)
        bus_ix = get_lookup(inputs)[bus_n]
        initial_guess[bus_ix] = PSY.get_magnitude(bus) * cos(PSY.get_angle(bus))
        initial_guess[bus_ix + bus_size] = PSY.get_magnitude(bus) * sin(PSY.get_angle(bus))
    end

    @debug "Updating Source internal voltages"
    sources = PSY.get_components(PSY.Source, sys)
    if !isempty(sources)
        for s in sources
            initialize_device!(s)
        end
    end

    @debug "Updating Dynamic Injection Component Initial Guess"
    for d in get_injectors_data(inputs)
        dynamic_device = PSY.get_dynamic_injector(d)
        @debug PSY.get_name(d) typeof(d) typeof(dynamic_device)
        bus = PSY.get_bus(d)
        bus_n = PSY.get_number(PSY.get_bus(d))
        bus_ix = get_lookup(inputs)[bus_n]
        n_states = PSY.get_n_states(dynamic_device)
        ix_range = range(injection_start, length = n_states)
        injection_start = injection_start + n_states
        x0_device = initialize_device!(d)
        @assert length(x0_device) == n_states
        initial_guess[ix_range] = x0_device
    end

    @debug "Updating Component Initial Guess"
    dyn_branches = PSY.get_components(PSY.DynamicBranch, sys)
    if !isempty(dyn_branches)
        for br in dyn_branches
            @debug PSY.get_name(br) typeof(br)
            arc = PSY.get_arc(br)
            n_states = PSY.get_n_states(br)
            from_bus_number = PSY.get_number(arc.from)
            to_bus_number = PSY.get_number(arc.to)
            bus_ix_from = get_lookup(inputs)[from_bus_number]
            bus_ix_to = get_lookup(inputs)[to_bus_number]
            ix_range = range(branches_start, length = n_states)
            branches_start = branches_start + n_states
            x0_branch = initialize_device!(br)
            @assert length(x0_branch) == n_states
            initial_guess[ix_range] = x0_branch
        end
    else
        @debug "No Dynamic Branches in the system"
    end

    dx0 = zeros(var_count) #Define a vector of zeros for the derivative
    inif! = (out, x) -> system!(
        out,    #output of the function
        dx0,    #derivatives equal to zero
        x,      #states
        inputs,    #Parameters
        0.0,    #time equals to zero.
    )
    #Refine initial solution
    @debug "Start NLSolve Run"
    sys_solve = NLsolve.nlsolve(
        inif!,
        initial_guess,
        xtol = STRICT_NL_SOLVE_TOLERANCE,
        ftol = STRICT_NL_SOLVE_TOLERANCE,
        method = :trust_region,
    ) #Solve using initial guess x0
    if !NLsolve.converged(sys_solve)
        @warn("Initialization failed, initial conditions do not meet conditions for an stable equilibrium.\nTrying to solve again reducing numeric tolerance from $(STRICT_NL_SOLVE_TOLERANCE):")
        sys_solve = NLsolve.nlsolve(
            inif!,
            initial_guess,
            xtol = RELAXED_NL_SOLVE_TOLERANCE,
            ftol = RELAXED_NL_SOLVE_TOLERANCE,
            method = :trust_region,
        ) #Solve using initial guess x0
        if NLsolve.converged(sys_solve)
            @info("Initialization succeeded with a relaxed tolerance of $(RELAXED_NL_SOLVE_TOLERANCE). Saving solution")
        else
            @warn("Initialization failed again. Initial conditions do not meet conditions for an stable equilibrium\nSaving best result.")
        end
    end
    @debug "Write result to initial guess vector"
    initial_guess .= sys_solve.zero
    return NLsolve.converged(sys_solve)
end

"""
Returns a Dictionary with the resulting initial conditions of the simulation
"""
function get_initial_conditions(sim::Simulation)
    system = get_system(sim.simulation_inputs)
    bus_size = get_bus_count(sim.simulation_inputs)
    V_R = Dict{Int64, Float64}()
    V_I = Dict{Int64, Float64}()
    Vm = Dict{Int64, Float64}()
    θ = Dict{Int64, Float64}()
    for bus in PSY.get_components(PSY.Bus, system)
        bus_n = PSY.get_number(bus)
        bus_ix = get_lookup(sim.simulation_inputs)[bus_n]
        V_R[bus_n] = sim.x0_init[bus_ix]
        V_I[bus_n] = sim.x0_init[bus_ix + bus_size]
        Vm[bus_n] = sqrt(sim.x0_init[bus_ix]^2 + sim.x0_init[bus_ix + bus_size]^2)
        θ[bus_n] = angle(sim.x0_init[bus_ix] + sim.x0_init[bus_ix + bus_size] * 1im)
    end
    results = Dict{String, Any}("V_R" => V_R, "V_I" => V_I, "Vm" => Vm, "θ" => θ)
    for device in PSY.get_components(PSY.DynamicInjection, system)
        states = PSY.get_states(device)
        name = PSY.get_name(device)
        global_index = get_global_index(sim.simulation_inputs)[name]
        x0_device = Dict{Symbol, Float64}()
        for (i, s) in enumerate(states)
            x0_device[s] = sim.x0_init[global_index[s]]
        end
        results[name] = x0_device
    end
    dyn_branches = PSY.get_components(PSY.DynamicBranch, system)
    if !isempty(dyn_branches)
        for br in dyn_branches
            states = PSY.get_states(br)
            name = PSY.get_name(br)
            global_index = get_global_index(sim.simulation_inputs)[name]
            x0_br = Dict{Symbol, Float64}()
            for (i, s) in enumerate(states)
                x0_br[s] = sim.x0_init[global_index[s]]
            end
            printed_name = "Line " * name
            results[printed_name] = x0_br
        end
    end
    return results
end


#Function to deal with positioners
struct POSRobotBody
    #positioner body is theta arm and head is for phi arm
    #Any thing stored in two element array will always asume body first and head second
    max_seg::Integer #maximum number of segements this object can store

    xy::Array{Float64,1} #To store the xy position of the positioner
    arms::Array{Float64,1} #To store the theta.phi arm of the positioner
    arms2_lim::Array{Float64,1} #To store the square of minimum and maximum limit of the robot arm

    nseg::Array{Integer,1} #Stores the number of segment in body and head
    restricted_angels::Array{Bool,1} #Store if any angular restriction is needed

    body_minmax::Array{Float64,1} #Stores the minimum and maximum length of the verted for body
    body_orig::Array{Float64, 2} #Stores the orginal polygon of body
    head_minmax::Array{Float64,1} #Stores the minimum and maximum length of the verted for head
    head_orig::Array{Float64, 2} #Stores the original polygon of head

    axis_orig::Array{Float64,1} #To store the orignal axis

    #To get the theta limit and phi limit: Angles are stored in radian
    body_theta_limit::Array{Float64,1} #Store the limit of angles for body MIN_T,MAX_T,OFFSET_T
    head_theta_limit::Array{Float64,1} #Store the limit of angles for head MIN_P,MAX_P,OFFSET_P

    body_on_target::Array{Float64, 2} #Stores the body polygon on target
    head_on_target::Array{Float64, 2} #Stores the head polygon on target
    axis_on_target::Array{Float64,1} #This goes throgh the same transformation as head to keep tract of its axis

    function  POSRobotBody(maximum_seg::Integer)
        new(maximum_seg,zeros(2),zeros(2),zeros(2),zeros(2),
            zeros(Bool,2),zeros(2),zeros(maximum_seg,2),zeros(2),
            zeros(maximum_seg,2),zeros(2),zeros(3),zeros(3),
            zeros(maximum_seg,2),zeros(maximum_seg,2),zeros(2))
    end
end

"""
Initialize the pos:POSRobotBody with the position at index in fp_dic
"""
function load_positioner!(pos::POSRobotBody,index,fp_dic,exc_dic)

    #get the exclusion names
    excl_name=fp_dic["EXCLUSION"][index]
    excl_body_name="$(excl_name)-theta-segments"
    excl_head_name="$(excl_name)-phi-segments"

    #Assign number of segments
    pos.nseg[1]=size(exc_dic[excl_body_name],1)
    pos.nseg[2]=size(exc_dic[excl_head_name],1)

    #Assign polygon for body
    Assign_polygon_with_maxmin_vertex!(pos.max_seg,pos.nseg[1],
        exc_dic[excl_body_name],pos.body_orig,pos.body_minmax)

    #Assign polygon for head
    Assign_polygon_with_maxmin_vertex!(pos.max_seg,pos.nseg[2],
        exc_dic[excl_head_name],pos.head_orig,pos.head_minmax)

    #get the angular limit of the robots body
    pos.body_theta_limit[1]=deg2rad(fp_dic["MIN_T"][index])
    pos.body_theta_limit[2]=deg2rad(fp_dic["MAX_T"][index])
    pos.body_theta_limit[3]=deg2rad(fp_dic["OFFSET_T"][index])

    #get the angular limit of the robots head
    pos.head_theta_limit[1]=deg2rad(fp_dic["MIN_P"][index])
    pos.head_theta_limit[2]=deg2rad(fp_dic["MAX_P"][index])
    pos.head_theta_limit[3]=deg2rad(fp_dic["OFFSET_P"][index])

    #Allocate xy
    pos.xy[1]=fp_dic["OFFSET_X"][index]
    pos.xy[2]=fp_dic["OFFSET_Y"][index]

    #Assign arms
    pos.arms[1]=fp_dic["LENGTH_R1"][index]
    pos.arms[2]=fp_dic["LENGTH_R2"][index]

    #arms limits
    pos.arms2_lim[1]=(pos.arms[1]-pos.arms[2])^2 #minimum
    pos.arms2_lim[2]=(pos.arms[1]+pos.arms[2])^2 #maximum

end

function Assign_polygon_with_maxmin_vertex!(max_seg,nseg,polygon,pos_poly,minmax)
    cent=[mean(polygon[:,1]),mean(polygon[:,2])]
    min_vert2=100000
    max_vert2=0

    for ii in 1:nseg
        pos_poly[ii,1]=polygon[ii,1]
        pos_poly[ii,2]=polygon[ii,2]
        rad2=(polygon[ii,1]-cent[1])^2 + (polygon[ii,2]-cent[2])^2
        if(max_vert2<rad2)
            max_vert2=rad2
        end
        if(min_vert2>rad2)
            min_vert2=rad2
        end
    end

    for ii in nseg+1:max_seg
        pos_poly[ii,1]= 0
        pos_poly[ii,2]= 0
    end

    minmax[1]=sqrt(min_vert2)
    minmax[2]=sqrt(max_vert2)
end

"""
Applies the tanslation on poly_in by x,y and return result in poly_out
"""
function poly_translate!(poly_in,poly_out,x,y)
    for ii in 1:size(poly_in,1)
        poly_out[ii,1]=poly_in[ii,1]+x
        poly_out[ii,2]=poly_in[ii,2]+y
    end
end

function point_translate!(pt_in,pt_out,x,y)
    pt_out[1]= pt_in[1] + x
    pt_out[2]= pt_in[2] + y
end

"""Rotates the points in poly around the axis
by cos(theta),sin(theta)

diff_v: optional to use pre-allocated array for intermediate step for speed
cos_sin: optional to use pre-allocated array for intermediate step for speed
cos_sin_sum: optional to use pre-allocated array for intermediate step for speed
"""
function poly_rotate_axis!(poly_in,poly_out,cstheta,axis;
    diff_v=zeros(Float64,2),cos_sin=zeros(Float64,2),cos_sin_sum=zeros(Float64,2))
    for ii in 1:size(poly_in,1)
        point_rotate_axis!(view(poly_in,ii,:),cstheta,axis,view(poly_out,ii,:);
         diff_v=diff_v,cos_sin=cos_sin,cos_sin_sum=cos_sin_sum)
    end
end

"""
Apply a rotation to a point along the axis
point_in: input point x,y coordinate
point_out: x,y co-ordinate after rotation
cstheta: [cos(theta, sin(theta)] for rotation by theta
axis: [0,0] : center about which rotation is needed
diff_v: optional to use pre-allocated array for intermediate step for speed
cos_sin: optional to use pre-allocated array for intermediate step for speed
cos_sin_sum: optional to use pre-allocated array for intermediate step for speed
"""
function point_rotate_axis!(point_in,cstheta,axis,point_out;
        diff_v=zeros(Float64,2),cos_sin=zeros(Float64,2),cos_sin_sum=zeros(Float64,2))
    diff_v .= point_in .- axis
    if((diff_v[1]==0) & (diff_v[2]==0))
        return point_in[1],point_in[2]
    end

    rad=sqrt(sum(diff_v.^2))

    #cos_sin of difference vector
    cos_sin .= diff_v ./rad

    #sum of angles cos(A+B) and sin (A+B) rules
    cos_sin_sum .= [cos_sin[1]*cstheta[1] - cos_sin[2]*cstheta[2],
                 cos_sin[1]*cstheta[2] + cos_sin[2]*cstheta[1]  ]

    #rotation and translation
    point_out[1]=axis[1]+(rad*cos_sin_sum[1])
    point_out[2]=axis[2]+(rad*cos_sin_sum[2])
end

"""moves the positioner to a theta phi position
Adapted from desihub/fiberassign
diff_v: optional to use pre-allocated array for intermediate step for speed
cos_sin: optional to use pre-allocated array for intermediate step for speed
cos_sin_sum: optional to use pre-allocated array for intermediate step for speed
csthetha: optional to use pre-allocated array for intermediate step for speed
csphi: optional to use pre-allocated array for intermediate step for speed
"""
function move_positioner_to_theta_phi!(pos::POSRobotBody,theta,phi;
    diff_v=zeros(Float64,2),cos_sin=zeros(Float64,2),cos_sin_sum=zeros(Float64,2),
    cstheta=zeros(Float64,2),csphi=zeros(Float64,2))

    cstheta .= [cos(theta),sin(theta)]
    csphi .= [cos(phi),sin(phi)]

    #// Move the phi polygon into the fully extended position along the X axis.
    #moving to LENGHT_R1,0
    nT=pos.nseg[1]
    nP=pos.nseg[2]

    poly_translate!(view(pos.head_orig,1:nP,:),view(pos.head_on_target,1:nP,:),pos.arms[1],0)
    #translate the axis
    point_translate!(pos.axis_orig,pos.axis_on_target,pos.arms[1],0)

    #@show pos.head_on_target[1:pos.nseg[2],2] .== (pos.head_orig[1:pos.nseg[2],2] .+ 0)
    #@show "1: ",pos.axis_on_target,pos.arms[1]

    #// Rotate fully extended positioner an angle of theta about the origin.
    poly_rotate_axis!(view(pos.body_orig,1:nT,:),view(pos.body_on_target,1:nT,:),cstheta,[0.0,0.0],
        diff_v=diff_v,cos_sin=cos_sin,cos_sin_sum=cos_sin_sum)
    poly_rotate_axis!(view(pos.head_on_target,1:nP,:),view(pos.head_on_target,1:nP,:),cstheta,[0.0,0.0],
        diff_v=diff_v,cos_sin=cos_sin,cos_sin_sum=cos_sin_sum)
    #rotate the axis
    point_rotate_axis!(pos.axis_on_target,cstheta,[0.0,0.0],pos.axis_on_target,
        diff_v=diff_v,cos_sin=cos_sin,cos_sin_sum=cos_sin_sum)

    #@show "2: ",pos.axis_on_target,cstheta

    #// Rotate just the phi arm an angle phi about the theta arm center.
    #shpphi.rotation(csphi);
    poly_rotate_axis!(view(pos.head_on_target,1:nP,:),view(pos.head_on_target,1:nP,:),csphi,pos.axis_on_target,
        diff_v=diff_v,cos_sin=cos_sin,cos_sin_sum=cos_sin_sum)

    #// Translate the whole positioner to the center.
    poly_translate!(view(pos.head_on_target,1:nP,:),view(pos.head_on_target,1:nP,:),pos.xy[1],pos.xy[2])
    poly_translate!(view(pos.body_on_target,1:nT,:),view(pos.body_on_target,1:nT,:),pos.xy[1],pos.xy[2])

    #Not needed just for plotting
    #point_translate!(pos.axis_on_target,pos.axis_on_target,pos.xy[1],pos.xy[2])
end

"""Determines if a positioner is valid of not
fp_dic : dictionary contaning the status of focal plane
pp: index of positioner considered
pos_type: purpose of validity testing
           assignment: whether this positioner can be assigned to a target
           neighbour_collision: whether this positioner should be considered for neighbour collision
"""
function valid_positionter(fp_dic,pp;pos_type="assignment")
    if(pos_type=="assignment")
        return ((fp_dic["DEVICE_TYPE"][pp]=="POS") & (fp_dic["LENGTH_R1"][pp]!=0.0))
    elseif(pos_type=="neighbour_collision")
        return ((fp_dic["DEVICE_TYPE"][pp]=="POS") & (fp_dic["LENGTH_R1"][pp]!=0.0))
    end
end

"""
loads the positioner negibours given pindex and zones of the central positioner
pos_ngb_dic: a dictionary of structure to hold the negibours
"""
function load_positioner_neighbours!(pos_cur,pindex,fp_bound,fp_dic,exc_dic,pos_ngb_dic)
    #This zone is already loaded so nothing to be done
    if(pos_ngb_dic["index_central_positioner"]==pindex)
        #already this positioner neighbours are loaded
        return
    else
        zone_x=fp_dic["zone_x"][pindex]
        zone_y=fp_dic["zone_y"][pindex]
        zone_xy=get_zonexy(zone_x,zone_y,fp_bound)
        pos_ngb_dic["index_central_positioner"]=pindex
    end

    #@show zone_x,zone_y,zone_xy
    count_pos=1
    for zx in maximum([zone_x-1,1]):minimum([zone_x+1,fp_bound.nx])
        for zy in maximum([zone_y-1,1]):minimum([zone_y+1,fp_bound.ny])
            zone_xy=get_zonexy(zx,zy,fp_bound)

            #@show zx,zy,zone_xy

            if(!(zone_xy in fp_dic["zone_map_keys"]))
                continue
            end
            imin=fp_dic["zone_map"][zone_xy][1]
            imax=fp_dic["zone_map"][zone_xy][2]
            #@show zx,zy,zone_xy,targets_dic["zone_map"][zone_xy]
            #@show targets_dic["zone_x"][imin:imax]
            count_this=imax-imin
            pos_ngb_dic["index"][count_pos:count_pos+count_this] .= collect(imin:imax)
            count_pos=count_pos+count_this+1
        end
    end

    #set everything else to 0
    if(count_pos<=size(pos_ngb_dic["index"],1))
        pos_ngb_dic["index"][count_pos:end] .= 0
    end
    #Due to Julia indexing convetion 1 need to be subtracte
    count_pos=count_pos-1

    #Determine the maximum distance to look for within positioner
    #This is defined as the maximum arams length + the maximum vertex of head
    #The above is conservative and specific choice will not matter
    lookup_rad2=(pos_cur.arms[1]+pos_cur.arms[2]+pos_cur.head_minmax[1])^2
    lookup_rad2=4*lookup_rad2 #factor of 4 is for twice quare the distance
    #@show lookup_rad2,pos_cur.arms[1],pos_cur.arms[2],pos_cur.head_minmax[2]

    #scan for positioners within look_up radius
    count_ngb=0
    for ii in pos_ngb_dic["index"][1:count_pos]
        if((ii==pindex) || (!valid_positionter(fp_dic,ii;pos_type="neighbour_collision")))#To not count self positioner and invalid positioner
            continue
        end
        rad2=(fp_dic["OFFSET_X"][ii]-fp_dic["OFFSET_X"][pindex])^2
        rad2 += (fp_dic["OFFSET_Y"][ii]-fp_dic["OFFSET_Y"][pindex])^2
        #@show ii,rad2,lookup_rad2,rad2<lookup_rad2

        if(rad2<lookup_rad2)
            count_ngb+=1
            #load this positioner info
            load_positioner!(pos_ngb_dic["pos"][count_ngb],ii,fp_dic,exc_dic)
            pos_ngb_dic["pos_index"][count_ngb]=ii
        end
    end
    pos_ngb_dic["count_ngb"]=count_ngb
    #@show count_ngb

    if(false)#To make plots
        tind=pos_ngb_dic["index"][1:count_pos]
        plot([fp_dic["OFFSET_X"][pindex]],[fp_dic["OFFSET_Y"][pindex]],
            seriestype=:scatter,color=:green,markersize=10)
        for ii in 1:count_ngb
            xy=pos_ngb_dic["pos"][ii].xy
            plot!([xy[1]],[xy[2]],seriestype=:scatter,color=:red,markersize=10)
        end

        plot!(fp_dic["OFFSET_X"][tind],fp_dic["OFFSET_Y"][tind],seriestype=:scatter,color=:black) |> display
    end

end

"""Compare the moved positioner positioner to check if either the body or head-boy is collided
body-body collision is not considered as it is not possible in the focal plane
pos1: first positioner of type POSRobotBody
pos2: second positioner of type POSRobotBody
"""
function positioner_collided(pos1,pos2)

    head1=view(pos1.head_on_target,1:pos1.nseg[2],:)
    head2=view(pos2.head_on_target,1:pos2.nseg[2],:)

    #check for the head1-head2 collision
    if(PolyIntersect_boundingbox(head1,head2))
        if(PolygonsIntersect(head1,head2))
            return true
        end
    end

    body1=view(pos1.body_on_target,1:pos1.nseg[1],:)
    body2=view(pos2.body_on_target,1:pos2.nseg[1],:)

    #head1-body2 collision
    if(PolyIntersect_boundingbox(head1,body2))
        if(PolygonsIntersect(head1,body2))
            return true
        end
    end

    #head2-body1 collision
    if(PolyIntersect_boundingbox(head2,body1))
        if(PolygonsIntersect(head2,body1))
            return true
        end
    end

    return false
end


macro enforce(expr) # strong @assert
    msg = string(expr)
    return :($(esc(expr)) ? $(nothing) : throw(AssertionError($msg)))
end

function _precompile_()
    ccall(:jl_generating_output, Cint, ()) == 1 || return nothing

    @enforce Base.precompile(Tuple{Type{PeriodicGraph},String})
    @enforce Base.precompile(Tuple{typeof(PeriodicGraphs.extended_gcd),Vector{Int}})

    for i in 0:3
        @enforce Base.precompile(Tuple{Type{PeriodicEdge{i}},Int,Int,NTuple{i, Int}})
        @enforce Base.precompile(Tuple{Type{PeriodicVertex{i}},Int})

        @enforce Base.precompile(Tuple{Type{PeriodicEdge},Int,Int,NTuple{i, Int}})
        @enforce Base.precompile(Tuple{Type{PeriodicEdge},Int,Int,SVector{i, Int}})
        @enforce Base.precompile(Tuple{Type{PeriodicEdge},Int,PeriodicVertex{i}})
        @enforce Base.precompile(Tuple{Type{PeriodicEdge},Tuple{Int,Int,NTuple{i, Int}}})

        @enforce Base.precompile(Tuple{Type{PeriodicGraph{i}}})
        @enforce Base.precompile(Tuple{Type{PeriodicGraph{i}},Vector{PeriodicEdge{i}}})
        @enforce Base.precompile(Tuple{Type{PeriodicGraph{i}},String})
        @enforce Base.precompile(Tuple{Type{PeriodicGraph},PeriodicGraph{i}})

        @enforce Base.precompile(Tuple{typeof(==),PeriodicVertex{i},PeriodicVertex{i}})
        @enforce Base.precompile(Tuple{typeof(<),PeriodicVertex{i},PeriodicVertex{i}})
        @enforce Base.precompile(Tuple{typeof(isless),PeriodicVertex{i},PeriodicVertex{i}})
        @enforce Base.precompile(Tuple{typeof(==),PeriodicEdge{i},PeriodicEdge{i}})
        @enforce Base.precompile(Tuple{typeof(<),PeriodicEdge{i},PeriodicEdge{i}})
        @enforce Base.precompile(Tuple{typeof(==),PeriodicGraph{i},PeriodicGraph{i}})

        @enforce Base.precompile(Tuple{typeof(Graphs._neighborhood),PeriodicGraph{i},Int,Int,Graphs.DefaultDistance,Function})
        @enforce Base.precompile(Tuple{typeof(Graphs.add_edge!),PeriodicGraph{i},PeriodicEdge{i}})
        @enforce Base.precompile(Tuple{typeof(add_vertex!),PeriodicGraph{i}})
        @enforce Base.precompile(Tuple{typeof(Graphs.add_vertices!),PeriodicGraph{i},Int})
        @enforce Base.precompile(Tuple{typeof(Graphs.rem_edge!),PeriodicGraph{i},PeriodicEdge{i}})
        @enforce Base.precompile(Tuple{typeof(Graphs.rem_vertex!),PeriodicGraph{i},Int})
        @enforce Base.precompile(Tuple{typeof(collect),PeriodicGraphs.PeriodicEdgeIter{i}})
        @enforce Base.precompile(Tuple{typeof(Graphs.connected_components),PeriodicGraph{i}})
        @enforce Base.precompile(Tuple{typeof(coordination_sequence),PeriodicGraph{i},Int,Int})
        @enforce Base.precompile(Tuple{typeof(dimensionality),PeriodicGraph{i}})
        @enforce Base.precompile(Tuple{typeof(find_edges),PeriodicGraph{i},Int,Int})
        @enforce Base.precompile(Tuple{typeof(Graphs.has_edge),PeriodicGraph{i},Int,Int})
        @enforce Base.precompile(Tuple{typeof(Graphs.has_edge),PeriodicGraph{i},PeriodicEdge{i}})
        @enforce Base.precompile(Tuple{typeof(Graphs.induced_subgraph),PeriodicGraph{i},Vector{Int}})
        @enforce Base.precompile(Tuple{typeof(isempty),PeriodicGraphs.PeriodicEdgeIter{i}})
        @enforce Base.precompile(Tuple{typeof(iterate),PeriodicEdgeIter{i}})
        @enforce Base.precompile(Tuple{typeof(Graphs.neighborhood),PeriodicGraph{i},Int,Int})
        @enforce Base.precompile(Tuple{typeof(offset_representatives!),PeriodicGraph{i},Vector{Any}})
        @enforce Base.precompile(Tuple{typeof(offset_representatives!),PeriodicGraph{i},Vector{Int}})
        @enforce Base.precompile(Tuple{typeof(offset_representatives!),PeriodicGraph{i},Vector{NTuple{i,Int}}})
        @enforce Base.precompile(Tuple{typeof(Graphs.reverse),PeriodicEdge{i}})
        @enforce Base.precompile(Tuple{typeof(Base.string),PeriodicEdge{i}})
        @enforce Base.precompile(Tuple{typeof(Base.string),PeriodicGraph{i}})
        @enforce Base.precompile(Tuple{typeof(Base.string),PeriodicVertex{i}})
        @enforce Base.precompile(Tuple{typeof(swap_axes!),PeriodicGraph{i},Vector{Int}})
        @enforce Base.precompile(Tuple{typeof(swap_axes!),PeriodicGraph{i},NTuple{i,Int}})
        @enforce Base.precompile(Tuple{typeof(vertex_permutation),PeriodicGraph{i},Vector{Any}})
        @enforce Base.precompile(Tuple{typeof(periodiccellgraph),PeriodicGraph{i}})
        @enforce Base.precompile(Tuple{typeof(PeriodicGraphs.hash_position),SVector{i,Int}})

        for j in 0:3
            @enforce Base.precompile(Tuple{typeof(PeriodicGraphs.change_dimension),Type{PeriodicGraph{i}},PeriodicGraph{j}})
        end
    end
end

_precompile_()


__precompile__()

module CausalTrees
import Statistics
import PyPlot
import Plots

include("tree.jl")
include("loss.jl")
include("utils.jl")
include("clearning.jl")
include("plearning.jl")
include("plot.jl")

export causal_tree
export causal_forest
export predict
export mse_tau
export decision_tree
export random_forest
export plot

end # module


#!TODO: Yet to write more tests here

#=
Disclaimer: Many of the tests in this file taken from:
    https://github.com/tpapp/DynamicHMC.jl/blob/master/test/test-NUTS.jl
=#

############################################################################################
for iter in eachindex(objectives)
    _obj = objectives[iter]
    _flattentype = _obj.model.info.reconstruct.default.output
    @testset "Kernel construction and propagation, all models" begin
        mcmcdefault = MCMCDefault(;
        stepsize = ConfigStepsize(;
            stepsizeadaption=UpdateFalse()),
            GradientBackend = backends[1],
            generated = UpdateTrue()
        )
        mcmckernel = MCMC(
            _rng,
            kernel,
            _obj,
            mcmcdefault
        )
    end
end

############################################################################################
@testset "random booleans" begin
    @test rand_bool_logprob(_rng, 0)
    @test !rand_bool_logprob(_rng, log(0))
end


include("mpii.annotation.reader.jl")
include("../helper.jl")
include("../models.jl")
include("loss.jl")
include("../utils.jl")
include("variant.data.jl")
using Images, FileIO, ImageDraw
using Knet

function run_visualizer(
    model_location_name = "29-12-20-training_model",
    Knet_data_location_name = "29-12-20-dval";
    visualization_idxs = 1:10,
    pred_threshold = 0.2,
    focus_on_argmax = false,
    add_loc_ref_offset = false,
    show_only_predictions = false,
    show_body_skeleton = false,
)
    model = Knet.load("$(model_name).jld2", model_name)
    dval = Knet.load("$(Knet_data_location_name).jld2", Knet_data_location_name)

    visualize_part_detection_results(
        model,
        data;
        pred_threshold = pred_threshold,
        focus_on_argmax = focus_on_argmax,
        add_loc_ref_offset = add_loc_ref_offset,
        show_only_predictions = show_only_predictions,
        visualization_idxs = visualization_idxs,
        show_body_skeleton = show_body_skeleton,
    )
end

# TODO: Can we add heatmap like visualization
function visualize_part_detection_results(
    model,
    data;
    visualization_idxs = 1:10,
    pred_threshold = 0.2,
    focus_on_argmax = false,
    add_loc_ref_offset = false,
    show_only_predictions = false,
    show_body_skeleton = false,
    should_color_scmap_for_gt = true,
)

    if isa(data, VariantData)

        for (i, (x, y_gold)) in enumerate(data)
            output = Array{Float32}(model(x))
            img = reduce_dim(Array(x))

            # TODO: you can make below seperate functions
            if !show_only_predictions
                gt_scmap = Array(y_gold)
                gt_images = show_scmap_on_image(
                    img,
                    gt_scmap;
                    confidence_threshold = 0.5,
                    should_use_scmap_size = false,
                    return_single_image = show_body_skeleton,
                    should_color_scmap = should_color_scmap_for_gt,
                )

                if (show_body_skeleton)
                    gt_images = draw_body_skeleton(gt_images, gt_scmap)
                end

                println("****** GROUND TRUTH ******")
                show_body_skeleton ? display(gt_images) : mosaicify(gt_images)
            end

            pred_scmap = output[:, :, :, 1]
            pred_images = show_scmap_on_image(
                img,
                pred_scmap;
                confidence_threshold = pred_threshold,
                should_use_scmap_size = false,
                focus_on_argmax = focus_on_argmax,
                add_loc_ref_offset = add_loc_ref_offset,
                return_single_image = show_body_skeleton,
            )

            if (show_body_skeleton)
                pred_images = draw_body_skeleton(pred_images, pred_scmap)
            end

            println("****** PART PREDICTIONS ******")
            show_body_skeleton ? display(pred_images) : mosaicify(pred_images)
        end
    else
        data_x = Knet.atype()(reshape(data.x[:, visualization_idxs], data.xsize))
        data_y = Knet.atype()(reshape(data.y[:, visualization_idxs], data.ysize))
        output = model(data_x)
        output = Array(output)
        for idx = 1:length(visualization_idxs)
            img = Array(data_x[:, :, :, idx])
            if !show_only_predictions
                gt_scmap = Array(data_y[:, :, :, idx])
                gt_images = show_scmap_on_image(
                    img,
                    gt_scmap;
                    confidence_threshold = 0.5,
                    should_use_scmap_size = false,
                    return_single_image = show_body_skeleton,
                    should_color_scmap = should_color_scmap_for_gt,
                )

                if (show_body_skeleton)
                    gt_images = draw_body_skeleton(gt_images, gt_scmap)
                end

                println("****** GROUND TRUTH ******")
                show_body_skeleton ? display(gt_images) : mosaicify(gt_images)
            end
            pred_scmap = output[:, :, :, idx]
            pred_images = show_scmap_on_image(
                img,
                pred_scmap;
                confidence_threshold = pred_threshold,
                should_use_scmap_size = false,
                focus_on_argmax = focus_on_argmax,
                add_loc_ref_offset = add_loc_ref_offset,
                return_single_image = show_body_skeleton,
            )

            if (show_body_skeleton)
                pred_images = draw_body_skeleton(pred_images, pred_scmap)
            end

            println("****** PART PREDICTIONS ******")
            show_body_skeleton ? display(pred_images) : mosaicify(pred_images)
        end

    end
end

function draw_body_skeleton(img, scmap)
    colors = [
        RGB{Float32}(0.9, 0.1, 0.1),
        RGB{Float32}(0.1, 0.9, 0.1),
        RGB{Float32}(0.1, 0.1, 0.9),
        RGB{Float32}(0.4, 0.1, 0.1),
        RGB{Float32}(0.1, 0.4, 0.1),
        RGB{Float32}(0.1, 0.1, 0.4),
        RGB{Float32}(0.2, 0.6, 0.2),
    ]

    all_joints = [
        [1, 2],
        [2, 3],
        [3, 4],
        [4, 5],
        [5, 6],
        [7, 8],
        [8, 9],
        [9, 10],
        [3, 9],
        [4, 10],
        [10, 11],
        [11, 12],
        [13, 14],
    ]

    for (i, (joint_group)) in enumerate(all_joints)
        color_idx = i % length(colors) + 1
        color = colors[color_idx]
        joint_1 = joint_group[1]
        joint_2 = joint_group[2]
        pred_f8_1 = fetch_loc_reffed_max_point(scmap, joint_1)
        pred_f8_2 = fetch_loc_reffed_max_point(scmap, joint_2)
        img = draw!(img, LineSegment(pred_f8_1, pred_f8_2), color)
    end
    return img
end


using Images, AxisArrays
using Base: OneTo
using Unitful: Time, Length, s

# TODO: Make this be generic over type of SIM acquisition like so
# SIMAcquisition{T} where T <: AbstractSIMType <19-05-21, kunzaatko> #
struct SIMAcquisition <: AbstractAcquisition # {{{
    "Acquired reconstructed images"
    reconstructedimages::AxisArray{<:AbstractGray,3}
    "Acquired unreconstructed images stored with there acquisition times and orientations"
    unreconstructedimages::AxisArray{<:AbstractGray,3}
    "Numerical aperature"
    NA::Real
    "Magnification"
    M::Unsigned
    "Fluorophore marker"
    marker::FluorophoreMarker
    "Pixel size"
    pixelsize::Length
end # }}}

function SIMAcquisition( # {{{
    reconstructedimages::AbstractArray{<:Color,3},
    unreconstructedimages::AbstractArray{<:Color,3},
    acquisitiontimes::AbstractVector{<:Time},
    NA,
    M,
    marker::FluorophoreMarker,
    pixelsize::Length,
)
    if !(eltype(reconstructedimages) <: AbstractGray)
        reconstructedimages = Gray.(reconstructedimages)
    end

    if !(eltype(unreconstructedimages) <: AbstractGray)
        unreconstructedimages = Gray.(unreconstructedimages)
    end

    # FIX: Add check for acquisition times having the same size <19-05-21, kunzaatko> #
    # TODO: This should be generic over AbstractSIMType <19-05-21, kunzaatko> #
    # TODO: x and y axes should be indexed by length instead of index <19-05-21, kunzaatko> #
    unreconstructedimages_axis = AxisArray(
        unreconstructedimages,
        Axis{:x}(OneTo(size(unreconstructedimages, 1))),
        Axis{:y}(OneTo(size(unreconstructedimages, 2))),
        Axis{:time}(acquisitiontimes),
    )

    # TODO: This should make an average for the times of acquisition for the unreconstructed sequence that soruced this <19-05-21, kunzaatko> #
    # TODO: x and y axes should be indexed by length instead of index <19-05-21, kunzaatko> #
    reconstructedimages_axis = AxisArray(reconstructedimages, :x, :y, :time)

    return SIMAcquisition(
        reconstructedimages_axis, unreconstructedimages_axis, NA, M, marker, pixelsize
    )
end # }}}

function SIMAcquisition( # {{{
    reconstructedimages::AbstractArray{<:Color,3},
    unreconstructedimages::AbstractArray{<:Color,3},
    acquisitiontimestep::Time,
    NA,
    M,
    marker::FluorophoreMarker,
    pixelsize::Length,
)
    return SIMAcquisition(
        reconstructedimages,
        unreconstructedimages,
        range(0s; length=size(unreconstructedimages, 3), step=acquisitiontimestep),
        NA,
        M,
        marker,
        pixelsize,
    )
end # }}}

function times(acquisition::SIMAcquisition; images=:unreconstructedimages) # {{{
    if images == :unreconstructedimages
        return acquisition.unreconstructedimages[Axis{:time}].val
    elseif images == :reconstructedimages
        return acquisition.reconstructedimages[Axis{:time}].val
    else
        # TODO: error handling when not reconstructed or unreconstructed <20-05-21, kunzaatko> #
        return error("Must be reconstructed or unreconstructed")
    end
end # }}}


using Colors, FixedPointNumbers, JLD2
using Test
using ColorTypes: eltype_default

@testset "Conversion" begin
    r8(x) = reinterpret(N0f8, x)

    # Promotions
    a, b = promote(RGB(1,0,0), Gray(0.8))
    @test isa(a, RGB{Float64}) && isa(b, RGB{Float64})
    a, b = promote(RGBA(1,0,0), Gray(0.8))
    @test isa(a, RGBA{Float64}) && isa(b, RGBA{Float64})
    a, b = promote(RGBA(1,0,0), GrayA(0.8))
    @test isa(a, RGBA{Float64}) && isa(b, RGBA{Float64})
    a, b = promote(RGB(1,0,0), GrayA(0.8))
    @test isa(a, RGBA{Float64}) && isa(b, RGBA{Float64})
    a, b = promote(RGB(1,0,0), AGray(0.8))
    @test isa(a, ARGB{Float64}) && isa(b, ARGB{Float64})

    # Color parsing
    redN0f8 = parse(Colorant, "red")
    @test colorant"red" == redN0f8
    @test isa(redN0f8, RGB{N0f8})
    @test redN0f8 == RGB(1,0,0)
    redF64 = convert(RGB{Float64}, redN0f8)
    @test parse(RGB{Float64}, "red") === RGB{Float64}(1,0,0)
    @test isa(parse(HSV, "blue"), HSV)
    @test parse(Colorant, "rgb(55,217,127)") === RGB{N0f8}(r8(0x37),r8(0xd9),r8(0x7f))
    @test colorant"rgb(55,217,127)" === RGB{N0f8}(r8(0x37),r8(0xd9),r8(0x7f))
    @test parse(Colorant, "rgba(55,217,127,0.5)") === RGBA{N0f8}(r8(0x37),r8(0xd9),r8(0x7f),0.5)
    @test parse(Colorant, "rgb(55,217,127)") === RGB{N0f8}(r8(0x37),r8(0xd9),r8(0x7f))
    @test parse(Colorant, "rgba(55,217,127,0.5)") === RGBA{N0f8}(r8(0x37),r8(0xd9),r8(0x7f),0.5)
    @test parse(Colorant, "hsl(120, 100%, 50%)") === HSL{Float32}(120,1.0,.5)
    @test colorant"hsl(120, 100%, 50%)" === HSL{Float32}(120,1.0,.5)
    @test parse(RGB{N0f8}, "hsl(120, 100%, 50%)") === convert(RGB{N0f8}, HSL{Float32}(120,1.0,.5))
    @test_throws ErrorException  parse(Colorant, "hsl(120, 100, 50)")
    @test parse(Colorant, "#D0FF58") === RGB(r8(0xD0),r8(0xFF),r8(0x58))
    @test parse(Colorant, "#FB0") === RGB(r8(0xFF),r8(0xBB),r8(0x00))

    @test parse(Colorant, :red) === colorant"red"
    @test parse(Colorant, colorant"red") === colorant"red"

    @test hex(RGB(1,0.5,0)) == "FF8000"
    @test hex(RGBA(1,0.5,0,0.25)) == "40FF8000"

    # srgb_compand / invert_srgb_compand
    @test Colors.srgb_compand(0.5) ≈ 0.7353569830524494 atol=eps()
    @test Colors.invert_srgb_compand(0.7353569830524494) ≈ 0.5 atol=eps()
    # issue #351
    l_pow_x_y() = for i=1:1000; (i/1000)^(1/2.4) end
    l_exp_y_log_x() = for i=1:1000; exp(1/2.4*log(i/1000)) end
    l_pow_x_y(); t_pow_x_y = @elapsed l_pow_x_y()
    l_exp_y_log_x(); t_exp_y_log_x = @elapsed l_exp_y_log_x()
    if t_exp_y_log_x > t_pow_x_y
        @warn "Optimization in `[invert_]srgb_compand()` may have the opposite effect."
    end

    fractional_types = (RGB, BGR, RGB1, RGB4)  # types that support Fractional

    red24 = reinterpret(RGB24, 0x00ff0000)
    red32 = reinterpret(ARGB32, 0xffff0000)
    for T in (Float64, Float32, N0f8)
        c = RGB(one(T), zero(T), zero(T))
        @test eltype(c) == T
        c64 = convert(RGB{Float64}, c)
        @test typeof(c64) == RGB{Float64}
        @test c64 == redF64
        cr = convert(RGB{T}, redF64)
        @test cr == c
    end
    @test RGB(1,0,0) == redF64
    @test RGB(convert(UInt8, 1),0,0) == redF64
    @test RGB(convert(UInt8, 1),convert(UInt8, 0),convert(UInt8, 0)) == redF64
    @test convert(RGB, red24) == redF64

    @test convert(Gray{N0f8}, Gray{N0f8}(0.1)) == Gray{N0f8}(0.1)
    @test convert(Gray{N0f8}, Gray(0.1))     == Gray{N0f8}(0.1)
    @test convert(Gray{N0f8}, Gray24(0.1))   == Gray{N0f8}(0.1)
    @test convert(Gray24, Gray{N0f8}(0.1))   == Gray24(0.1)

    @test convert(RGB{N0f8}, Gray{N0f8}(0.1)) == RGB{N0f8}(0.1,0.1,0.1)
    @test convert(RGB{N0f8}, Gray24(0.1))   == RGB{N0f8}(0.1,0.1,0.1)

    for Cto in ColorTypes.parametric3
        for Cfrom in ColorTypes.parametric3
            for Tto in (Float32, Float64)
                for Tfrom in (Float32, Float64)
                    c = convert(Cfrom{Tfrom}, redF64)
                    @test typeof(c) == Cfrom{Tfrom}
                    c1 = convert(Cto, c)
                    @test eltype(c1) == Tfrom
                    c2 = convert(Cto{Tto}, c)
                    @test typeof(c2) == Cto{Tto}
                end
            end
        end
    end
    for Cto in ColorTypes.parametric3
        @test typeof(convert(Cto, red24)) == Cto{eltype_default(Cto)}
        @test typeof(convert(Cto{Float64}, red24)) == Cto{Float64}
    end

    # Test conversion from Normed types
    for Cto in ColorTypes.parametric3
        for Cfrom in fractional_types
            for Tto in (Float32, Float64)
                for Tfrom in (N0f8, N6f10, N4f12, N2f14, N0f16)
                    c = convert(Cfrom{Tfrom}, redF64)
                    @test typeof(c) == Cfrom{Tfrom}
                    if !(eltype_default(Cto) <: FixedPoint)
                        c1 = convert(Cto, c)
                        @test eltype(c1) == eltype_default(Cto)
                    end
                    c2 = convert(Cto{Tto}, c)
                    @test typeof(c2) == Cto{Tto}
                end
            end
        end
    end

    # Test conversion to Normed types
    for Cto in fractional_types
        for Cfrom in ColorTypes.parametric3
            for Tto in (N0f8, N6f10, N4f12, N2f14, N0f16)
                for Tfrom in (Float32, Float64)
                    c = convert(Cfrom{Tfrom}, redF64)
                    @test typeof(c) == Cfrom{Tfrom}
                    c2 = convert(Cto{Tto}, c)
                    @test typeof(c2) == Cto{Tto}
                end
            end
        end
    end

    ac = RGBA(redF64)

    @test convert(RGB, ac) == RGB(1,0,0)
    @test convert(RGB{N0f8}, ac) == RGB{N0f8}(1,0,0)
    @test convert(RGBA{N0f8}, ac) == RGBA{N0f8}(1,0,0,1)
    @test convert(HSVA, ac) == HSVA{Float64}(convert(HSV, redF64), 1.0)
    @test convert(HSVA{Float32}, ac) == HSVA{Float32}(convert(HSV{Float32}, redF64), 1.0f0)
    @test convert(RGBA, redF64) == ac

    @test convert(ARGB32, ac) == reinterpret(ARGB32, 0xffff0000)
    @test convert(ARGB32, RGB(1,0,0)) == reinterpret(ARGB32, 0xffff0000)
    ac32 = convert(ARGB32, RGB(1,0,0), 0.5)
    @test ac32 == reinterpret(ARGB32, 0x80ff0000)
    @test convert(ARGB32, ac32) === ac32
    @test convert(ARGB32, BGRA(1,0,0,0.5)) == reinterpret(ARGB32, 0x80ff0000)
    @test reinterpret(UInt32, convert(ARGB32, ac)) == 0xffff0000
    @test convert(RGB24, RGB(r8(0xff),r8(0x00),r8(0x00))) == reinterpret(RGB24, 0x00ff0000)
    @test reinterpret(UInt32, convert(RGB24, RGB(r8(0xff),r8(0x00),r8(0x00)))) == 0x00ff0000
    redhsv = convert(HSV, redF64)
    @test convert(RGB24, red24) === red24
    @test convert(RGB24, redhsv) == reinterpret(RGB24, 0x00ff0000)
    @test_throws ArgumentError convert(RGB24,  RGB(0, 1.1, 0))
    @test_throws ArgumentError convert(ARGB32, RGBA(0, 1.1, 0, 0.8))
    @test_throws ArgumentError convert(ARGB32, RGBA(0, 0.8, 0, 1.1))

    @test convert(RGB{N0f8}, red24) == RGB{N0f8}(1,0,0)
    @test convert(RGBA{N0f8}, red32) == RGBA{N0f8}(1,0,0,1)
    @test convert(HSVA{Float64}, red32) == HSVA{Float64}(360, 1, 1, 1)

    @test_throws MethodError AlphaColor(RGB(1,0,0), r8(0xff))

    # whitepoint conversions
    @test isa(convert(XYZ, convert(Lab, redF64), Colors.WP_DEFAULT), XYZ{Float64})
    @test isa(convert(XYZ{Float32}, convert(Lab, redF64), Colors.WP_DEFAULT), XYZ{Float32})
    @test isa(convert(XYZ, convert(Luv, redF64), Colors.WP_DEFAULT), XYZ{Float64})
    @test isa(convert(XYZ{Float32}, convert(Luv, redF64), Colors.WP_DEFAULT), XYZ{Float32})
    @test isa(convert(Lab, convert(XYZ, redF64), Colors.WP_DEFAULT), Lab{Float64})
    @test isa(convert(Lab{Float32}, convert(XYZ, redF64), Colors.WP_DEFAULT), Lab{Float32})
    @test isa(convert(Luv, convert(XYZ, redF64), Colors.WP_DEFAULT), Luv{Float64})
    @test isa(convert(Luv{Float32}, convert(XYZ, redF64), Colors.WP_DEFAULT), Luv{Float32})

    # Test vector space operations
    @test LMS{Float64}(0.125,0.5,0.0)+LMS{Float64}(0.2,0.7,0.4) ≈ LMS{Float64}(0.325,1.2,0.4) atol=91eps()
    @test 3LMS{Float64}(0.125,0.5,0.03) ≈ LMS{Float64}(0.375,1.5,0.09) atol=91eps()

    @test XYZ{Float64}(0.125,0.5,0.0)+XYZ{Float64}(0.2,0.7,0.4) ≈ XYZ{Float64}(0.325,1.2,0.4) atol=91eps()
    @test 3XYZ{Float64}(0.125,0.5,0.03) ≈ XYZ{Float64}(0.375,1.5,0.09) atol=91eps()

    #59
    @test Colors.xyz_to_uv(XYZ{Float64}(0.0, 0.0, 0.0)) === (0.0, 0.0)
    @test Colors.xyz_to_uv(XYZ{Float64}(0.0, 1.0, 0.0)) === (0.0, 0.6)
    @test Colors.xyz_to_uv(XYZ{Float64}(0.0, 1.0, 1.0)) === (0.0, 0.5)
    @test Colors.xyz_to_uv(XYZ{Float64}(1.0, 0.0, 1.0)) === (1.0, 0.0)
    @test Colors.xyz_to_uv(XYZ{Float64}(1.0, 0.0, 0.0)) === (4.0, 0.0)

    # ColorTypes.jl issue #40
    @test convert(HSL, RGB{N0f8}(0.678, 0.847, 0.902)) ≈ HSL{Float32}(194.73685f0,0.5327105f0,0.7901961f0) atol=100eps(Float32)

    # YIQ
    @test convert(YIQ, RGB(1,0,0)) == YIQ{Float32}(0.299, 0.595716, 0.211456)
    @test convert(YIQ, RGB(0,1,0)) == YIQ{Float32}(0.587, -0.274453, -0.522591)
    @test convert(YIQ, RGB(0,0,1)) == YIQ{Float32}(0.114, -0.321263, 0.311135)
    @test convert(RGB, YIQ(1.0,0.0,0.0)) == RGB(1,1,1)
    v = 0.5957
    @test convert(RGB, YIQ(0.0,1.0,0.0)) == RGB(0.9563*v,0,0)
    v = -0.5226
    @test convert(RGB, YIQ(0.0,0.0,-1.0)) == RGB(0,-0.6474*v,0)

    # Gray
    c = Gray{N0f16}(0.8)
    @test convert(RGB, c) == RGB{N0f16}(0.8,0.8,0.8)
    @test convert(RGB{Float32}, c) == RGB{Float32}(0.8,0.8,0.8)

    for C in ColorTypes.parametric3
        c = convert(C, RGB(1,1,1))
        @test gray(convert(Gray, c)) ≈ 1 atol=0.01
        @test gray(convert(Gray{Float64}, c)) ≈ 1 atol=0.01
    end

    # More AbstractRGB
    r4 = RGB4(1,0,0)
    @test convert(RGB, r4) == RGB(1,0,0)
    @test convert(RGB{N0f8}, r4) == RGB{N0f8}(1,0,0)
    @test convert(RGB4{N0f8}, r4) == RGB4{N0f8}(1,0,0)
    @test convert(RGB4{Float32}, r4) == RGB4{Float32}(1,0,0)
    @test convert(BGR{Float32}, r4) == BGR{Float32}(1,0,0)

    # Test accuracy of conversion
    csconv = jldopen(joinpath(dirname(@__FILE__), "test_conversions.jld2")) do file
        read(file, "csconv")
    end

    function convcompare(from, to, eps; showfailure::Bool=false)
        errmax = 0.0
        for i = 1:length(from)
            t = to[i]
            f = convert(typeof(t), from[i])
            diff = abs(comp1(t)-comp1(f)) + abs(comp2(t)-comp2(f)) + abs(comp3(t)-comp3(f))
            mag = abs(comp1(t)+comp1(f)) + abs(comp2(t)+comp2(f)) + abs(comp3(t)+comp3(f))
            if showfailure && diff>eps*mag
                original = from[i]
                @show original f t
            end
            errmax = max(errmax, diff/mag)
        end
        errmax > eps && @warn("Error on conversions from $(eltype(from)) to $(eltype(to)), relative error = $errmax")
        errmax <= eps
    end

    for i = 1:length(csconv)
        f, t = csconv[i]
        if !convcompare(f, t, 1e-3)
            println("  index $i")
        end
    end

    # Images issue #382
    @test convert(Gray, RGBA(1,1,1,1)) == Gray(N0f8(1))

    # https://github.com/timholy/Images.jl/pull/445#issuecomment-189866806
    @test convert(Gray, RGB{N0f8}(0.145,0.145,0.145)) == Gray{N0f8}(0.145)

    # Issue #257
    c = RGB{Float16}(0.9473,0.962,0.9766)
    hsi = convert(HSI, c)
    @test hsi.i > 0.96 && hsi.h ≈ 210
end


"""
    Schedule

Represents a scheduling queue in which events can be scheduled to occur at future time.
The schedule time is the time of the most recent scheduled event.
The schedule time starts from 0.0. The time -1.0 is the time before simulation.
The time after simulation is the time maxiumum Float64.
You can schedule your events at time greater then 0.0, such as 0.0 + eps().

The events are cloned before each simulation step, when an agent is scheduled,
there are no changing in its memory, until the  next simulation step.
This ensures that the agent memory remains unchanged during the simulation step,
and the agents can see the status of the other agents consistent with the current simulation time.

**Constructors**

    Schedule()

Construct `Schedule` with empty events list and initial time 0.0.

"""

mutable struct Schedule
    simstate::SimState
    events::PriorityQueue{Agent,Priority}
    endevents::Dict{Agent,Agent}
    steps::Int64
    time::Float64
    Schedule(simstate::SimState) =
        new(simstate,PriorityQueue{Agent,Priority}(), Dict{Agent,Agent}(), 0, 0.0)
end

"""
    scheduleOnce!(schedule::Schedule, agent::Agent, time::Float64=1.0; ordering::Int=0)

Schedules the `agent` agent to act at the `schedule` at time interval `time`
in the order for the given time  `ordering`.
"""
function scheduleOnce!(schedule::Schedule, agent::Agent, time::Float64=1.0; ordering::Int=0)
    push!(schedule.endevents,agent=>agent)
    enqueue!(schedule.events, agent, Priority(schedule.time+time,0))
end

function stop!(schedule::Schedule,agent::Agent)
    push!(schedule.endevents,agent=>agent)
end

"""
    scheduleRepeating!(schedule::Schedule, agent::Agent, time::Float64=1.0; ordering::Int=0)

Schedules the `agent` agent to repeatedly act at the `schedule` at time interval `time`
in the order for the given time  `ordering`.

"""
function scheduleRepeating!(schedule::Schedule, agent::Agent, time::Float64=1.0; ordering::Int=0)
    enqueue!(schedule.events, agent, Priority(time,ordering))
end

"""
"""
function simtime(schedule::Schedule)
    schedule.time
end

"""
"""
function step!(schedule::Schedule)
    schedule.steps+=1
    if length(schedule.events) == 0 return end
    schedule.time = peek(schedule.events).second.time
    ctime = schedule.time #current simulation time given by the event with small time
    cevents = Vector{Pair{Agent,Priority}}() #current events to be scheduled for this time
    while true
        if(isempty(schedule.events)) break end # no other events in the scheduling queue
        @inbounds event = peek(schedule.events)
        if(event.second.time > schedule.time) break end #no other events for currenttime
        ctime = event.second.time
        @inbounds dequeue!(schedule.events)
        push!(cevents,event)
    end #while loop

    #@sync @distributed
    for e in cevents
        #Double buffering on the event memory
    #    ce = deepcopy(e.first)
        ce = e.first
        ce.step(schedule.simstate,e.first)
        if(!haskey(schedule.endevents,e.first))
            enqueue!(schedule.events, ce, Priority(ctime+1.0, e.second.priority))
        end
    end
    #Reset the stopped events
    schedule.endevents = Dict{Agent,Agent}()
end