# Testing reference values and precisions
# Each test block of varr and parr should be followed by an append to refVals, refPrecs arrays.
# e.g.
#   refVals=[]
#   refPrecs=[]
#
#   varr = ..........
#   par  = ..........
#
#   append!(refVals ,[ varr ] )
#   append!(refPrecs,[ parr ] )
#
#   varr = ..........
#   par  = ..........
#
#   append!(refVals ,[ varr ] )
#   append!(refPrecs,[ parr ] )
#
#   varr = ..........
#   par  = ..........
#
#   append!(refVals ,[ varr ] )
#   append!(refPrecs,[ parr ] )
#
#   etc.....
#
#   Now for real!
#
#! format: off

    refVals=[]
    refPrecs=[]
# SC ========== Test number 1 reference values and precision match template. =======
# SC ========== /Users/chrishill/projects/clima/cm/test/Ocean/HydrostaticBoussinesq/test_ocean_gyre.jl test reference values ======================================
# BEGIN SCPRINT
# varr - reference values (from reference run)
# parr - digits match precision (hand edit as needed)
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [       "Q",   "u[1]", -2.18427865357219835873e-02,  4.54905817704320050709e-02,  2.91799084468619303323e-03,  9.86739717788472255056e-03 ],
 [       "Q",   "u[2]", -6.47928098321387119229e-02,  7.44690631237251432495e-02, -1.82439830964300966215e-03,  1.02482752029024841434e-02 ],
 [       "Q",       :η, -6.35241759974374819997e-01,  6.25677877693153861038e-01, -8.60068718305024041901e-04,  2.24583755068219675932e-01 ],
 [       "Q",       :θ,  9.03726335555428482264e-05,  9.03968461560725344839e+00,  2.49953258705220227043e+00,  2.19711762918947606238e+00 ],
 [   "s_aux",       :y,  0.00000000000000000000e+00,  4.00000000000000046566e+06,  2.00000000000000000000e+06,  1.18025270281469495967e+06 ],
 [   "s_aux",       :w, -4.50036075461459223737e-05,  4.09915220804811021062e-05,  3.06722602595587297903e-07,  8.22999489798410955438e-06 ],
 [   "s_aux",    :pkin, -9.00231747934270076783e-01,  0.00000000000000000000e+00, -3.32080874523538072118e-01,  2.56218992025050162908e-01 ],
 [   "s_aux",     :wz0, -2.22688579102610652283e-05,  1.52895210358102938127e-05, -2.78589672590168561413e-08,  7.72520757253795920232e-06 ],
 [ "s_gflux", "ν∇u[1]", -1.84915426791050691892e-03,  1.81823541139391206892e-03,  4.41670652876216560588e-08,  1.43003334026535461722e-04 ],
 [ "s_gflux", "ν∇u[2]", -2.18923622988041743920e-03,  1.83543406028729679316e-03,  3.98744828991449433208e-06,  3.02040505665129290977e-04 ],
 [ "s_gflux", "ν∇u[3]", -1.31369743560772895408e-05,  6.60039477945146449384e-06, -1.12119891567145960325e-08,  9.06617548358093396365e-07 ],
 [ "s_gflux", "ν∇u[4]", -3.29092487463896604671e-03,  3.71582885889340866492e-03, -2.53728229769815408283e-06,  2.97060576409856780689e-04 ],
 [ "s_gflux", "ν∇u[5]", -3.81669587050501284836e-03,  7.74606451320633592264e-04, -5.95763172118155189140e-06,  2.09071039491200258605e-04 ],
 [ "s_gflux", "ν∇u[6]", -7.41189897881484784109e-06,  1.26149616846000803353e-05,  1.41878743800085061125e-08,  1.12070628757683659611e-06 ],
 [ "s_gflux", "κ∇θ[1]", -5.86968584224644812338e-05,  4.34652887121820283387e-05, -2.82719382567508372835e-06,  8.15301391557321884090e-06 ],
 [ "s_gflux", "κ∇θ[2]", -3.70308432284375849564e-05,  3.11721545144766831389e-03,  9.81447035540739714035e-04,  8.31872571479874375368e-04 ],
 [ "s_gflux", "κ∇θ[3]", -2.15846132870457054810e-05,  2.00382267962942220408e-05, -5.17436031104969784121e-07,  1.79150072112032684306e-06 ],
]
parr = [
 [       "Q",   "u[1]",    12,    12,    12,    12 ],
 [       "Q",   "u[2]",    12,    12,    12,    12 ],
 [       "Q",       :η,    12,    12,    12,    12 ],
 [       "Q",       :θ,    12,    12,    12,    12 ],
 [   "s_aux",       :y,    12,    12,    12,    12 ],
 [   "s_aux",       :w,    12,    12,    12,    12 ],
 [   "s_aux",    :pkin,    12,    12,    12,    12 ],
 [   "s_aux",     :wz0,    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[3]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[4]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[5]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[6]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[3]",    12,    12,     8,    12 ],
]
# END SCPRINT

    append!(refVals ,[ varr ] )
    append!(refPrecs,[ parr ] )

# SC ========== Test number 2 reference values and precision match template. =======
# SC ========== /Users/chrishill/projects/clima/cm/test/Ocean/HydrostaticBoussinesq/test_ocean_gyre.jl test reference values ======================================
# BEGIN SCPRINT
# varr - reference values (from reference run)
# parr - digits match precision (hand edit as needed)
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [       "Q",   "u[1]", -2.14403161060213384714e-02,  4.51092912722281316751e-02,  2.91883577365915263327e-03,  9.85543151135863208789e-03 ],
 [       "Q",   "u[2]", -6.47849908268208068973e-02,  7.44377079062340796245e-02, -1.82528856707940708402e-03,  1.02465626974943338490e-02 ],
 [       "Q",       :η, -6.35241717401733962944e-01,  6.25675643032199912952e-01, -8.59997171611655590248e-04,  2.24578663698960928619e-01 ],
 [       "Q",       :θ,  1.21624650983828366162e-04,  9.05348870526675320036e+00,  2.49953463844110279624e+00,  2.19723475068906504148e+00 ],
 [   "s_aux",       :y,  0.00000000000000000000e+00,  4.00000000000000046566e+06,  2.00000000000000000000e+06,  1.18025270281469495967e+06 ],
 [   "s_aux",       :w, -4.48750663401728297594e-05,  4.07425796079212936126e-05,  3.05580091823959372610e-07,  8.20754136359875149074e-06 ],
 [   "s_aux",    :pkin, -9.00231018468760080253e-01,  0.00000000000000000000e+00, -3.32080814803801249724e-01,  2.56218661345592568779e-01 ],
 [   "s_aux",     :wz0, -2.24168211854045644460e-05,  1.54464254869175437296e-05, -2.71999565492576589218e-08,  7.76164707621753630128e-06 ],
 [ "s_gflux", "ν∇u[1]", -1.85093041460835305498e-03,  1.81880838666876047845e-03,  4.56526424711389789871e-08,  1.42643910766357428534e-04 ],
 [ "s_gflux", "ν∇u[2]", -2.20819366719671563229e-03,  1.84128108384750657951e-03,  3.98074575776293245019e-06,  3.03797417171225480129e-04 ],
 [ "s_gflux", "ν∇u[3]", -1.32036367846675870225e-05,  6.56099008285145167548e-06, -1.24087351933463812712e-08,  9.05500396408119435374e-07 ],
 [ "s_gflux", "ν∇u[4]", -3.28419921150994737619e-03,  3.70704618050693088249e-03, -2.54827541969113706709e-06,  2.96228406753828749521e-04 ],
 [ "s_gflux", "ν∇u[5]", -3.80879251696829247351e-03,  7.74834565383260352560e-04, -5.93513695165015881065e-06,  2.08479602570907525654e-04 ],
 [ "s_gflux", "ν∇u[6]", -7.39839704966590930855e-06,  1.26017207509591005658e-05,  1.46129449482955510494e-08,  1.11733219690338497597e-06 ],
 [ "s_gflux", "κ∇θ[1]", -5.81601257251870891614e-05,  4.70677319813339798652e-05, -2.82659431063543911260e-06,  8.16581551142350108366e-06 ],
 [ "s_gflux", "κ∇θ[2]", -3.76891063975906055088e-05,  3.10874693302757654570e-03,  9.82036601802479310747e-04,  8.31189786431030404844e-04 ],
 [ "s_gflux", "κ∇θ[3]", -2.42394160392341078493e-06,  8.68196014640797840416e-08, -5.04918136536487334668e-07,  3.01622429419160335704e-07 ],
]
parr = [
 [       "Q",   "u[1]",    12,    12,    12,    12 ],
 [       "Q",   "u[2]",    12,    12,    12,    12 ],
 [       "Q",       :η,    12,    12,    12,    12 ],
 [       "Q",       :θ,    12,    12,    12,    12 ],
 [   "s_aux",       :y,    12,    12,    12,    12 ],
 [   "s_aux",       :w,    12,    12,    12,    12 ],
 [   "s_aux",    :pkin,    12,    12,    12,    12 ],
 [   "s_aux",     :wz0,    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[3]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[4]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[5]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[6]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[3]",    12,     8,    12,    12 ],
]

    append!(refVals ,[ varr ] )
    append!(refPrecs,[ parr ] )

# SC ========== Test number 3 reference values and precision match template. =======
# SC ========== /Users/chrishill/projects/clima/cm/test/Ocean/HydrostaticBoussinesq/test_ocean_gyre.jl test reference values ======================================
# BEGIN SCPRINT
# varr - reference values (from reference run)
# parr - digits match precision (hand edit as needed)
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [       "Q",   "u[1]", -1.32150247347356725180e-01,  1.64031203423611798842e-01,  4.86341857180362744512e-03,  1.48290239846339700486e-02 ],
 [       "Q",   "u[2]", -6.36612598877540902809e-02,  7.44560153894480020176e-02, -2.47624053318063700596e-03,  1.31303104153666864301e-02 ],
 [       "Q",       :η, -6.17793649659350796455e-01,  6.06966232332265209770e-01, -1.29090689600396902117e-03,  2.25023233970694097605e-01 ],
 [       "Q",       :θ,  8.59363830767245827184e-05,  9.03903412174035025828e+00,  2.49947426680983575409e+00,  2.19708020497952460914e+00 ],
 [   "s_aux",       :y,  0.00000000000000000000e+00,  4.00000000000000046566e+06,  2.00000000000000000000e+06,  1.18025270281469495967e+06 ],
 [   "s_aux",       :w, -4.88109548294750840891e-05,  5.07779930498253729947e-05,  3.71427081618204492464e-07,  9.36953994676888197318e-06 ],
 [   "s_aux",    :pkin, -9.00072683420330310611e-01,  0.00000000000000000000e+00, -3.32075420474812310623e-01,  2.56210240101286457204e-01 ],
 [   "s_aux",     :wz0, -2.37163738698016553002e-05,  1.61967572887668093411e-05, -4.64606200633942812947e-08,  7.86254275820167091446e-06 ],
 [ "s_gflux", "ν∇u[1]", -6.67376825614485365279e-03,  6.97979211528434052153e-03,  1.01948640467980356077e-07,  2.29183131022463946721e-04 ],
 [ "s_gflux", "ν∇u[2]", -7.69310192750550202179e-03,  1.37445482375157909845e-03, -3.35376028298812290740e-05,  5.28895496282001859913e-04 ],
 [ "s_gflux", "ν∇u[3]", -1.70129300483239544585e-05,  1.30704917153533933284e-05, -4.30738980896587680255e-08,  1.59157632234111246367e-06 ],
 [ "s_gflux", "ν∇u[4]", -4.05738110420349878948e-03,  4.20907825808012042018e-03,  2.59008171395293793600e-06,  4.12110908407640325318e-04 ],
 [ "s_gflux", "ν∇u[5]", -4.15559862073769641783e-03,  1.11805092953295753984e-03, -7.36416840356856789053e-06,  2.39482266912490057102e-04 ],
 [ "s_gflux", "ν∇u[6]", -7.43240009691175527648e-06,  2.23076045923041977150e-05,  2.75166886765055948798e-08,  1.43297925696224871009e-06 ],
 [ "s_gflux", "κ∇θ[1]", -7.28126206584962977356e-05,  6.51042269918416155244e-05, -3.06828578003712164183e-06,  9.88453091533718813987e-06 ],
 [ "s_gflux", "κ∇θ[2]", -5.64043404057886982382e-05,  3.11699982926543850761e-03,  9.81313644046314260266e-04,  8.32013174688638057486e-04 ],
 [ "s_gflux", "κ∇θ[3]", -2.16059510376949300253e-05,  2.05358490375227276093e-05, -5.17225691641381943229e-07,  1.79077714232301003982e-06 ],
]
parr = [
 [       "Q",   "u[1]",    12,    12,    12,    12 ],
 [       "Q",   "u[2]",    12,    12,    12,    12 ],
 [       "Q",       :η,    12,    12,    12,    12 ],
 [       "Q",       :θ,    12,    12,    12,    12 ],
 [   "s_aux",       :y,    12,    12,    12,    12 ],
 [   "s_aux",       :w,    12,    12,    12,    12 ],
 [   "s_aux",    :pkin,    12,    12,    12,    12 ],
 [   "s_aux",     :wz0,    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[3]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[4]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[5]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[6]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[3]",    12,    12,    12,    12 ],
]
# END SCPRINT
# SC ====================================================================================

    append!(refVals ,[ varr ] )
    append!(refPrecs,[ parr ] )

# SC ========== Test number 4 reference values and precision match template. =======
# SC ========== /Users/chrishill/projects/clima/cm/test/Ocean/HydrostaticBoussinesq/test_ocean_gyre.jl test reference values ======================================
# BEGIN SCPRINT
# varr - reference values (from reference run)
# parr - digits match precision (hand edit as needed)
#
# [
#  [ MPIStateArray Name, Field Name, Maximum, Minimum, Mean, Standard Deviation ],
#  [         :                :          :        :      :          :           ],
# ]
varr = [
 [       "Q",   "u[1]", -1.32522061560302495398e-01,  1.64403805704383537689e-01,  4.86426609935955867742e-03,  1.48297097518645653452e-02 ],
 [       "Q",   "u[2]", -6.36533081952946194759e-02,  7.44240286788056537581e-02, -2.47705914433662445495e-03,  1.31287947739917917195e-02 ],
 [       "Q",       :η, -6.17792146223299143415e-01,  6.06964210100974344009e-01, -1.29086897946112909297e-03,  2.25017541338622450997e-01 ],
 [       "Q",       :θ,  1.18930697909414831350e-04,  9.05193063653280205472e+00,  2.49947692790997599843e+00,  2.19719805926361999582e+00 ],
 [   "s_aux",       :y,  0.00000000000000000000e+00,  4.00000000000000046566e+06,  2.00000000000000000000e+06,  1.18025270281469495967e+06 ],
 [   "s_aux",       :w, -4.86355979078713029592e-05,  5.05698562144389179547e-05,  3.70131833873677610995e-07,  9.34929694591741612729e-06 ],
 [   "s_aux",    :pkin, -9.00073156073595503912e-01,  0.00000000000000000000e+00, -3.32075359710396145196e-01,  2.56209888633661342361e-01 ],
 [   "s_aux",     :wz0, -2.38160453952588820092e-05,  1.63295706147758099968e-05, -4.63837554347600732956e-08,  7.90783852649017554050e-06 ],
 [ "s_gflux", "ν∇u[1]", -6.68560102683574044441e-03,  6.98986179365855032908e-03,  1.00393045025810862916e-07,  2.29127846391889087837e-04 ],
 [ "s_gflux", "ν∇u[2]", -7.69995573676401749708e-03,  1.37839162179005680388e-03, -3.35981134949950310548e-05,  5.29965375899606985950e-04 ],
 [ "s_gflux", "ν∇u[3]", -1.70723021047607414786e-05,  1.30922704334956021486e-05, -4.44618327893026796269e-08,  1.59336218473792970699e-06 ],
 [ "s_gflux", "ν∇u[4]", -4.05946426934079233201e-03,  4.21052254828468888043e-03,  2.59778123025053437326e-06,  4.12726714325575502491e-04 ],
 [ "s_gflux", "ν∇u[5]", -4.14814071152546075955e-03,  1.11726677539041160600e-03, -7.34353368996733004402e-06,  2.38907876999768631733e-04 ],
 [ "s_gflux", "ν∇u[6]", -7.41889880459672764628e-06,  2.22738260107801488963e-05,  2.78343092988202756278e-08,  1.42972197860415487431e-06 ],
 [ "s_gflux", "κ∇θ[1]", -7.25681212396385411870e-05,  7.16011471902109551376e-05, -3.07008530876072845326e-06,  9.89983434920985542473e-06 ],
 [ "s_gflux", "κ∇θ[2]", -5.79706743991101401507e-05,  3.10853728745289804858e-03,  9.81905426072233198168e-04,  8.31329782146290126484e-04 ],
 [ "s_gflux", "κ∇θ[3]", -3.24307611623586362264e-06,  1.48205642064209931587e-06, -5.05122832861326802479e-07,  3.05070684012206338078e-07 ],
]
parr = [
 [       "Q",   "u[1]",    12,    12,    12,    12 ],
 [       "Q",   "u[2]",    12,    12,    12,    12 ],
 [       "Q",       :η,    12,    12,    12,    12 ],
 [       "Q",       :θ,    12,    12,    12,    12 ],
 [   "s_aux",       :y,    12,    12,    12,    12 ],
 [   "s_aux",       :w,    12,    12,    12,    12 ],
 [   "s_aux",    :pkin,    12,    12,    12,    12 ],
 [   "s_aux",     :wz0,    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[3]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[4]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[5]",    12,    12,    12,    12 ],
 [ "s_gflux", "ν∇u[6]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[1]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[2]",    12,    12,    12,    12 ],
 [ "s_gflux", "κ∇θ[3]",    12,    12,    12,    12 ],
]
# END SCPRINT
# SC ====================================================================================

    append!(refVals ,[ varr ] )
    append!(refPrecs,[ parr ] )

#! format: on


module FWI

using jInv.Mesh
using jInv.Utils
using jInv.LinearSolvers
using jInv.InverseSolve
using EikonalInv
using KrylovMethods
using Multigrid
using ForwardHelmholtz
using MAT

import jInv.ForwardShare.getData
import jInv.ForwardShare.getSensTMatVec
import jInv.ForwardShare.getSensMatVec
import jInv.LinearSolvers.copySolver

import jInv.ForwardShare.ForwardProbType

FieldsType = Complex32

useSommerfeldBC = true;

fieldsFilenamePrefix = "tempFWIfields"

function getFieldsFileName(omega::Float64)
	omRound = string(round((omega/(2*pi))*100.0)/100.0);
	tfilename = string(fieldsFilenamePrefix,"_f",omRound,"_worker",myid(),".mat");
end

export clear!
export setSourceSelection,setSourceSelectionRatio,setSourceSelectionNum

export FWIparam, getFWIparam
type FWIparam <: ForwardProbType
    omega					:: Float64     # frequencies
	WaveletCoef				:: Complex128
    gamma					:: Vector{Float64}     # attenuation
    Sources					:: Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}}   # Sources
    Receivers				:: Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}}
	Fields					:: Array{FieldsType}
    Mesh      				:: RegularMesh
	ForwardSolver			:: AbstractSolver
	forwardSolveBatchSize	:: Int64
	sourceSelection			:: Array{Int64,1}
	useFilesForFields		:: Bool
end

function getFWIparam(omega::Float64, WaveletCoef::Complex128, gamma::Vector{Float64},
							Sources::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							Receivers::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							Mesh::RegularMesh, ForwardSolver:: AbstractSolver, workerList::Array{Int64},forwardSolveBatchSize::Int64=size(Sources,2),useFilesForFields::Bool = false)
	return getFWIparam([omega], [WaveletCoef],gamma,Sources,Receivers, Mesh,ForwardSolver, workerList,forwardSolveBatchSize,useFilesForFields);
end

function getFWIparam(omega::Array{Float64}, WaveletCoef::Array{Complex128},gamma::Vector{Float64},
							Sources::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							Receivers::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							Mesh::RegularMesh, ForwardSolver::AbstractSolver, workerList::Array{Int64},forwardSolveBatchSize::Int64=size(Sources,2),useFilesForFields::Bool = false)

	continuationDivision = zeros(Int64,length(omega)+1);
	continuationDivision[1] = 1;

	if workerList==[]
		ActualWorkers = workers();
	else
		ActualWorkers = intersect(workerList,workers());
		if length(ActualWorkers)<length(workerList)
			warn("FWI: workerList included indices of non-existing workers.")
		end
	end
	numWorkers = length(ActualWorkers);
	pFor   = Array{RemoteChannel}(numWorkers*length(omega));
	SourcesSubInd = Array{Array{Int64,1}}(numWorkers*length(omega));
	for k=1:length(omega)
		getFWIparamInternal(omega[k],WaveletCoef[k], gamma,Sources,Receivers,zeros(FieldsType,0), Mesh,
									ForwardSolver, forwardSolveBatchSize ,ActualWorkers,pFor,(k-1)*numWorkers+1,SourcesSubInd,useFilesForFields);
		continuationDivision[k+1] = k*numWorkers+1;
	end
	return pFor,continuationDivision,SourcesSubInd # Array of Remote Refs
end

function getFWIparamInternal(omega::Float64, WaveletCoef::Complex128,gamma::Vector{Float64},
							Sources::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							Receivers::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							fields::Array{FieldsType}, Mesh::RegularMesh,
							ForwardSolver:: AbstractSolver, forwardSolveBatchSize::Int64,
							Workers::Array{Int64}, pFor::Array{RemoteChannel},startPF::Int64,SourcesSubInd::Array{Array{Int64,1},1},useFilesForFields::Bool = false)
	i = startPF; nextidx() = (idx=i; i+=1; idx)
	nsrc  = size(Sources,2);
	numWorkers = length(Workers);
	# send out jobs
	@sync begin
		for p=Workers
			@async begin
				while true
					idx = nextidx()
					if idx > startPF + numWorkers - 1
						break
					end
					I_k = getSourcesIndicesOfKthWorker(numWorkers,idx - startPF + 1,nsrc)
					SourcesSubInd[idx] = I_k;
					pFor[idx] = initRemoteChannel(getFWIparamInternal,p, omega,WaveletCoef,  gamma, Sources[:,I_k], Receivers, fields, Mesh,
																			copySolver(ForwardSolver),forwardSolveBatchSize,useFilesForFields);
					wait(pFor[idx]);
				end
			end
		end
	end
	return pFor # Array of Remote Refs
end

function getFWIparamInternal(omega::Float64,WaveletCoef::Complex128,
							gamma::Vector{Float64},
	                        Sources::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							Receivers::Union{Vector{Float64},SparseMatrixCSC,Array{Float64,2}},
							Fields::Array{FieldsType},
							Mesh::RegularMesh, ForwardSolver:: AbstractSolver, forwardSolveBatchSize::Int64,useFilesForFields::Bool = false)
	return FWIparam(omega,WaveletCoef,gamma,Sources,Receivers,Fields,Mesh,ForwardSolver,forwardSolveBatchSize,Array{Int64}(0),useFilesForFields)
end

# function setSourceSelection(pForRF::RemoteRef{Channel{Any}}, selection::Array{Int64,1})
	# s = copy(selection);
	# if minimum(s) < 1
		# error("FWI: sources selection out of range");
	# end
	# pFor  = take!(pForRF);
	# s = [];
	# if isa(pFor,FWIparam)
		# Sources = pFor.Sources
		# if maximum(s)>size(Sources,2)
			# s = s[s.<=size(Sources,2)];
			# warn("FWI: reducing selection: s = s[s.<=size(Sources,2)]");
		# end
		# pFor.sourceSelection = s;
	# end
	# put!(pForRF,pFor);
	# return s;
# end

# function setSourceSelectionRatio(pForRF::RemoteRef{Channel{Any}}, selectionRatio::Float64)
	# if selectionRatio <= 0.0 || selectionRatio > 1.0
		# error("selectionRatio has to be between 0 and 1");
	# end
	# if selectionRatio == 1.0
		# return;
	# end
	# pFor  = take!(pForRF);
	# s = [];
	# if isa(pFor,FWIparam)
		# Q = pFor.Sources
		# s = randperm(size(Q,2))[1:ceil(Int64,size(Q,2)*selectionRatio)];
		# pFor.sourceSelection = s;
	# end
	# put!(pForRF,pFor);
	# return s;
# end

# function setSourceSelectionNum(pForRF::RemoteRef{Channel{Any}}, selectionNum::Int64)
	# if selectionNum <= 0
		# error("selectionNum has to be bigger than 1");
	# end
	# pFor  = take!(pForRF);
	# s = [];
	# if isa(pFor,FWIparam)
		# Q = pFor.Sources
		# s = randperm(size(Q,2))[1:selectionNum];
		# pFor.sourceSelection = s;
	# end
	# put!(pForRF,pFor);
	# return s;
# end

import jInv.Utils.clear!
function clear!(pFor::FWIparam)
	clear!(pFor.ForwardSolver);
	pFor.Fields = zeros(FieldsType,0);
	clear!(pFor.Mesh);
	return pFor;
end

include("getData.jl")
include("getSensMatVec.jl")
include("getSensTMatVec.jl")
include("FourthOrderHesPrec.jl")
include("freqCont.jl")
include("timeDomainFWI.jl")
include("firstArivalPicking.jl")
end


using MLKernels
using JuMP
using MLLabelUtils
using StatsBase
using NearestNeighbors
import MLBase

const learning_pool_enc = LabelEnc.NativeLabels([:U, :Lin, :Lout])
const class_label_enc = LabelEnc.NativeLabels([:inlier, :outlier])
const OPT_PRECISION = 1e-7

@enum ModelState model_created=1 model_initialized=2 model_fitted=3

const Scope = Union{Val{:Subspace}, Val{:Global}}

abstract type ModelException <: Exception end

struct ModelStateException <: ModelException
    actual::ModelState
    expected::ModelState
end

Base.showerror(io::IO, e::ModelStateException) = print(io, "Invalid model state: $(e.actual). Model must at least be in state $(e.expected).")

struct ModelInvariantException <: ModelException
    msg
end

Base.showerror(io::IO, e::ModelInvariantException) = print(io, "Model invariant error. $(e.msg)")

abstract type OCClassifier end

abstract type SVDDClassifier <: OCClassifier end

abstract type SubOCClassifier <: OCClassifier end

abstract type InitializationStrategy end


using CSV
using Statistics
using Random

import Gurobi
const GRBENV = Gurobi.Env()

import MathOptInterface
const MOI = MathOptInterface

const SAF = MOI.ScalarAffineFunction
const SAT = MOI.ScalarAffineTerm

import Linda

const DATA_DIR = joinpath(@__DIR__, "../dat")
const RMPDIR = joinpath(@__DIR__, "rmp")

include("../src/DemandResponse.jl")
const DR = DemandResponse

"""
    generate_resources()

# Arguments
- `num_der`: The number of households
- `T`: The number of time-steps
- `devices_rates`: Ownership rates of each device
- `load_norm`: Normalized reference load profile
- `PV_norm`: Normalized PV output profile
- `price`: Electricity price, in dollar per kW.h.
"""
function generate_resources(
    num_der, T,
    devices_rates::Dict{Symbol, Float64},
    load_norm::Vector,
    PV_norm::Vector,
    temp_ext::Vector,
    price::Vector;
    seed=0
)
    # Sanity checks
    T == length(load_norm) || throw(DimensionMismatch(
        "T=$T but load_norm has length $(length(load_norm))"
    ))
    T == length(PV_norm) || throw(DimensionMismatch(
        "T=$T but load_norm has length $(length(PV_norm))"
    ))
    T == length(price) || throw(DimensionMismatch(
        "T=$T but load_norm has length $(length(price))"
    ))

    Random.seed!(seed)
    resources = Vector{DR.DER.Resource}(undef, num_der)

    # Create a pool of `num_der` houses
    # Each household owns appliances according to `devices_rates`
    for i in 1:num_der

        appliances = DR.DER.Resource[]
        γ = 0.5 + rand()  # household's scaling factor

        # ==================
        #    Fixed load
        # ==================
        # All houses have a fixed load
        if rand() < devices_rates[:fixed]
            l = DR.DER.FixedLoad(
                index=i,
                T=T,
                load= γ .* max.(0.0, (load_norm .+ 0.05 .* randn(T)))
            )
            push!(appliances, l)
        end

        # ==================
        #    Fixed load
        # ==================
        # Dishwasher
        if rand() < devices_rates[:dishwasher]
            dishwasher = DR.DER.ShiftableLoad(
                index = 10000000 + i,  # TODO: handle index number properly
                T=T,
                cycle_begin_min=1,
                cycle_begin_max=T-1,
                cycle_length=2,
                cycle_load=[1.0, 1.0]
            )
            push!(appliances, dishwasher)
        end

        # ======================
        #    Shiftable load
        # ======================
        # Clothes washer
        if rand() < devices_rates[:clothes_washer]
            clothes_washer = DR.DER.ShiftableLoad(
                index = 20000000 + i,  # TODO: handle index number properly
                T=T,
                cycle_begin_min=1,
                cycle_begin_max=T-1,
                cycle_length=2,
                cycle_load=[1.0, 1.0]
            )
            push!(appliances, clothes_washer)

            # Clothes dryer
            # Only household with clothes washer may have a clothes dryer
            if rand() < (
                devices_rates[:clothes_dryer] / devices_rates[:clothes_washer]
            )

                clothes_dryer = DR.DER.ShiftableLoad(
                    index = 30000000 + i,  # TODO: handle index number properly
                    T=T,
                    cycle_begin_min=1,
                    cycle_begin_max=T-2,
                    cycle_length=3,
                    cycle_load=[1.0, 1.0, 1.0]
                )
                push!(appliances, clothes_dryer)
            end
        end
        # ===================
        #    Thermal load
        # ===================
        # Electric heating
        if rand() < devices_rates[:thermal]
            th = DR.DER.ThermalLoad(
                index=i, T=T,
                temp_min=fill(18.0, T), temp_max=fill(22.0, T),
                temp_init=20.0,
                temp_ext=temp_ext + 0.5 .* randn(T),
                pwr_min=zeros(T), pwr_max=10.0 .* ones(T),
                C=3.0, η=1.0, μ=0.2,
                binflag=false
            )
            push!(appliances, th)
        end

        # =====================
        #    PV, EV & ES
        # =====================
        if rand() < devices_rates[:renew]

            # Electric vehicle
            if (T % 24) == 0
                ndays = div(T, 24)

                ev = DR.DER.DeferrableLoad(
                    index=i, T=T,
                    ncycles=ndays,
                    cycle_begin=[16 + 24*(d-1) for d in 1:ndays],
                    cycle_end=[24 + 24*(d-1) for d in 1:ndays],
                    cycle_energy_min=fill(10.0, ndays),
                    cycle_energy_max=fill(10.0, ndays),
                    pwr_min=repeat([zeros(15); 1.1*ones(9)], ndays),
                    pwr_max=repeat([zeros(15); 7.7*ones(9)], ndays),
                    binflag=true
                )
                push!(appliances, ev)

            else
                # Not an integer number of days
                # Do not include EV
            end

            # PV
            ζ = rand(T)
            pv = DR.DER.CurtailableLoad(
                index=i,
                T=T,
                load= - γ .* PV_norm .* ζ,
                binflag=true
            )
            push!(appliances, pv)

            # Battery
            bat = DR.DER.Battery(
                index=i,
                T=T,
                soc_min=zeros(T),
                soc_max=13.5 .* ones(T),
                soc_init=0.0,
                selfdischargerate=1.0,
                charge_pwr_min=0.0, charge_pwr_max=5.0,
                discharge_pwr_min=0.0, discharge_pwr_max=5.0,
                charge_eff=0.95, discharge_eff=0.95
            )
            push!(appliances, bat)
        end

        # Household
        h = DR.DER.House(
            index=i,
            T=T,
            netload_min = zeros(T),
            netload_max = 10.0 .* ones(T),
            price=price,
            appliances=appliances
        )
        resources[i] = h
    end

    return resources
end

function main(T, R, T0=408+7)

    # Load data
    ont_load = CSV.read(joinpath(DATA_DIR, "load2016.csv"))
    ont_price = CSV.read(joinpath(DATA_DIR, "price2016.csv"))
    ont_prod = CSV.read(joinpath(DATA_DIR, "prod2016.csv"))
    ont_temp = CSV.read(joinpath(DATA_DIR, "temperature2016.csv"))

    # Market and Time-Of-Use prices
    p_tou = Vector{Float64}(ont_price.TOU[T0:(T0+T-1)])
    p_mkt = Vector{Float64}(ont_price.HOEP[T0:(T0+T-1)])

    # Normalized load
    load_norm = (ont_load.OntDemand / mean(ont_load.OntDemand))[T0:(T0+T-1)]

    # Normalized PV output
    pv_norm = (ont_prod.SOLAR / mean(ont_prod.SOLAR))[T0:(T0+T-1)]

    # Outside temperature
    temp_ext = ont_temp.Temperature[T0:(T0+T-1)]

    # Min and max total net load
    total_load_min = 0.0 * R * ones(T)
    total_load_max = 6.0 * R * ones(T)

    # Devices' ownership rates
    own_rates = Dict(
        :fixed => 1.0,
        :dishwasher => 0.65,
        :clothes_washer => 0.9,
        :clothes_dryer => 0.75,
        :thermal => 0.6,
        :renew => 0.33
    )

    # Generate resources
    resources = generate_resources(
        R, T, own_rates,
        load_norm, pv_norm, temp_ext, p_mkt,
        seed=5
    )

    # Instantiate oracles
    oracles = [
    DR.build_oracle(
            MOI.Bridges.full_bridge_optimizer(Gurobi.Optimizer(GRBENV, OutputFlag=0, Threads=1), Float64),
            hh
        )
        for hh in resources
    ]

    # Instantiate the RMP
    rmp = Gurobi.Optimizer(GRBENV);
    MOI.set(rmp, MOI.RawParameter("Threads"), 1)
    MOI.set(rmp, MOI.RawParameter("Method"), 2)
    MOI.set(rmp, MOI.RawParameter("OutputFlag"), 0)
    M = 1e3

    # Linking variables & artificial slacks for linking constraints
    y  = MOI.add_variables(rmp, T)
    zp = MOI.add_variables(rmp, T)
    zm = MOI.add_variables(rmp, T)
    for t in 1:T
        MOI.add_constraint(rmp, MOI.SingleVariable(zp[t]), MOI.GreaterThan(0.0))
        MOI.add_constraint(rmp, MOI.SingleVariable(zm[t]), MOI.GreaterThan(0.0))
        MOI.add_constraint(rmp, MOI.SingleVariable(y[t]), MOI.Interval(total_load_min[t], total_load_max[t]))
    end

    var_link = vcat(y, zp, zm)

    # Linking constraints
    con_link = MOI.add_constraints(rmp,
        [
            SAF([SAT(-1.0, y[t]), SAT(1.0, zp[t]), SAT(-1.0, zm[t])], 0.0)
            for t in 1:T
        ],
        [
            MOI.EqualTo(0.0)
            for t in 1:T
        ]
    );

    # Convexity constraints
    con_cvx = MOI.add_constraints(rmp,
        [SAF(SAT{Float64}[], 0.0) for _ in 1:R],
        [MOI.EqualTo(1.0) for _ in 1:R]
    );

    # Objective
    MOI.set(rmp, MOI.ObjectiveSense(), MOI.MIN_SENSE)
    fobj = SAF{Float64}([
        [
            SAT(M, zp[t]) for t in 1:T
        ]; [
            SAT(M, zm[t]) for t in 1:T
        ]], 0.0
    )
    MOI.set(rmp, MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}(), fobj)

    rhs = zeros(T)
    mp = Linda.Master(rmp, con_cvx, con_link, rhs, var_link, MOI.VariableIndex[])

    # Add initial columns
    for oracle in oracles
        Linda.update!(oracle, false, zeros(T), 0.0)
        Linda.optimize!(oracle)

        cols = Linda.get_columns(oracle)
        col, _ = cols[1]
        Linda.add_column!(mp, col)
    end

    env = Linda.LindaEnv()
    env.verbose.val = 1
    env.num_columns_max.val = div(R, 8)
    env.num_cgiter_max.val = 200

    MOI.set(rmp, MOI.Silent(), true)

    cglog = Dict()

    function cg_callback()
        niter::Int = cglog[:n_cg_iter]
        if niter > 0 && niter %10 == 0
            @info "Saving RMP at iter $niter"
            Gurobi.write_model(rmp.inner, joinpath(RMPDIR, "DER_$(T)_$(R)_$(niter).mps.bz2"))
        end
        return nothing
    end

    # Print MP solver
    println("MP solver: ", MOI.get(mp.rmp, MOI.SolverName()), "\n")

    Random.seed!(0)
    Linda.solve_colgen!(env, mp, oracles, cg_log=cglog, callback=cg_callback)
    Gurobi.write_model(rmp.inner, joinpath(RMPDIR, "DER_$(T)_$(R)_$(cglog[:n_cg_iter]).mps.bz2"))
    return
end

if abspath(PROGRAM_FILE) == @__FILE__
    T = parse(Int, ARGS[1])
    R = parse(Int, ARGS[2])
    @info "" T R
    main(T, R)
end