module TIFFtoRegression

############################################################################
# Reading the values of the TIFF according to TIFF v0.6

function FieldValues(ValueOffset::Int64,FileNumber::Array{Int16},FieldCount::Int64,FieldType=4::Int64) #v2
# =========================== Value or Pointer ? ===============================
ValuePointer=0::Int64
if FieldCount==1 # Simplified criterion of "can hold that much info on 4 bytes"
  Value=ValueOffset
else
  ValuePointer=ValueOffset+1 # correction due to the first being 0 and not 1
end

if ValuePointer==0
  DesiredV=Value
else
  #println("$ValuePointer, Count=$FieldCount, Type=$FieldType")
  DesiredV=zeros(1,FieldCount)
  for i=1:FieldCount
    Vi=0
    if FieldType == 2 || FieldType==1 #ASCII or BYTE
      FieldTypeByte=1
    elseif FieldType==3 # Short
      FieldTypeByte=2
    elseif FieldType==4 # Long
      FieldTypeByte=4
    elseif FieldType==5 # Rational
      FieldTypeByte=8
    end

    for jj=1:FieldTypeByte
      Vjj=FileNumber[ValuePointer+FieldTypeByte*(i-1)+jj-1]*256^(jj-1)
      Vi=Vi+Vjj
    end
    DesiredV[i]=convert(Int64,Vi)
  end
end

return DesiredV
end
#############################################################################

# Reading a TIFF (also able to read special Dantec TIFF that are pseudo Std)

function AquiTiff(FilePath::String)
  # Opening File
  FileString=read(FilePath)
  L=length(FileString)
  FileNumber=zeros(Int16,L,1)
  for i=1:L
    NumberI=convert(Int16,FileString[i])
    FileNumber[i]=NumberI
  end
  # ============================ Reading Header ================================
  IFD=FileNumber[5]+1+FileNumber[6]*256+FileNumber[7]*256^2+FileNumber[8]*256^3 # indicates where the Image File Directory starts (i e where the meta data start)
  # +1 in the indexes is a consequence of the starting count from 0 to 1 in tiff ==> JuliaVector
  if FileNumber[1]==73 & FileNumber[2]==73 # Small endian (determines the order of the bites from bigger to smaller)
  # ======================== Reading Metadata: ================================
  # ------------------------- Opening Values ----------------------------------
    Entries=FileNumber[IFD]+256*FileNumber[IFD+1] # Number of Entries in the IFD
    # Opening Values that might be in the Metadata
    Height=0::Int64
    Width=0::Int64
    BPS=0::Int64 #Bit per Sample
    CompType=0::Int64 #Type of Compression
    BlackWhite=0::Int64 #Photometric Interpretation # 0=White is Zero; 1<=Black is Zero
    StripOffsets=0::Int64 #Places where data is stored
    NFragData=0::Int64 # Number of those Places
    Orientation=0::Int64 # 1=Start Top-left
    SPPx=0::Int64 # Sample per Pixel
    RowsPStrip=0::Int64 # Rows per data strip: ceilling(Height/RowsPStrip)=Number of strip (p19)
    StripByteCount=0::Int64 #For each strip the number of bytes in that strip AFTER compression
    XRes=0::Int64 # X-Resolution : nbr of px per Resolution unit (cm/inch see 296)
    YRes=0::Int64 # Y-Resolution
    PlanConfig=0::Int64 # Planar Configuration ???
    ResUnit=0::Int64 # Resolution Unit : abstract (=0), cm (=1) or inch (=2)
    Software=0::Int64 # Software used to create the Tiff
    SBCType=0::Int64
    StripOffsets=[]
    NBytesPPx=0::Int64 # Spec Dantec Dynamics
    # --------------------------------------------------------------------------

    for i=1:Entries # All MetaData there are
      # ----------------------- Initiating Entry--------------------------------
      # Delimiting entry
      StartE=IFD+12*(i-1)+2
      EndE=StartE+11
      EntryN=FileNumber[StartE:EndE] #Taking one entry of 12 hexa bytes
      # Reading Entry
      FieldID=EntryN[1]+EntryN[2]*256
      FieldType=EntryN[3]+EntryN[4]*256
      FieldCount=EntryN[5]+EntryN[6]*256+EntryN[7]*65536+EntryN[8]*16777216
      ValueOffset=EntryN[9]+EntryN[10]*256+EntryN[11]*65536+EntryN[12]*16777216
      # Is either the value of the entry or a pointer towards the place where the values are kept if the entry can't be held on 8bits
      # ----------------------- Switch on FieldID ------------------------------
      if FieldID==254 # NewSubFileType
        NewSubFileType=FieldValues(ValueOffset,FileNumber,FieldCount)
        # 2 : Multipage
        # 4 : Defines a transparency mask
      elseif FieldID==256 # Image Width= n° of column in the Image
        Width=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==257 # Height
        Height=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==258 # Bits/sample ==> number of shade of gray allowable
        BPS=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==259 # Compression Type
        CompType=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==262 # Photometric Interpretation
        # 0=White is Zero; 1<=Black is Zero
        BlackXhite=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==273 # StripOffsets= Position of all data StripOffsets
        NFragData=FieldCount
        StripOffsets=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==274 # Orientation of the Picture
        # 1=Start Top-left
        Orientation=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==277 # Sample/px (3 or more ==> Number of colors allowable)
        SPPx=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==278 # Rows per Strip
        RowsPStrip=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==279 # StripByteCount
        StripByteCount=FieldValues(ValueOffset,FileNumber,FieldCount)
        SBCType=FieldType
      elseif FieldID==282 # X-Resolution
        XRes=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==283 # Y-Resolution
        YRes=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==284 # Planar Configuration
        PlanConfig=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==285
        PageName=FieldValues(ValueOffset,FileNumber,FieldCount,FieldType)
      elseif FieldID==296 # Resolution Unit (cm/inch)
        ResUnit=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==297
        PageNumber=FieldValues(ValueOffset,FileNumber,FieldCount,FieldType)
      elseif FieldID==305 # Software
        #Software=FieldValues(ValueOffset,FileNumber,FieldCount)
      elseif FieldID==32997
        #Specific to DynamicStudio, Dantec
        # Number of bytes/ px? TBC
        NBytesPPx=FieldValues(ValueOffset,FileNumber,FieldCount)
        if SPPx==0
          SPPx=2 #NBytesPPx/8 # Works?
          SPPx=convert(Int64,SPPx)
        end
      end  # Metadata Aquisition
    end
    EndIFD=IFD+12*(Entries-1)+13
  elseif FileNumber[1]==77 & FileNumber[2]==77 #big endian
    println("Big Endian Still to be implemented, nearly same as small endian")
  end
  # ======================= Creating the Img Matrix ============================
  # ----------------------- Finding and Ordering Px ---------------------------
  PixelString=[]
  PixelNumber=ones(1,length(FileNumber))
  CountLeng=0
  for i=1:NFragData
    PxStrip=FileNumber[convert(Int64,StripOffsets[i]):convert(Int64,StripOffsets[i]+StripByteCount[i])]
    l=length(PxStrip)
    PixelNumber[1+1+CountLeng:1+l+CountLeng]=PxStrip[1:l]
    CountLeng=CountLeng+l
  end
  PixelNumber=PixelNumber[1:CountLeng] # cutting the exedentary ones (due to the last strip being sometimes shorter than the others)
  # ------------------------ Making Order in the Px ----------------------------
  ImgArray=zeros(Int64,Height,Width,SPPx)
  for i=1:Height
    for  j=1:Width
      for k=1:SPPx
        if NBytesPPx==0
          Pxijk=((i-1)*Width+(j-1))*SPPx+k
        else # Why ? 171 dtmed empirically for Dynamic Studio Tif)
          #Pxijk=((i-1)*Width+(j-1))*SPPx+k
          Pxijk=((i-1)*Width+(j+convert(Int64,floor(171*i/Height))-1))*SPPx+k # ???????????????????????????????????????????????????
        end
        ImgArray[i,j,k]=PixelNumber[Pxijk]
      end
    end
  end
  #--------------------- Making it Gray Matrix ---------------------------------
  if NBytesPPx==0 # Suposedly "Normal Img or Color Img"
    #GrayArray=sum(ImgArray,3)/SPPx
    GrayArray=ImgArray[:,:,3]
  else
    GrayArray=ImgArray[:,:,1]*256+ImgArray[:,:,2] # Dantec Spec
  end
return GrayArray[:,:,1]
end

###########################################################################

# Loading a Serie of TIFF from Dantec Dynamics

function AquiSerieD(Folder::String, File0::String, NbrImg=189::Int64)
  # ========================== Making Img Path =================================
  # ----------------Cutting File0 (Dantec DynamicsStudio Naming)----------------
  j=1
  while File0[j]!='.'
    j=j+1
  end
  p1=j
  #println(p1)
  BaseName=File0[1:p1-1] # Input by User exporting Img
  j=j+1
  while File0[j]!='.'
    j=j+1
  end
  p2=j
  FileSerie=File0[p1+1:p2-1] #Auto Serie Numbering
  NumberInSerie=File0[p2+1:end] # Auto Img Numbering
  N=NbrImg-1 # Numbering Starts at 0 not 1
  ImgArray=[]
  OneAdress=[]
  # ------------------------ Making Image Number String -------------------------
  for i=0:N
    # Finding k
    LNIS=length(NumberInSerie)
    k=LNIS
    while floor(i/(10.0^k))<1 && k>=0
      k=k-1
    end
    # completing the NumberIn Serie String
    NIS=""
    ik=i
    for kk=LNIS:-1:1
      if kk>k+1
        NIS="$(NIS)0"
      else
        V=convert(Int64,floor(ik/(10^(kk-1))))
        NIS="$(NIS)$V"
        ik=ik-V*10^(kk-1)
      end
    end
    OneAdress="$(Folder)\\$(BaseName).$(FileSerie).$(NIS).tif"
    #println(OneAdress)
    # ========================= Aquiring Images ==================================
    OneImg=AquiTiff(OneAdress)
    if isempty(ImgArray)
      ImgArray=zeros(Int64,size(OneImg,1),size(OneImg,2),N+1)
      ImgArray[:,:,1]=OneImg
    else
      ImgArray[:,:,i+1]=OneImg
    end
  end
return ImgArray # to assure Julia wont print the Array
end
#############################################################################

# Loading a Serie of TIFF automaticaly renamed with Windows 10

function AquiSerieW(Folder::String, File0::String, NbrImg=189::Int64)
  # ========================== Making Img Path =================================
  # ----------------Cutting File0 (Dantec DynamicsStudio Naming)----------------
  j=1
  while File0[j]!='('
    j=j+1
  end
  p1=j
  #println(p1)
  BaseName=File0[1:p1-1] # Input by User + 1 space (std windows renumerotation)
  j=j+1

  ImgArray=[]
  OneAdress=[]
  # ------------------------ Making Image Number String -------------------------
  for i=1:NbrImg
    OneAdress="$(Folder)\\$(BaseName)($i).tif"
    #println(OneAdress)
    # ========================= Aquiring Images ==================================
    OneImg=AquiTiff(OneAdress)
    if isempty(ImgArray)
      ImgArray=zeros(Int64,size(OneImg,1),size(OneImg,2),NbrImg)
      ImgArray[:,:,1]=OneImg
    else
      ImgArray[:,:,i]=OneImg
    end
  end
return ImgArray # to assure Julia wont print the Array
end

#############################################################################

# Creating a Special type to stock the start of a line and its values
type Line # Creates type Line
  YValues::Vector
  XStart::Int64
end
##############################################################################

# Detecting the Lines in an image matrix (also retourning a Binary matrix)

function DetectLine(Img::Matrix)
  # only for bicolor ImgLine
  BiMat=zeros(Bool,size(Img))
  BiMat0=zeros(Bool,size(Img))
  Max=maximum(Img)
  min=minimum(Img)
  moy=(Max+min)/2
# --------------------------BiColoring----------------------------------
  k=0
  for i=1:size(Img,1) #BiLeveling
    for j=1:size(Img,2)
      if Img[i,j]<=moy
        BiMat[i,j]=0
        BiMat0[i,j]=0
      else
        BiMat[i,j]=1
        BiMat0[i,j]=1
      end
    end
  end

SumMat=sum(BiMat)
H=size(BiMat,1)
L=size(BiMat,2)
NPx=H*L
if SumMat<NPx/2
  # 1=Line, 0=Background
  LinePx=true
  BckGrnd=false #BckGrnd LinePx
else
  # 0=Line, 1=Background
  LinePx=false
  BckGrnd=true
end
CountLinePx=abs(SumMat-NPx) # nbr of Px that are part of a line
Lines=[] # need Array to stock the XStart
Xcount=1::Int64
Split=false
Ymean2=H
YmeanM1=H
Xcount2=1
while Xcount<L+1
  x=Xcount
 # 1 Line code  !
  if Split
    y=Ymean2
    x=Xcount2
    Xcount=Xcount-1
    Split=false
  else
    y=H
    x=Xcount
  end
  ConLine=true
  # ========================= Following the Line =========================
  while y>=2 && ConLine# going up to top
    # find [Background; Line]
    YmeanV=[]
    if [LinePx; BckGrnd]==[BiMat[y-1,x]; BiMat[y,x]] # Line Start Detect
      #Xcount=Xcount-1 #Stay in the same column and check further
      # Special Start in case of Split
      if YmeanM1 != H
        push!(YmeanV,YmeanM1)
        XStart=x-1
        YmeanM1=H
      else #End Spec Split
        XStart=x
      end
      y=y-1
      YStart=y
      # Start of a Line
      while BiMat[y,x]==LinePx
        y=y-1
      end
      YEnd=y+1
      # following that one line
      Width=abs(YStart-YEnd)+1
      BiMat[YEnd:YStart,x]=repmat([BckGrnd],Width,1)
      Ymean=(YStart+YEnd)/2
      push!(YmeanV,Ymean)
      while ConLine && x<L
        x=x+1
        if YStart<H && YEnd>1
          if sum(BiMat[YEnd-1:YStart+1,x].==repmat([BckGrnd],Width+2,1))==Width+2
            # Line Stop
            ConLine=false
            YStart=0
            YEnd=0
            x=Xcount
            Xcount=Xcount-1
          else
            # Line continue
            if BiMat[YEnd-1:YStart+1,x]==[BckGrnd;repmat([LinePx],Width,1);BckGrnd]
             # Line constant in Value and Width
            else
              # Line continue but changed
              if (BiMat[YStart:YStart+1,x]==[LinePx;LinePx])||(BiMat[YEnd-1:YEnd,x]==[LinePx;LinePx])
                # Line increase or decrease
                if (BiMat[YStart:YStart+1,x]==[LinePx;LinePx]) && !(BiMat[YEnd-1:YEnd,x]==[LinePx;LinePx])
                  # Decreasing
                  YEnd=YEnd+minimum(findin(BiMat[YEnd:YStart,x].==repmat([LinePx],Width,1),true))-1
                  yy=YStart
                  while BiMat[yy+1,x]==LinePx && yy<H
                    yy=yy+1
                  end
                  YStart=yy
                elseif !(BiMat[YStart:YStart+1,x]==[LinePx;LinePx]) && (BiMat[YEnd-1:YEnd,x]==[LinePx;LinePx])# Increasing
                  YStart=YEnd+maximum(findin(BiMat[YEnd:YStart,x].==repmat([LinePx],Width,1),true))-1
                  # YEnd
                  yy=YEnd
                  while BiMat[yy-1,x]==LinePx && yy>1
                    yy=yy-1
                  end
                  YEnd=yy
                elseif (BiMat[YStart:YStart+1,x]==[LinePx;LinePx]) && (BiMat[YEnd-1:YEnd,x]==[LinePx;LinePx]) # Increse in up and down
                  if sum(BiMat[YEnd-1:YStart+1,x].==LinePx)==(YStart+1-(YEnd-1)+1)# No hole
                    # Increse in Width in both side
                    ydown=YStart
                    while BiMat[ydown+1,x]==LinePx && ydown<H
                      ydown=ydown+1
                    end
                    YStart=ydown
                    yup=YEnd
                    while BiMat[yup-1,x]==LinePx && yup>1
                      yup=yup-1
                    end
                    YEnd=yup
                  else # there is a hole
                    # Split in two curves
                    Split= true
                    PosHole=findin(BiMat[YEnd-1:YStart+1,x].==BckGrnd,true)
                    # Down Part Continues
                    YEnd=(YEnd-1+minimum(PosHole))-1 # (below hole)
                    ydown=YStart
                    while BiMat[ydown+1,x]==LinePx && ydown<H
                      ydown=ydown+1
                    end
                    YStart=ydown
                    # Up Going part
                    YStart2=(YEnd-1+maximum(PosHole))+1
                    Ymean2=YStart2+1
                    Xcount2=x
                    YmeanM1=Ymean # kept for next line
                  end
                end
              else
                #Line Width decrease within previous limits
                PxPos=findin(BiMat[YEnd:YStart,x].==repmat([LinePx],Width,1),true)
                minPxPos=minimum(PxPos)
                MaxPxPos=maximum(PxPos)
                YStart=YEnd+MaxPxPos-1
                YEnd=YEnd+minPxPos-1
              end
            end
          end
        else
          # border effect
        end
        if !ConLine
          Width=0
        else
          Width=abs(YStart-YEnd)+1
          BiMat[YEnd:YStart,x]=repmat([BckGrnd],Width,1)
          Ymean=(YEnd+YStart)/2
          push!(YmeanV,Ymean)
        end
      end #start of a line detected
    else
      y=y-1 # No Line, go on climbing
    end
    if length(YmeanV)>1 #!isempty(YmeanV)
      Line1=Line(H+1-YmeanV,XStart) # Assuming left bottom axis =0
      #then complete Lines
      push!(Lines,Line1)
      ConLine=false # end current line
    end
  end
    Xcount=Xcount+1
end
# from bottom to top
# 1 First column -> detect LinePx
# 1b if no LinePx go to the right
# 2 continue climbing until Px!=LinePx
# 3 from there moove to the right following the line
# 3a check same Px and Px+1 and Px-1
# 3b if Px+-1 check also Px+-2, etc
#    if no Px: line stop
# 3c repeat 3 until line stop
# 4 remoove selected Px
# 5 repeat operation until top right
return BiMat0, Lines
end

#############################################################################

# Allowing to plot the new type created, eventually on the matrix (thanks to PyPlot)

function PlotLine(Lines::Array{Any},H=0::Int64)
  for i=1:length(Lines)
    XS=Lines[i].XStart
    YV=Lines[i].YValues
    if H==0
      plot(XS:XS+length(YV)-1,YV,"-")
    else
      plot(XS-1:XS-1+length(YV)-1,H-YV,"-")
    end
  end
end

############################################################################

# Creating the next function for the Multiple Regression Function below

function NextFun(Xin::Vector,RegFuns="Poly"::String,i=1::Int64)
  n=length(Xin)
  if RegFuns=="Poly"
    Xout=Xin.^(i)
    XT=("Poly Order $i")
  elseif RegFuns=="Trigo"
    if isinteger(i/2)
      Xout=cos.(Xin/maximum(Xin)*(i/2))
      XT=("Standirdize cos($i x/2)")
    else
      Xout=cos.(Xin/maximum(Xin)/((i+1)/2))
      XT=("Standirdize cos($(1/(i+1)) x/2)")
    end
  elseif RegFuns=="Frac"
    F=findin(Xin,0)
    for j=1:length(F)
      X[F[j]]=mean(abs(Xin))*10^-5
    end
    Xout=1./Xin.^i
    if abs(minimum(Xout))< 2e-15 # all becoming too small
      println("Elements becoming too small X=ones")
      Xout=ones(size(Xin))
      XT=("Ones")
    else
      XT=("Frac Order $i")
    end
  elseif RegFuns=="Expo"
    Xout=exp.(i*Xin/maximum(Xin))
    XT=("Standardize exp($i x)")
  elseif RegFuns=="Loga"
    Xout=log.(abs(Xin+(i-2)*sqrt(maximum(Xin))))
    XT=("log(x+$(i-2) sqrt(Max))")
  elseif RegFuns=="Mixte"
    NFam=5 #nbr of family used (code written could be improoved)
    if isinteger((i+NFam-1)/NFam)
      k=(i+NFam-1)/NFam
      Xout=Xin.^k
      XT=("Poly Order $k") #Poly
    elseif isinteger((i+NFam-2)/NFam)
      k=(i+NFam-2)/NFam
      F=findin(Xin,0)
      for j=1:length(F)
        X[F[j]]=mean(abs(Xin))*10^-5
      end
      Xout=1./Xin.^k
      if abs(minimum(Xout))< 2e-15 # all becoming too small
        println("Elements becoming too small X=rand")
        Xout=rand(size(Xin))
      else
        XT=("Frac Order $k")
      end #Frac
    elseif isinteger((i+NFam-3)/NFam)
      k=(i+NFam-3)/NFam
      #NTrigo=4
      if isinteger(k/2)
        Xout=cos.(Xin/maximum(Xin)*(k/2))
        XT=("Standirdize cos($k x/2)")
      else
        Xout=cos.(Xin/maximum(Xin)/(k+1)/2)
        XT=("Standirdize cos($(1/(k+1)) x/2)")
      end #Trigo
    elseif isinteger((i+NFam-4)/NFam)
      k=(i+NFam-3)/NFam
      Xout=exp.(k*Xin/maximum(Xin))
      XT=("Standardize exp($k x)") #Expo
    elseif isinteger((i+NFam-5)/NFam)
      k=(i+NFam-5)/NFam
      Xout=log.(abs(Xin+(k-2)*sqrt(maximum(Xin))))
      XT=("log(x+$(k-2) sqrt(Max))")
    end
  elseif RegFuns=="Custom" # Would their be another interesting family
    println("What next fun do you want ? (Existing type: Poly, Trigo, Frac, Expo or Loga) \n Your choice is :")
    SwCu=input() #Switch Custom
    if SwCu=="Poly"
      println("Order of the wanted term : y=x^{Order} ? \n Order=")
      u=parse(Float64,input())
      if isinteger(u)
        Xu=Xin
      else
        Xu=Xin
        Xu[find(Xu.<0)]=zeros(size(find(Xu.<0)))
        println("All smaller than 0 elements have been replaced by zero for Poly Order $u")
      end
      Xout=Xu.^(u)
      XT=("Poly Order $u")
    elseif SwCu=="Trigo"
      println("Normal or Inverse ? \n Your choice is :")
      TTrig=input()
      if TTrig=="Normal"
        println("What type do you want ? (Possible choices: cos, sin, tan or cot) \n Your choice is :")
        TrigN=input()
        if TrigN=="sin"
          pritnln("What Angulat fequency (omega) do you want ? \n omega =")
          omega=parse(Float64,input())
          println("What Dephasage (phi) do you want ? \n phi =")
          phi=parse(Float64,input())
          Xout=sin.(omega*Xin+phi)
          XT=("sin($omega x+$phi)")
        elseif TrigN=="cos"
          pritnln("What Angulat fequency (omega) do you want ? \n omega =")
          omega=parse(Float64,input())
          println("What Dephasage (phi) do you want ? \n phi =")
          phi=parse(Float64,input())
          Xout=cos.(omega*Xin+phi)
          XT=("cos($omega x+$phi)")
        elseif TrigN=="tan"
          pritnln("What Angulat fequency (omega) do you want ? \n omega =")
          omega=parse(Float64,input())
          println("What Dephasage (phi) do you want ? \n phi =")
          phi=parse(Float64,input())
          Xout=tan.(omega*Xin+phi)
          XT=("tan($omega x+$phi)")
        elseif TrigN=="cot"
          pritnln("What Angulat fequency (omega) do you want ? \n omega =")
          omega=parse(Float64,input())
          println("What Dephasage (phi) do you want ? \n phi =")
          phi=parse(Float64,input())
          Xout=cot.(omega*Xin+phi)
          XT=("cot($omega x+$phi)")
        end
      elseif TTrig=="Inverse"
        println("What type do you want ? (Possible choices: acos, asin, atan or acot) \n Your choice is :")
        TrigN=input()
        if TrigN=="asin"
          pritnln("Vector is standardize (Divided by Max)")
          println("f ? : asin(XStd/f) (abs(f)>1)? \n f=")
          f=parse(Float64,input())
          if abs(f)<1
            f=1
          end
          Xout=asin.(Xin/maximum(abs(Xin))/f)
          XT=("asin(XStd/$f)")
        elseif TrigN=="acos"
          pritnln("Vector is standardize (Divided by Max)")
          println("f ? : acos(XStd/f) (abs(f)>1)? \n f=")
          f=parse(Float64,input())
          if abs(f)<1
            f=1
          end
          Xout=acos.(Xin/maximum(abs(Xin))/f)
          XT=("acos(XStd/$f)")
        elseif TrigN=="atan"
          println("What Factor f ? (atan(fX+t)) \n f=")
          f=parse(Float64,input())
          println("What terme ? \n t=")
          t=parse(Float64,input())
          Xout=atan.(f*Xin+t)
          XT=("atan(fX+t)")
        elseif TrigN=="acot"
          println("What Factor f ? (acot(fX+t)) \n f=")
          f=parse(Float64,input())
          println("What terme ? \n t=")
          t=parse(Float64,input())
          Xout=acot.(f*Xin+t)
          XT=("acot(fX+t)")
        end
      end
    elseif SwCu=="Frac"
      println("Order of the wanted term : y=x^-{Order} ? \n Order=")
      u=parse(Float64,input())
      if isinteger(u)
        Xu=Xin
      else
        Xu=Xin
        Xu[find(Xu.<=0)]=ones(size(find(Xu.<0)))
        println("All smaller than or equal to 0 elements have been replaced by one for Frac Order $u")
      end
      Xout=1./(Xu.^(u))
      XT=("Frac Order $u")
    elseif SwCu=="Expo"
      println("Order of the wanted Exponential ? : y=exp(x*{Order}) ? \n Order=")
      u=parse(Float64,input())
      Xout=exp.(u*Xin)
      XT=("exp($u x)")
    elseif SwCu=="Loga"
      println("Terme of the wanted Logarithme ? : y=log|x+t|) ? \n t=")
      t=parse(Float64,input())
      Xout=log.(abs(Xin+t))
      XT=("log(x+t)")
    end
  end
return Xout,XT
end

###############################################################################

# Statistics discrabing the Regression using ANOVA analysis thanks to distributions
# functions

function StatReg(Xin,Yin,XMat,XName,p)
  Yb=mean(Yin)
  SquareX=(transpose(XMat)*XMat)
  n=length(Yin)
  Bh=(SquareX^-1)*transpose(XMat)*Yin # Estimator for Inv(Mat) ?
  Yh=XMat*Bh
  SSE=sum((Yin-Yh).^2)
  SSR=sum((Yh-Yb).^2)
  SSTot=sum((Yin-Yb).^2)
  R2=SSR/SSTot # the biggest the best
  dfR=p-1
  MSR=SSR/dfR
  dfE=n-p
  MSE=SSE/dfE
  F=MSE/MSR
  # ---------------------- Calc p-Value Model --------------------------
  FD=FDist(dfR,dfE)
  pModel=cdf(FD,F) #Smallest the best
  # p- Values indie
  TD=TDist(dfE)
  pVect=ones(size(Bh))
  for i=1:length(Bh)
    Yh=XMat[:,i]*Bh[i]
    SSE=sum((Yin-Yh).^2)
    if XName[i]=="Constant"
      SX2=sum((Xin-mean(Xin)).^2)
      SEBh=sqrt(SSE*(1/n+mean(Xin)^2/SX2))
    else
      X=XMat[:,i]
      SX2=sum((X-mean(X)).^2)
      SEBh=sqrt(SSE/SX2)
    end
    t=Bh[i]*sqrt(dfE)/SEBh
    pVar=2*(1-cdf(TD,t))
    pVect[i]=pVar
  end

  return pVect,pModel,Bh
end

#############################################################################

# Main function of Multiple Regression implementing ANOVA tests
# and a user friendly approach to regression

function MultiRegOpt(Xin::Vector, Yin::Vector, FunFamily="Poly"::String, p=3::Int64,Stepwise="None"::String, threshold=0.05::Float64)
  n=length(Xin)
  pVModels=[]
  XMatArray=[]
  BArray=[]
  YhA=[]
  XMatA=[]
  pVarA=[]
  XNameA=[]
  NbrModelTested=0
  if FunFamily=="Custom"
    NbrModel=p
    XMat=ones(Float64,n,NbrModel)
    XName=[]
    for Counti=0:NbrModel-1 # Nombre of function involved
      if Counti==0
        FX=ones(size(Xin))
        push!(XName,"Constant")
      else
        Xout=NextFun(Xin,FunFamily,Counti)
        FX=Xout[1]
        push!(XName,Xout[2])
      end
      XMat[:,Counti+1]=FX
    end
    pVect,pModel,Bh=StatReg(Xin,Yin,XMat,XName,p)
    # Stepwise, taking out not necessary variables
    OldXMat=XMat
    OldpVect=pVect
    OldBh=Bh
    OldXMat=XMat
    OldpModel=pModel
    while sum(pVect.>threshold)>0 && Stepwise=="Student"# There is pVect>0.05
      i=findmax(pVect)[2] #eliminated fun
      if (NbrModel-1)==0
        println("Impossible to find a acceptable regression with given number of function for threshold=$threshold, First Result given")
        XMat=OldXMat
        pVect=OldpVect
        Bh=OldBh
        XMat=OldXMat
        break
      else
        NewXMat=ones(Float64,n,NbrModel-1)
        NewXName=[]
        for j=1:NbrModel
          if j<i
            NewXMat[:,j]=XMat[:,j]
            push!(NewXName,XName[j])
          elseif j>i
            NewXMat[:,j-1]=XMat[:,j]
            push!(NewXName,XName[j])
          end
        end
        println("MultiReg New:NewXName=$NewXName")
        pVect,pModel,Bh=StatReg(Xin,Yin,NewXMat,NewXName,p)
        XMat=NewXMat
        XName=NewXName
        NbrModel=NbrModel-1
      end
    end
    push!(pVarA,pVect)
    push!(pVModels,pModel)
    push!(BArray,Bh)
    push!(XMatA,XMat)
    push!(XNameA,XName)
  else
    for NbrModel=1:p
      XMat=ones(Float64,n,NbrModel)
      XName=[]
      for Counti=0:NbrModel-1 # Nombre of function involved
        if Counti==0
          FX=ones(size(Xin))
          push!(XName,"Constant")
        else
          Xout=NextFun(Xin,FunFamily,Counti)
          FX=Xout[1]
          push!(XName,Xout[2])
        end
        XMat[:,Counti+1]=FX
      end
      pVect,pModel,Bh=StatReg(Xin,Yin,XMat,XName,p)
      # Stepwise, taking out not necessary variables
      OldpVect=pVect
      OldBh=Bh
      OldXMat=XMat
      OldpModel=pModel
      while sum(pVect.>threshold)>0 && Stepwise=="Student"# there is pVect>0.05
        i=findmax(pVect)[2] #eliminated fun
        if (NbrModel-1)==0
          println("Impossible to find a acceptable regression with given number of function for threshold=$threshold, First Result given")
          XMat=OldXMat
          pVect=OldpVect
          Bh=OldBh
          XMat=OldXMat
          break
        else
          NewXMat=ones(Float64,n,NbrModel-1)
          NewXName=[]
          for j=1:NbrModel
            if j<i
              NewXMat[:,j]=XMat[:,j]
              push!(NewXName,XName[j])
            elseif j>i
              NewXMat[:,j-1]=XMat[:,j]
              push!(NewXName,XName[j])
            end
          end
          pVect,pModel,Bh=StatReg(Xin,Yin,NewXMat,NewXName,p)
          XMat=NewXMat
          XName=NewXName
          NbrModel=NbrModel-1
        end
      end
      if Stepwise=="Fleussu"
        Yb=mean(Yin)
        if sum(abs(Bh).<Yb*10.0^-5)>0
          println(abs(Bh).<Yb*10.0^-5)
          Bh[abs(Bh).<Yb*10.0^-5]=maximum(Bh)*2
          i=findin(X,maximum(X))
          NewXName=[]
          NewBh=[]
          for j=1:length(Bh)
            if sum(findin(i,j))==0
              push!(NewXName,XName[j])
              push!(NewBh,Bh[j])
            end
          end
          XName=NewXName
          Bh=NewBh
          NbrModel=length(Bh)
        end
      end
      push!(pVarA,pVect)
      push!(pVModels,pModel)
      push!(BArray,Bh)
      push!(XMatA,XMat)
      push!(XNameA,XName)
    end
  end
  return BArray,pVModels,XMatA,pVarA,XNameA
end

############################################################################
end # module


const DevicesModelContainer = Dict{Symbol, DeviceModel}
const BranchModelContainer = Dict{Symbol, DeviceModelForBranches}
const ServicesModelContainer = Dict{Tuple{String, Symbol}, ServiceModel}

"""
    OperationsProblemTemplate(::Type{T}) where {T<:PM.AbstractPowerFormulation}
Creates a model reference of the PowerSimulations Optimization Problem.
# Arguments
- `model::Type{T<:PM.AbstractPowerFormulation}`:
# Example
```julia
template = OperationsProblemTemplate(CopperPlatePowerModel)
```
"""
mutable struct OperationsProblemTemplate
    transmission::Type{<:PM.AbstractPowerModel}
    devices::DevicesModelContainer
    branches::BranchModelContainer
    services::ServicesModelContainer
    function OperationsProblemTemplate(::Type{T}) where {T <: PM.AbstractPowerModel}
        new(T, DevicesModelContainer(), BranchModelContainer(), ServicesModelContainer())
    end
end

OperationsProblemTemplate() = OperationsProblemTemplate(CopperPlatePowerModel)
# TODO: make getter functions here
# Note: use the file test_operations_template to test the getter functions
get_transmission_model(template::OperationsProblemTemplate) = template.transmission

# Note to devs. PSY exports set_model! these names are choosen to avoid name clashes

"""Sets the transmission model in a template"""
function set_transmission_model!(
    template::OperationsProblemTemplate,
    model::Type{<:PM.AbstractPowerModel},
)
    template.transmission = model
    return
end

"""
    Sets the device model in a template using the component type and formulation.
    Builds a default DeviceModel
"""
function set_device_model!(
    template::OperationsProblemTemplate,
    component_type::Type{<:PSY.Device},
    formulation::Type{<:AbstractDeviceFormulation},
)
    set_device_model!(template, DeviceModel(component_type, formulation))
    return
end

"""
    Sets the device model in a template using a DeviceModel instance
"""
function set_device_model!(
    template::OperationsProblemTemplate,
    model::DeviceModel{<:PSY.Device, <:AbstractDeviceFormulation},
)
    _set_model!(template.devices, model)
    return
end

function set_device_model!(
    template::OperationsProblemTemplate,
    model::DeviceModel{<:PSY.Branch, <:AbstractDeviceFormulation},
)
    _set_model!(template.branches, model)
    return
end

"""
    Sets the service model in a template using a name and the service type and formulation. Builds a default ServiceModel with use_service_name set to true.
"""
function set_service_model!(
    template::OperationsProblemTemplate,
    service_name::String,
    service_type::Type{<:PSY.Service},
    formulation::Type{<:AbstractServiceFormulation},
)
    set_service_model!(
        template,
        service_name,
        ServiceModel(service_type, formulation, use_service_name = true),
    )
    return
end

"""
    Sets the service model in a template using a ServiceModel instance.
"""
function set_service_model!(
    template::OperationsProblemTemplate,
    service_type::Type{<:PSY.Service},
    formulation::Type{<:AbstractServiceFormulation},
)
    set_service_model!(template, ServiceModel(service_type, formulation))
    return
end

function set_service_model!(
    template::OperationsProblemTemplate,
    service_name::String,
    model::ServiceModel{<:PSY.Service, <:AbstractServiceFormulation},
)
    _set_model!(template.services, service_name, model)
    return
end

function set_service_model!(
    template::OperationsProblemTemplate,
    model::ServiceModel{<:PSY.Service, <:AbstractServiceFormulation},
)
    _set_model!(template.services, model)
    return
end


# TODO move this constructor to mcts.jl
function MCTS.Env(gspec::AbstractGameSpec, oracle, params::MctsParams; S=nothing)
	return MCTS.Env(gspec, oracle;
	gamma=params.gamma,
	cpuct=params.cpuct,
	noise_ϵ=params.dirichlet_noise_ϵ,
	noise_α=params.dirichlet_noise_α,
	prior_temperature=params.prior_temperature,
	S=S)
end

"""
		MuPlayer{P,D,R} <: AbstractPlayer

- `prediction_oracle` 		f(sᵏ) -> (pᵏ,vᵏ)
- `dynamics_oracle`				g(sᵏ⁻¹,aᵏ) -> (rᵏ,sᵏ)
- `representation_oracle` h(o) -> (s⁰)
- `mcts_params::MctsParams`
- `timeout::Union{Float64, Nothing}` time that MCTS has for thinking
		nothing means infinite (till num_iters_per_turn ends)
"""
# struct MuPlayer{P,D,R} <: AbstractPlayer
struct MuPlayer{I,R} <: AbstractPlayer
	# mcts :: M
	# prediction_oracle :: P
	# dynamics_oracle :: D
	# representation_oracle :: R
	initial_inference :: I
	recurrent_inference :: R
	mcts_params :: MctsParams
	timeout :: Union{Float64, Nothing}
	# niters :: Int
	# τ :: AbstractSchedule{Float64} # Temperature
	# function MuPlayer(mcts::MCTS.Env, representation_oracle, dynamics_oracle; τ, niters, timeout=nothing)
	function MuPlayer((init, recur), mcts_params::MctsParams; timeout=nothing)
		@assert mcts_params.num_iters_per_turn > 0
		@assert isnothing(timeout) || timeout > 0
		new{typeof(init), typeof(recur)}(init, recur, mcts_params, timeout)
	end
end

# # constructor with MuNetwork
# function MuPlayer(
# 	μnetwork::Union{MuNetwork,NamedTuple}, params::MctsParams; timeout=nothing)
#   return MuPlayer(μnetwork.f, μnetwork.g, μnetwork.h, params;
# 	# niters=params.num_iters_per_turn,
# 	# τ=params.temperature,
# 	timeout=timeout)
# end

function AlphaZero.player_temperature(p::MuPlayer, game, turn)
	return p.mcts_params.temperature[turn]
end

# function reset_player!(player::Union{MctsPlayer,MuPlayer})
# 	MCTS.reset!(player.mcts)
# end

# normalize policy - probs of legal actions sum up to 1, and illegals are 0
function normalize_p(P, actions_mask)
	P = P .* actions_mask # Zygote doesn't work with .*=
	sp = sum(P, dims=1)
	P = P ./ (sp .+ eps(eltype(P)))
	return P
end

# ? does rootinfo.vest should be there, or just
# ? sum without γ
# compute_rootvalue(rootinfo::MCTS.StateInfo, γ) = rootinfo.Vest + γ*sum([st.W for st in rootinfo.stats])
# compute_rootvalue(ri::MCTS.StateInfo, γ) = max(st.W/st.N for st in ri.stats)
compute_rootvalue(ri::MCTS.StateInfo) = sum(st.W for st in ri.stats) / MCTS.Ntot(ri)
# TODO compare max and sum

"""
		think(::MuPlayer, game)

Return avaliable actions, policy, and rootvalue
"""
function AlphaZero.think(plr::MuPlayer, game)
	#initial inference:  h(o) → s⁰,  f(s⁰) → (p⁰,v⁰)
	# rootstate = p.representation_oracle(GI.current_state(game))
	# (P⁰, V⁰) = p.prediction_oracle(rootstate)
	p⁰, v⁰, s⁰ = plr.initial_inference(GI.current_state(game))
	actions_mask = GI.actions_mask(game)
	p⁰ = normalize_p(p⁰, actions_mask)[actions_mask]

	recur_res = Ref((p⁰, v⁰, s⁰, zero(v⁰))) # g and f are distincted in mcts.jl, so there is need to pass by reference (or some kind of channel)
	prediction_recurrent_oracle(state) = state==recur_res[][3] ? recur_res[][1:2] : @error "states don't match" state recur_res[][3]

	mcts = MCTS.Env(GI.spec(game), prediction_recurrent_oracle, plr.mcts_params, S=typeof(s⁰))
	mcts.tree[s⁰] = MCTS.init_state_info(p⁰,v⁰,mcts.prior_temperature)

	mugame = MuGameEnvWrapper(
		game,
		#	CachedOracle(oracle, Statetype, Actiontype, Rewardtype)
		CachedOracle(plr.recurrent_inference,typeof(recur_res[]),typeof(s⁰),GI.action_type(mcts.gspec)),
		s⁰,
		true, #isroot
		GI.white_playing(game),
		0f0,
		recur_res)
	niters = plr.mcts_params.num_iters_per_turn - 1 # -1, because (P⁰,V⁰) is already there
	if isnothing(plr.timeout) # Fixed number of MCTS simulations
		MCTS.explore!(mcts, mugame, niters)
	else # Run simulations until timeout
		start = time()
		while time() - start < plr.timeout
		MCTS.explore!(mcts, mugame, niters)
		end
	end
	rootvalue = compute_rootvalue(mcts.tree[s⁰])
	actions, π_target = MCTS.policy(mcts, mugame)
	return actions, π_target, rootvalue
end

"""
		play_game(gspec, ::MuPlayer)

overloaded AlphaZero.play_game() function,
that additionally push selected action, and rootvalue into trace
"""
function AlphaZero.play_game(gspec, player::Union{MuPlayer,MinMax.Player,MctsPlayer}; flip_probability=0., initstate=nothing)
  game = isnothing(initstate) ? GI.init(gspec) : GI.init(gspec, initstate)
	# @debug "initstate" initstate GI.current_state(game)
  trace = MuTrace(GI.current_state(game))
  while true
    if GI.game_terminated(game)
      return trace
    end
    if !iszero(flip_probability) && rand() < flip_probability
      GI.apply_random_symmetry!(game)
    end
    actions, π_target, rootvalue = think(player, game)
		rootvalue = GI.white_playing(game) ? rootvalue : -rootvalue# ? reversing rootvalue wp?
    τ = AlphaZero.player_temperature(player, game, length(trace))
    π_sample = apply_temperature(π_target, τ)
    a = actions[Util.rand_categorical(π_sample)]
    GI.play!(game, a)
    push!(trace, π_target, GI.white_reward(game), GI.current_state(game), a, rootvalue) # pᵏ, rᵏ, sᵏ, aᵏ, vᵏ
  end
end


# export PARAMETER_DEFAULT, PARAMETER_UNSTABLE_BUT_FAST, PARAMETER_STABLE_BUT_SLOW

using MathOptInterface
MOI = MathOptInterface
const MOIU = MOI.Utilities
const AFFEQ{T} = MOI.ConstraintIndex{MOI.ScalarAffineFunction{T}, MOI.EqualTo{T}}

abstract type AbstractBlockMatrix{T} <: AbstractMatrix{T} end
function nblocks end
function block end

function Base.size(bm::AbstractBlockMatrix)
    n = mapreduce(blk -> LinearAlgebra.checksquare(block(bm, blk)),
                  +, 1:nblocks(bm), init=0)
    return (n, n)
end
function Base.getindex(bm::AbstractBlockMatrix, i::Integer, j::Integer)
    (i < 0 || j < 0) && throw(BoundsError(i, j))
    for k in 1:nblocks(bm)
        blk = block(bm, k)
        n = size(blk, 1)
        if i <= n && j <= n
            return blk[i, j]
        elseif i <= n || j <= n
            return 0
        else
            i -= n
            j -= n
        end
    end
    i, j = (i, j) .+ size(bm)
    throw(BoundsError(i, j))
end
Base.getindex(A::AbstractBlockMatrix, I::Tuple) = getindex(A, I...)

abstract type BlockSolution{T} <: AbstractBlockMatrix{T} end
struct PrimalSolution{T} <: BlockSolution{T}
    blocks::Vector{Matrix{T}}
end
# getptr(X::PrimalSolution, blk) = getResultYMat(X.problem, blk)
struct VarDualSolution{T} <: BlockSolution{T}
    blocks::Vector{Matrix{T}}
end
# getptr(X::VarDualSolution, blk) = getResultXMat(X.problem, blk)
nblocks(X::BlockSolution) = length(X.blocks)
function block(X::BlockSolution, blk::Integer)
    return X.blocks[blk]
end
# Needed by MPB_wrapper
function Base.getindex(A::BlockSolution, i::Integer)
    block(A, i)
end

mutable struct Optimizer{T} <: MOI.AbstractOptimizer
    objconstant::T
    objsign::Int
    blockdims::Vector{Int}
    varmap::Vector{Tuple{Int, Int, Int}} # Variable Index vi -> blk, i, j
    b::Vector{T}
    solve_time::Float64
    silent::Bool
    options::Dict{Symbol, Any}
    y::Vector{T}
    X::PrimalSolution{T}
    Z::VarDualSolution{T}
    primalobj::T
    dualobj::T
    phasevalue::Symbol
    tempfile::String
    elemdata::Vector{Any}
	presolve::Bool
    function Optimizer{T}(; presolve::Bool = true) where T
		optimizer = new(
            zero(T), 1, Int[], Tuple{Int, Int, Int}[], T[],
            NaN, false, Dict{Symbol, Any}(), T[], PrimalSolution{T}(Matrix{T}[]), VarDualSolution{T}(Matrix{T}[]), zero(T), zero(T), :noINFO, mktempdir(), [], true)
		if !presolve
			optimizer.presolve = false
		end
		return optimizer
    end
end

varmap(optimizer::Optimizer, vi::MOI.VariableIndex) = optimizer.varmap[vi.value]

function MOI.supports(optimizer::Optimizer, param::MOI.RawParameter)
	return param.name in keys(SET_PARAM)
end
function MOI.set(optimizer::Optimizer, param::MOI.RawParameter, value)
	if !MOI.supports(optimizer, param)
		throw(MOI.UnsupportedAttribute(param))
	end
	optimizer.options[param.name] = value
end
function MOI.get(optimizer::Optimizer, param::MOI.RawParameter)
	# TODO: This gives a poor error message if the name of the parameter is invalid.
	return optimizer.options[param.name]
end

MOI.supports(::Optimizer, ::MOI.Silent) = true
function MOI.set(optimizer::Optimizer, ::MOI.Silent, value::Bool)
	optimizer.silent = value
end
MOI.get(optimizer::Optimizer, ::MOI.Silent) = optimizer.silent

MOI.get(::Optimizer, ::MOI.SolverName) = "SDPA-GMP"

# See https://www.researchgate.net/publication/247456489_SDPA_SemiDefinite_Programming_Algorithm_User's_Manual_-_Version_600
# "SDPA (SemiDefinite Programming Algorithm) User's Manual — Version 6.00" Section 6.2
const RAW_STATUS = Dict(
    :noINFO        => "The iteration has exceeded the maxIteration and stopped with no informationon the primal feasibility and the dual feasibility.",
    :pdOPT => "The normal termination yielding both primal and dual approximate optimal solutions.",
    :pFEAS => "The primal problem got feasible but the iteration has exceeded the maxIteration and stopped.",
    :dFEAS => "The dual problem got feasible but the iteration has exceeded the maxIteration and stopped.",
    :pdFEAS => "Both primal problem and the dual problem got feasible, but the iterationhas exceeded the maxIteration and stopped.",
    :pdINF => "At least one of the primal problem and the dual problem is expected to be infeasible.",
    :pFEAS_dINF => "The primal problem has become feasible but the dual problem is expected to be infeasible.",
    :pINF_dFEAS => "The dual problem has become feasible but the primal problem is expected to be infeasible.",
    :pUNBD => "The primal problem is expected to be unbounded.",
    :dUNBD => "The dual problem is expected to be unbounded.")

function MOI.get(optimizer::Optimizer, ::MOI.RawStatusString)
	return RAW_STATUS[optimizer.phasevalue]
end
function MOI.get(optimizer::Optimizer, ::MOI.SolveTime)
	return optimizer.solve_time
end

function MOI.is_empty(optimizer::Optimizer)
    return iszero(optimizer.objconstant) &&
        optimizer.objsign == 1 &&
        isempty(optimizer.blockdims) &&
        isempty(optimizer.varmap) &&
        isempty(optimizer.b) &&
        optimizer.elemdata == []
end
function MOI.empty!(optimizer::Optimizer{T}) where T
    optimizer.objconstant = zero(Cdouble)
    optimizer.objsign = 1
    empty!(optimizer.blockdims)
    empty!(optimizer.varmap)
    empty!(optimizer.b)
    optimizer.X = PrimalSolution{T}(Matrix{T}[])
    optimizer.Z = VarDualSolution{T}(Matrix{T}[])
    optimizer.y = T[]
    optimizer.phasevalue = :noINFO
    optimizer.tempfile = mktempdir()
    optimizer.elemdata = []
    optimizer.primalobj = zero(T)
    optimizer.dualobj = zero(T)
end

function MOI.supports(
    optimizer::Optimizer,
    ::Union{MOI.ObjectiveSense,
            MOI.ObjectiveFunction{<:Union{MOI.SingleVariable,
                                          MOI.ScalarAffineFunction{T}}}}) where T
    return true
end

function MOI.supports_constraint(
    ::Optimizer, ::Type{MOI.VectorOfVariables}, ::Type{MOI.Reals})
    return false
end
const SupportedSets = Union{MOI.Nonnegatives, MOI.PositiveSemidefiniteConeTriangle}
function MOI.supports_constraint(
    ::Optimizer, ::Type{MOI.VectorOfVariables},
    ::Type{<:SupportedSets})
    return true
end
function MOI.supports_constraint(
    ::Optimizer, ::Type{MOI.ScalarAffineFunction{T}},
    ::Type{MOI.EqualTo{T}}) where T
    return true
end

function MOI.copy_to(dest::Optimizer, src::MOI.ModelLike; kws...)
    return MOIU.automatic_copy_to(dest, src; kws...)
end
MOIU.supports_allocate_load(::Optimizer, copy_names::Bool) = !copy_names

function MOIU.allocate(optimizer::Optimizer, ::MOI.ObjectiveSense, sense::MOI.OptimizationSense)
    # To be sure that it is done before load(optimizer, ::ObjectiveFunction, ...), we do it in allocate
    optimizer.objsign = sense == MOI.MIN_SENSE ? -1 : 1
end
function MOIU.allocate(::Optimizer, ::MOI.ObjectiveFunction, ::Union{MOI.SingleVariable, MOI.ScalarAffineFunction}) end

function MOIU.load(::Optimizer, ::MOI.ObjectiveSense, ::MOI.OptimizationSense) end
# Loads objective coefficient α * vi
function load_objective_term!(optimizer::Optimizer{T}, α, vi::MOI.VariableIndex) where {T}
    blk, i, j = varmap(optimizer, vi)
    coef = optimizer.objsign * α
    if i != j
        coef /= 2
    end
    # in SDP format, it is max and in MPB Conic format it is min
    inputElement(optimizer, 0, blk, i, j, convert(T, coef))
end
function MOIU.load(optimizer::Optimizer, ::MOI.ObjectiveFunction, f::MOI.ScalarAffineFunction)
    obj = MOIU.canonical(f)
    optimizer.objconstant = f.constant
    for t in obj.terms
        if !iszero(t.coefficient)
            load_objective_term!(optimizer, t.coefficient, t.variable_index)
        end
    end
end
function MOIU.load(optimizer::Optimizer{T}, ::MOI.ObjectiveFunction, f::MOI.SingleVariable) where T
    load_objective_term!(optimizer, one(T), f.variable)
end

function new_block(optimizer::Optimizer, set::MOI.Nonnegatives)
    push!(optimizer.blockdims, -MOI.dimension(set))
    blk = length(optimizer.blockdims)
    for i in 1:MOI.dimension(set)
        push!(optimizer.varmap, (blk, i, i))
    end
end

function new_block(optimizer::Optimizer, set::MOI.PositiveSemidefiniteConeTriangle)
    push!(optimizer.blockdims, set.side_dimension)
    blk = length(optimizer.blockdims)
    for i in 1:set.side_dimension
        for j in 1:i
            push!(optimizer.varmap, (blk, i, j))
        end
    end
end

function MOIU.allocate_constrained_variables(optimizer::Optimizer,
                                             set::SupportedSets)
    offset = length(optimizer.varmap)
    new_block(optimizer, set)
    ci = MOI.ConstraintIndex{MOI.VectorOfVariables, typeof(set)}(offset + 1)
    return [MOI.VariableIndex(i) for i in offset .+ (1:MOI.dimension(set))], ci
end

function MOIU.load_constrained_variables(
    optimizer::Optimizer, vis::Vector{MOI.VariableIndex},
    ci::MOI.ConstraintIndex{MOI.VectorOfVariables},
    set::SupportedSets)
end

function MOIU.allocate_variables(model::Optimizer, nvars)
end

function MOIU.load_variables(optimizer::Optimizer{T}, nvars) where T
    @assert nvars == length(optimizer.varmap)
    dummy = isempty(optimizer.b)
    if dummy
        optimizer.b = [one(T)]
        optimizer.blockdims = [optimizer.blockdims; -1]
    end
    if dummy
        inputElement(optimizer, 1, length(optimizer.blockdims), 1, 1, one(T))
    end
end

function MOIU.allocate_constraint(optimizer::Optimizer{T},
                                  func::MOI.ScalarAffineFunction,
                                  set::MOI.EqualTo) where T
    push!(optimizer.b, MOI.constant(set))
    return AFFEQ{T}(length(optimizer.b))
end

function MOIU.load_constraint(m::Optimizer{T}, ci::AFFEQ,
                              f::MOI.ScalarAffineFunction, s::MOI.EqualTo) where T
    if !iszero(MOI.constant(f))
        throw(MOI.ScalarFunctionConstantNotZero{
            T, MOI.ScalarAffineFunction{T}, MOI.EqualTo{T}}(
                MOI.constant(f)))
    end
    f = MOIU.canonical(f) # sum terms with same variables and same outputindex
    for t in f.terms
        if !iszero(t.coefficient)
            blk, i, j = varmap(m, t.variable_index)
            coef = t.coefficient
            if i != j
                coef /= 2
            end
            inputElement(m, ci.value, blk, i, j, convert(T, coef))
        end
    end
end

function MOI.optimize!(m::Optimizer)
	start_time = time()
    # SDPA.initializeUpperTriangle(m.problem, false)
    redundant_F = initializeSolve(m)
    # SDPA.solve(m)
    inputname = "input.dat-s"
    outputname = "output.dat"
    full_input_path = joinpath(m.tempfile, inputname)
    full_output_path = joinpath(m.tempfile, outputname)
    sdpa_gmp_binary_solve!(m, full_input_path, full_output_path, redundant_entries = redundant_F)
    m.solve_time = time() - start_time
end

function MOI.get(m::Optimizer, ::MOI.TerminationStatus)
    status = m.phasevalue
    if status == :noINFO
        return MOI.OPTIMIZE_NOT_CALLED
    elseif status == :pFEAS
        return MOI.SLOW_PROGRESS
    elseif status == :dFEAS
        return MOI.SLOW_PROGRESS
    elseif status == :pdFEAS
        return MOI.OPTIMAL
    elseif status == :pdINF
        return MOI.INFEASIBLE_OR_UNBOUNDED
    elseif status == :pFEAS_dINF
        return MOI.DUAL_INFEASIBLE
    elseif status == :pINF_dFEAS
        return MOI.INFEASIBLE
    elseif status == :pdOPT
        return MOI.OPTIMAL
    elseif status == :pUNBD
        return MOI.DUAL_INFEASIBLE
    elseif status == :dUNBD
        return MOI.INFEASIBLE
    end
end

function MOI.get(m::Optimizer, ::MOI.PrimalStatus)
    status = m.phasevalue
    if status == :noINFO
        return MOI.UNKNOWN_RESULT_STATUS
    elseif status == :pFEAS
        return MOI.FEASIBLE_POINT
    elseif status == :dFEAS
        return MOI.UNKNOWN_RESULT_STATUS
    elseif status == :pdFEAS
        return MOI.FEASIBLE_POINT
    elseif status == :pdINF
        return MOI.UNKNOWN_RESULT_STATUS
    elseif status == :pFEAS_dINF
        return MOI.INFEASIBILITY_CERTIFICATE
    elseif status == :pINF_dFEAS
        return MOI.INFEASIBLE_POINT
    elseif status == :pdOPT
        return MOI.FEASIBLE_POINT
    elseif status == :pUNBD
        return MOI.INFEASIBILITY_CERTIFICATE
    elseif status == :dUNBD
        return MOI.INFEASIBLE_POINT
    end
end

function MOI.get(m::Optimizer, ::MOI.DualStatus)
    status = m.phasevalue
    if status == :noINFO
        return MOI.UNKNOWN_RESULT_STATUS
    elseif status == :pFEAS
        return MOI.UNKNOWN_RESULT_STATUS
    elseif status == :dFEAS
        return MOI.FEASIBLE_POINT
    elseif status == :pdFEAS
        return MOI.FEASIBLE_POINT
    elseif status == :pdINF
        return MOI.UNKNOWN_RESULT_STATUS
    elseif status == :pFEAS_dINF
        return MOI.INFEASIBLE_POINT
    elseif status == :pINF_dFEAS
        return MOI.INFEASIBILITY_CERTIFICATE
    elseif status == :pdOPT
        return MOI.FEASIBLE_POINT
    elseif status == :pUNBD
        return MOI.INFEASIBLE_POINT
    elseif status == :dUNBD
        return MOI.INFEASIBILITY_CERTIFICATE
    end
end

MOI.get(m::Optimizer, ::MOI.ResultCount) = 1
function MOI.get(m::Optimizer, ::MOI.ObjectiveValue)
    return m.objsign * m.primalobj + m.objconstant
end
function MOI.get(m::Optimizer, ::MOI.DualObjectiveValue)
    return m.objsign * m.dualobj + m.objconstant
end
struct PrimalSolutionMatrix <: MOI.AbstractModelAttribute end
MOI.is_set_by_optimize(::PrimalSolutionMatrix) = true
MOI.get(optimizer::Optimizer, ::PrimalSolutionMatrix) = optimizer.X

struct DualSolutionVector <: MOI.AbstractModelAttribute end
MOI.is_set_by_optimize(::DualSolutionVector) = true
function MOI.get(optimizer::Optimizer, ::DualSolutionVector)
    return optimizer.y
end

struct DualSlackMatrix <: MOI.AbstractModelAttribute end
MOI.is_set_by_optimize(::DualSlackMatrix) = true
MOI.get(optimizer::Optimizer, ::DualSlackMatrix) = optimizer.Z

function block(optimizer::Optimizer, ci::MOI.ConstraintIndex{MOI.VectorOfVariables})
    return optimizer.varmap[ci.value][1]
end
function dimension(optimizer::Optimizer, ci::MOI.ConstraintIndex{MOI.VectorOfVariables})
    blockdim = optimizer.blockdims[block(optimizer, ci)]
    if blockdim < 0
        return -blockdim
    else
        return MOI.dimension(MOI.PositiveSemidefiniteConeTriangle(blockdim))
    end
end
function vectorize_block(M, blk::Integer, s::Type{MOI.Nonnegatives})
    return diag(block(M, blk))
end
function vectorize_block(M::AbstractMatrix{T}, blk::Integer, s::Type{MOI.PositiveSemidefiniteConeTriangle}) where T
    B = block(M, blk)
    d = LinearAlgebra.checksquare(B)
    n = MOI.dimension(MOI.PositiveSemidefiniteConeTriangle(d))
    v = Vector{T}(undef, n)
    k = 0
    for j in 1:d
        for i in 1:j
            k += 1
            v[k] = B[i, j]
        end
    end
    @assert k == n
    return v
end

function MOI.get(optimizer::Optimizer, ::MOI.VariablePrimal, vi::MOI.VariableIndex)
    blk, i, j = varmap(optimizer, vi)
    return block(MOI.get(optimizer, PrimalSolutionMatrix()), blk)[i, j]
end

function MOI.get(optimizer::Optimizer, ::MOI.ConstraintPrimal,
                 ci::MOI.ConstraintIndex{MOI.VectorOfVariables, S}) where S<:SupportedSets
    return vectorize_block(MOI.get(optimizer, PrimalSolutionMatrix()), block(optimizer, ci), S)
end
function MOI.get(m::Optimizer, ::MOI.ConstraintPrimal, ci::AFFEQ)
    return m.b[ci.value]
end

function MOI.get(optimizer::Optimizer, ::MOI.ConstraintDual,
                 ci::MOI.ConstraintIndex{MOI.VectorOfVariables, S}) where S<:SupportedSets
    return vectorize_block(MOI.get(optimizer, DualSlackMatrix()), block(optimizer, ci), S)
end
function MOI.get(optimizer::Optimizer, ::MOI.ConstraintDual, ci::AFFEQ)
    return -MOI.get(optimizer, DualSolutionVector())[ci.value]
end


using JLD2
using Statistics

num_bins = 12

mutable struct OneDStateVisitation <: LoggerKeyData
    state_visitations::Array{Int32,3}
    temp_state_visitations::Array{Int32, 2}
    log_interval::Int

    function OneDStateVisitation(logger_init_info)
        num_logged_steps = fld(logger_init_info[LoggerInitKey.TOTAL_STEPS], logger_init_info[LoggerInitKey.INTERVAL]) + 1
        new(zeros(num_bins, num_bins, num_logged_steps), zeros(num_bins, num_bins), logger_init_info[LoggerInitKey.INTERVAL])
    end
end

function lg_start!(self::OneDStateVisitation, env, agent)
end

function lg_step!(self::OneDStateVisitation, env, agent, s, a, s_next, r, is_terminal, cur_step_in_episode, cur_step_total)
    x = ceil(Int32, s_next[1] * num_bins)
    y = ceil(Int32, s_next[2] * num_bins)
    if (x == 0)
        x = 1
    end
    if (y == 0)
        y = 1
    end
    if (s_next[1] == 0.49999999999999994)
        println()
        println("found state")
        println("state: $(s[1:2])")
        println("next state: $(s_next[1:2])")
        print("================")
    end
    if (y == 3 && x != 6)
        println()
        println("BAD STATE")
        println("state: $(s[1:2])")
        println("next state: $(s_next[1:2])")
    end

    # println("state: $(s[1]), $(s[2])")
    # println("$(x), $(y)")

    # println(s)
    # print(x, y)

    self.temp_state_visitations[x, y] += 1
    if (rem(cur_step_total, self.log_interval) == 0)
        ind = fld(cur_step_total, self.log_interval) + 1
        self.state_visitations[:, :, ind] = self.temp_state_visitations

        # Reset temp state Visitation
        # println(sum(self.state_visitations))
        self.temp_state_visitations .*= 0
    end
end

function lg_episode_end!(self::OneDStateVisitation, cur_step_in_episode, cur_step_total)
end

function save_log(self::OneDStateVisitation, save_dict::Dict)
    save_dict[:oned_tmaze_state_visitation] = self.state_visitations
end


using CSV
using Dates
using DataFrames
using DelimitedFiles
using XLSX
using YAML

using SLiDE  # see src/SLiDE.jl

READ_DIR = joinpath("data", "readfiles")
NROW = 4;

# ******************************************************************************************
files_parse = XLSXInput("generate_yaml.xlsx", "parse", "M1:M180", "parse")
files_parse = write_yaml(READ_DIR, files_parse)

y = read_file(files_parse[1])
files = [[y[k] for k in collect(keys(y))[occursin.("Input", keys(y))]]...;]
file = files[1]

# Find which of these edits are represented in the yaml file of defined edits.
EDITS = ["Rename", "Group", "Match", "Melt", "Add", "Map", "Replace", "Drop", "Operate", "Order"]
KEYS = intersect(EDITS, collect(keys(y)))
if ("Drop" in KEYS && all(get_val.(y["Drop"]) .== "all"))
    KEYS = ["Drop"; setdiff(KEYS, ["Drop"])]
end

# Read data file and shorten to appropriate length.
df = read_file(y["PathIn"], file)
df = df[df[:,:State] .== "WISCONSIN", :][1:NROW,:]

#
desc = "nass"
pathout = joinpath(SLIDE_DIR, "data", "output_example")
CSV.write(joinpath(pathout, desc * "_00.csv"), df)

for ii in 1:length(KEYS)-1
    # editname = lowercase(KEYS[ii])
    editfile = desc * "_" * lpad(ii,2,'0') * "_" * lowercase(KEYS[ii]) * ".csv"

    global df = edit_with(df, y[KEYS[ii]])

    # Ugly, but fine for now.
    # if (desc == "nass" && editname == "operate")
    #     cols = [:yr,:r,:n,:value,:units,:value_0,:factor,:units_0]
    #     global df = df[:,cols]
    # end
    # CSV.write(joinpath(pathout, editfile), df)
    # show(first(df,4))
end

x = y["Operate"]
<<<<<<< HEAD
cols = [setdiff(propertynames(df), unique([x.from; x.to; x.input; x.output]))[1:end-2]; x.output; x.from]
=======
cols = [setdiff(names(df), unique([x.from; x.to; x.input; x.output]))[1:end-2]; x.output; x.from]
>>>>>>> dev
[inp in ensurearray(x.output) ? Symbol(inp, :_0) : inp for inp in x.input]


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct AGC <: Service
        name::String
        available::Bool
        bias::Float64
        K_p::Float64
        K_i::Float64
        K_d::Float64
        delta_t::Float64
        area::Union{Nothing, Area}
        initial_ace::Float64
        ext::Dict{String, Any}
        internal::InfrastructureSystemsInternal
    end

# Arguments
- `name::String`
- `available::Bool`
- `bias::Float64`
- `K_p::Float64`: PID Proportional Constant
- `K_i::Float64`: PID Integral Constant
- `K_d::Float64`: PID Derrivative Constant
- `delta_t::Float64`: PID Discretization period [Seconds]
- `area::Union{Nothing, Area}`: the area controlled by the AGC
- `initial_ace::Float64`: PID Discretization period [Seconds]
- `ext::Dict{String, Any}`
- `internal::InfrastructureSystemsInternal`: power system internal reference, do not modify
"""
mutable struct AGC <: Service
    name::String
    available::Bool
    bias::Float64
    "PID Proportional Constant"
    K_p::Float64
    "PID Integral Constant"
    K_i::Float64
    "PID Derrivative Constant"
    K_d::Float64
    "PID Discretization period [Seconds]"
    delta_t::Float64
    "the area controlled by the AGC"
    area::Union{Nothing, Area}
    "PID Discretization period [Seconds]"
    initial_ace::Float64
    ext::Dict{String, Any}
    "power system internal reference, do not modify"
    internal::InfrastructureSystemsInternal
end

function AGC(name, available, bias, K_p, K_i, K_d, delta_t, area=nothing, initial_ace=0.0, ext=Dict{String, Any}(), )
    AGC(name, available, bias, K_p, K_i, K_d, delta_t, area, initial_ace, ext, InfrastructureSystemsInternal(), )
end

function AGC(; name, available, bias, K_p, K_i, K_d, delta_t, area=nothing, initial_ace=0.0, ext=Dict{String, Any}(), )
    AGC(name, available, bias, K_p, K_i, K_d, delta_t, area, initial_ace, ext, )
end

# Constructor for demo purposes; non-functional.
function AGC(::Nothing)
    AGC(;
        name="init",
        available=false,
        bias=0.0,
        K_p=0.0,
        K_i=0.0,
        K_d=0.0,
        delta_t=0.0,
        area=Area(nothing),
        initial_ace=0.0,
        ext=Dict{String, Any}(),
    )
end

InfrastructureSystems.get_name(value::AGC) = value.name
"""Get AGC available."""
get_available(value::AGC) = value.available
"""Get AGC bias."""
get_bias(value::AGC) = value.bias
"""Get AGC K_p."""
get_K_p(value::AGC) = value.K_p
"""Get AGC K_i."""
get_K_i(value::AGC) = value.K_i
"""Get AGC K_d."""
get_K_d(value::AGC) = value.K_d
"""Get AGC delta_t."""
get_delta_t(value::AGC) = value.delta_t
"""Get AGC area."""
get_area(value::AGC) = value.area
"""Get AGC initial_ace."""
get_initial_ace(value::AGC) = value.initial_ace
"""Get AGC ext."""
get_ext(value::AGC) = value.ext
"""Get AGC internal."""
get_internal(value::AGC) = value.internal

InfrastructureSystems.set_name!(value::AGC, val::String) = value.name = val
"""Set AGC available."""
set_available!(value::AGC, val::Bool) = value.available = val
"""Set AGC bias."""
set_bias!(value::AGC, val::Float64) = value.bias = val
"""Set AGC K_p."""
set_K_p!(value::AGC, val::Float64) = value.K_p = val
"""Set AGC K_i."""
set_K_i!(value::AGC, val::Float64) = value.K_i = val
"""Set AGC K_d."""
set_K_d!(value::AGC, val::Float64) = value.K_d = val
"""Set AGC delta_t."""
set_delta_t!(value::AGC, val::Float64) = value.delta_t = val
"""Set AGC area."""
set_area!(value::AGC, val::Union{Nothing, Area}) = value.area = val
"""Set AGC initial_ace."""
set_initial_ace!(value::AGC, val::Float64) = value.initial_ace = val
"""Set AGC ext."""
set_ext!(value::AGC, val::Dict{String, Any}) = value.ext = val
"""Set AGC internal."""
set_internal!(value::AGC, val::InfrastructureSystemsInternal) = value.internal = val


using Revise
#]add RiskyCommunity
using RiskyCommunity
using Parameters
using CSV
using DataFrames
using Statistics
using Glob
using JuMP
using Gurobi
using Query
using StatsBase

###########FUNCTIONS TO USE#########
    function read_result(model::JuMP.Model)
        cost_AGG = JuMP.value(model[:cost_AGG])
        cost_DA_market = sum(JuMP.value.(model[:cost_DA_market]))
        cost_RT_market = sum(JuMP.value.(model[:cost_RT_market]))
        @show cost_AGG, cost_DA_market, cost_RT_market

        soc = JuMP.value.(model[:soc])
        cap = model.ext[:param][:capsₑ][1]

        viol_balance_up = soc.data[soc.data .> (0.001 + cap)]
        viol_balance_down = soc.data[soc.data .< -0.001]
        sum_balance_down = sum(viol_balance_down)
        sum_balance_up = sum(viol_balance_up)
        numb_balance_up = length(viol_balance_up)
        numb_balance_down = length(viol_balance_down)

        reliability_in = (1-(numb_balance_up + numb_balance_down)/(5*96*65))

        @show numb_balance_up, numb_balance_down

        in_res_dict = Dict("cost_AGG" => cost_AGG, "reliability_in" => reliability_in , "cost_DA_market" => cost_DA_market, "cost_RT_market" => cost_RT_market,
        "numb_viol_UP" => numb_balance_up, "numb_viol_DOWN" => numb_balance_down, "sum_balance_down" => sum_balance_down, "sum_balance_up" => sum_balance_up)

        return in_res_dict
    end

    function read_test_result(model::JuMP.Model)
        cost_AGG = JuMP.value(model[:true_cost_AGG])
        cost_DA_market = sum(JuMP.value.(model[:cost_DA_market]))
        cost_RT_market = sum(JuMP.value.(model[:cost_RT_market]))

        slack_DOWN = JuMP.value.(model.ext[:dec_vars][:slack_down])
        slack_UP = JuMP.value.(model.ext[:dec_vars][:slack_up])

        #Violations of Balance
        numb_upper_violation = length(slack_DOWN.data[slack_DOWN.data .≠ 0])
        numb_lower_violation = length(slack_UP.data[slack_UP.data .≠ 0])
        numb_viol = (numb_upper_violation + numb_lower_violation)

        max_DOWN = max(slack_DOWN...)
        max_UP = max(slack_UP...)

        sum_DOWN = sum(slack_DOWN)
        sum_UP= sum(slack_UP)

        @show cost_AGG, cost_DA_market, cost_RT_market

        res_dict = Dict("numb_viol" => numb_viol,  "cost_AGG" => cost_AGG, "cost_DA_market" => cost_DA_market, "cost_RT_market" => cost_RT_market,
         "max_DOWN" => max_DOWN, "max_UP" => max_UP, "sum_UP" => sum_UP, "sum_DOWN" => sum_DOWN)
        return res_dict
    end

##############INIT THE MARKET DATA################
    period = Period(number_of_days = 1, number_of_t_steps= 96)
    # ##########OBTAIN THE RT IMBALANCE PRICE SCENARIOS FOR THE TRAINING SET##########
    scenarios_train = Dict()
    scen_price_RT_train = Dict()
    dat = glob("*.csv","data/imb_price_scenarios/cluster_train_5/")
    train_dat = dat
    for d in 1:length(train_dat)
        scenario_train = DataFrame!(CSV.File(train_dat[d], header = true, decimal='.'))
        scenario_train = scenario_train[2:end, :]
        scenarios_train[d] = scenario_train
        scen_price_RT_train[d] = Scenario(id = "RT_price_$d", counter = d, prob = 1/length(train_dat)) #Equal probability
    end

    prices_RT_train = []
    for d in 1:length(train_dat)
        push!(prices_RT_train, Price(period = period,  scenario = scen_price_RT_train[d], id = scen_price_RT_train[d].id))
        prices_RT_train[d].values[1,:] = scenarios_train[d].values
    end

##########CHOSING ONE FORECAST FOR DA PRICES##########
    da_price_init =  DataFrame!(CSV.File("data/Day-ahead Prices_202003150000-202003160000.csv", header = true, decimal=','))[:,2]
    da_price = [parse(Float64, da_price_init[i]) for i in range(1, stop = length(da_price_init))]
    da_price_96 = repeat(da_price, inner = [4]) #EUR/MWh
    #Single scenario for DA price
    scen_price_DA = Scenario(id = :DA_price, prob = 1.0) #Single DA scenario
    #Init DA price forecast
    price_DA = Price(period = period)
    price_DA.values[1,:] = da_price_96

    #Use this form when passing to the JuMP model
    prices_train = Dict(:DA => price_DA, :RT => prices_RT_train)

###########INIT THE DER AGENT DATA AS A SINGLE AGENT WITH BESS#############
    dat_agent = DataFrame!(CSV.File("data/WCVaR_dat/single_site/63/da_fc/63_err_1.csv", header=true, delim= ","))
    values(dat_agent[1,2:end])
    # sort(dat_agent, [:timestamp])
    loads = collect(values(dat_agent[1,2:end]) ./1000) # To get something reasonable in MW
    my_der_agent = DER_Agent(period = period)
    my_der_agent.demand.values[1,:] = loads
    my_der_agent.dev_ID.values[1,:] = zeros(length(loads)) #No ID dev
    my_der_agent.dev_RT.values[1,:] = zeros(length(loads)) #No RT dev
    #The model complains if there are no scenarios added as it expects
    scen_ID_dev_1 = Scenario(id = :ID_dev_1, counter = 1, prob = 1.0)
    dev_ID_1 = Demand(period = period, id= :ID_dev_1, scenario = scen_ID_dev_1, values = zeros(period.number_of_days, period.number_of_t_steps))
    deviations_ID = [dev_ID_1]
    my_der_agent.dev_ID = deviations_ID

#####LOADING RT DEV SCENARIOS##########FOR  WCVAR OPT#######
    # loc = "data/WCVaR_dat/diff_per_site/mix2/split"
    # # loc = "data/WCVaR_dat/single_site/58/clustered"
    # groups = readdir(loc)
    #
    # errors_dict = Dict()
    # # errors_dict["$curr_group"] = glob("*.csv","data/WCVaR_dat/diff_per_site/mix2/split/$curr_group")
    # err_scenarios_train = Dict()
    # scen_err_RT_train = Dict()
    # global counter = 0
    # for g in groups
    #     curr_group = g
    #     errors_dict["$curr_group"] = glob("*.csv","$loc/$curr_group")
    #     for d in 1:length(errors_dict["$curr_group"])
    #         global counter = counter + 1
    #         ###GET THE PROBABILITY OF THE SCENARIO STORED IN THE NAME######
    #         str = errors_dict[g][d]
    #         prob_str = last(str, 10)
    #         prob_str = first(prob_str, 6)
    #         prob_float = parse(Float64, prob_str)
    #         #########UPDATE VALUES##########
    #         err_scenario_train = DataFrame!(CSV.File(errors_dict["$curr_group"][d], header = true, decimal='.'))
    #         err_scenario_train = err_scenario_train[1:end, :].values .* loads
    #         err_scenarios_train[g,d] = err_scenario_train
    #         scen_err_RT_train[g,d] = Scenario(id = "RT_error_SITE$curr_group", counter = counter, prob = prob_float/5) #Equal probability
    #     end
    # end
    #
    # errors_RT_train = []
    # for g in groups
    #     curr_group = g
    #     for d in 1:length(errors_dict["$curr_group"])
    #         push!(errors_RT_train, Demand(period = period,  scenario = scen_err_RT_train[g,d], id = scen_err_RT_train[g,d].id, values = permutedims(err_scenarios_train[g,d])))
    #         # errors_RT_train[d].values[1,:] = err_scenarios_train[g,d]
    #     end
    # end

    # FOR  CVAR OPT#######
    error_dat = glob("*.csv","data/WCVaR_dat/diff_per_site/mix2/")
    err_scenarios_train = Dict()
    scen_err_RT_train = Dict()
    for d in 1:length(error_dat)
        ###GET THE PROBABILITY OF THE SCENARIO STORED IN THE NAME######
        str = error_dat[d]
        prob_str = last(str, 10)
        prob_str = first(prob_str, 6)
        prob_float = parse(Float64, prob_str)
        #########UPDATE VALUES##########
        err_scenario_train = DataFrame!(CSV.File(error_dat[d], header = true, decimal='.'))
        err_scenario_train = err_scenario_train[1:end, :].values .* loads
        err_scenarios_train[d] = err_scenario_train
        scen_err_RT_train[d] = Scenario(id = "RT_error_group_1", counter = d, prob = prob_float/5) #Equal probability
    end

    errors_RT_train = []
    for d in 1:length(error_dat)
        push!(errors_RT_train, Demand(period = period,  scenario = scen_err_RT_train[d], id = scen_err_RT_train[d].id))
        errors_RT_train[d].values[1,:] = err_scenarios_train[d]
    end

    my_der_agent.dev_RT = errors_RT_train

##########BESS SETTING############
    batt_1 = Battery(capacity_power = 3.0, capacity_energy = 0.4, efficiency_dch = 0.99, efficiency_ch = 0.99)
    my_der_agent.battery = batt_1
    my_der_agent_2 = deepcopy(my_der_agent)
    my_aggregator = Aggregator(name ="Mihalys_aggregator")
    DER_agents = [my_der_agent]
    my_aggregator.DER_agents = DER_agents

#OPEXs OF THE DIFFERENT STAGES
    my_aggregator.DER_agents[1].OPEX_DA = mean(price_DA.values) .* 0.01
    my_aggregator.DER_agents[1].OPEX_ID = 0.001
    my_aggregator.DER_agents[1].OPEX_RT = sum(mean(prices_RT_train[j].values) for j in [1,2,3,4,5])./5 .* 0.05

    my_aggregator.preferences.bigM = 3000.0 #For the VaR version
    my_aggregator.max_DA = 2 * max(loads...)
    my_aggregator.max_RT = 0.5 * max(Iterators.flatten([errors_RT_train[j].values for j in range(1, 65, step = 1)])...) #Max error among the scenarios

#TESTS
NEW_in_DICT_CVAR = Dict()
NEW_out_DICT_CVAR = Dict()
# NEW_in_DICT_WCVAR = Dict()
# NEW_out_DICT_WCVAR = Dict()
#
# ADD_in_DICT_WCVAR = Dict()
# ADD_out_DICT_WCVAR = Dict()

#OR LOAD THE EXISTING RESULTS
# using JLD
# out_DICT_CVAR = load("data/results/WCVaR/out_DICT_CVAR.jld")["NEW_out_DICT_CVAR"]
# NEW_out_DICT_WCVAR = load("data/results/WCVaR/out_DICT_WCVAR.jld")["NEW_out_DICT_WCVAR"]
# NEW_in_DICT_WCVAR = load("data/results/WCVaR/in_DICT_WCVAR.jld")["NEW_in_DICT_WCVAR"]
#
# NEW_out_DICT_CVAR = load("data/results/WCVaR/out_DICT_CVAR.jld")["NEW_out_DICT_CVAR"]
# NEW_in_DICT_CVAR = load("data/results/WCVaR/in_DICT_CVAR.jld")["NEW_in_DICT_CVAR"]
#
# ADD_out_DICT_WCVAR = load("data/results/WCVaR/ADD_out_DICT_WCVAR.jld")["ADD_out_DICT_WCVAR"]
# ADD_in_DICT_WCVAR = load("data/results/WCVaR/ADD_in_DICT_WCVAR.jld")["ADD_in_DICT_WCVAR"]

sites_test = ["12", "58", "40", "17", "29", "70", "68"]
for s in sites_test
    in_results_CVAR =  DataFrame(ϵ = [], reliability = [], cost = [], numb_violation_UP = [],  numb_violation_DOWN = [], sum_UP = [], sum_DOWN = [], t_build = [], t_solve = [])
    out_results_CVAR = DataFrame(ϵ = [], numb_viol = [], iter = [], cost_AGG = [], max_upper = [], max_lower = [], sum_upper = [], sum_lower = [])
    rng = range(0.01, 0.1, step = 0.01)
    for epsilon in rng
            println("epsilon was $epsilon")
            # epsilon = 0.1
            my_aggregator.preferences.ϵ = epsilon
            (aggregator_model) , time_to_build = @timed build_model(my_aggregator, period, prices_train)
            #Some debugging options
            # aggregator_model.ext[:param][:scens_RT_dev][1][2].values[1,2]
            # aggregator_model.ext[:param][:group_RT_dev]
            # aggregator_model[:energy_balance]
            #
            # aggregator_model.ext[:param][:ϵ]
            #
            # aggregator_model.ext[:param][:scens_RT_market][1]

            optimizer = Gurobi.Optimizer
            max_time = 4000
            psolve = -1
            meth = -1
            #CVAR MODEL
            (solved_aggregator_model) , time_to_solve = @timed solve_model(aggregator_model, optimizer; timelimit = max_time, method = meth, presolve = psolve)
            in_res = read_result(solved_aggregator_model)
            push!(in_results_CVAR, [epsilon, in_res["reliability_in"], in_res["cost_AGG"], in_res["numb_viol_UP"], in_res["numb_viol_DOWN"], in_res["sum_balance_up"], in_res["sum_balance_down"], time_to_build, time_to_solve])

            #Copying the aggregator for the out-of-sample tests
            test_aggregator = deepcopy(my_aggregator)
            test_DER_agent = deepcopy(my_der_agent)
            test_aggregator.DER_agents[1].dev_RT
            test_DER_agent.dev_RT = []

            #Attaining test data
            error_dat_test = glob("*.csv", "data/WCVaR_dat/single_site/$s")
            err_scenarios_test = Dict()
            scen_err_RT_test = Dict()
            for d in 1:length(error_dat_test)
                err_scenario_test = DataFrame!(CSV.File(error_dat_test[d], header = true, decimal='.'))
                err_scenario_test = err_scenario_test[1:end, :].values .* loads
                err_scenarios_test[d] = err_scenario_test
                scen_err_RT_test[d] = Scenario(id = "RT_error_group_1", counter = d, prob = 1) #Equal probability
            end

            # results_CVAR = DataFrame(ϵ = [], iter = [], cost_AGG = [], max_upper = [], max_lower = [], sum_upper = [], sum_lower = [])

            for i in 1:length(error_dat_test)
                errors_RT_test = []
                errors_RT_test = Demand(period = period,  scenario = scen_err_RT_test[i], id = scen_err_RT_test[i].id)
                errors_RT_test.values[1,:] = err_scenarios_test[i]
                test_DER_agent.dev_RT = []
                test_DER_agent.dev_RT = [errors_RT_test]
                test_aggregator.DER_agents = [test_DER_agent]
                test_aggregator_model = build_test_model(test_aggregator, period, prices_train)
                ####FIX DA DECISIONS TO CVAR MODEL#########
                # fix_model_variable!(test_aggregator_model, :fᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:fᴰᴬ]))
                fix_model_variable!(test_aggregator_model, :aᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:aᴰᴬ]))
                fix_model_variable!(test_aggregator_model, :chᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:chᴰᴬ]))
                fix_model_variable!(test_aggregator_model, :dchᴰᴬ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:dchᴰᴬ]))

                # fix_model_variable!(test_aggregator_model, :fᴵᴰ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:fᴵᴰ]))
                # fix_model_variable!(test_aggregator_model, :chᴵᴰ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:chᴵᴰ]))
                # fix_model_variable!(test_aggregator_model, :dchᴵᴰ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:dchᴵᴰ]))
                #Fixing everything except the imbalance market position
                # fix_model_variable!(test_aggregator_model, :soc, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:soc])[:,:,:,1,:,:])
                # fix_model_variable!(test_aggregator_model, :chᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:chᴿᵀ])[:,:,:,1,:,:])
                # fix_model_variable!(test_aggregator_model, :dchᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:dchᴿᵀ])[:,:,:,1,:,:])
                fix_model_variable!(test_aggregator_model, :aᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:aᴿᵀ]))
                # fix_model_variable!(test_aggregator_model, :fᴿᵀ, JuMP.value.(solved_aggregator_model.ext[:dec_vars][:fᴿᵀ]))

                test_solved_aggregator_model = solve_model(test_aggregator_model, optimizer, timelimit = max_time)
                term_stat = termination_status(test_solved_aggregator_model)

                # Gurobi.compute_conflict(test_solved_aggregator_model.moi_backend.optimizer.model)
                # MOI.get(test_solved_aggregator_model.moi_backend, Gurobi.ConstraintConflictStatus(), soc1.index)
                # test_solved_aggregator_model.computeIIS()
                # using Gurobi
                # compute_conflict!(test_solved_aggregator_model)
                # grb_model = test_solved_aggregator_model.moi_backend.optimizer.model
                # computeIIS(grb_model.inner)
                # test_solved_aggregator_model.feasRelax()
                # Gurobi.compute_conflict(grb_model)
                # feasRelax(grb_model.inner)
                # Gurobi.computeIIS(test_solved_aggregator_model)
                if term_stat == MOI.OPTIMAL || term_stat == MOI.LOCALLY_SOLVED
                    println("######The result was feasible at the $i-th iteration...####")
                    cost_AGG = JuMP.value(test_solved_aggregator_model[:true_cost_AGG])
                    println("######The cost was $cost_AGG...####")

                    res = read_test_result(test_solved_aggregator_model)
                    # res["sum_UP"]
                    # res["sum_DOWN"]
                    # res["max_UP"]
                    # res["max_DOWN"]
                    # real_cost = res["cost_AGG"]
                    push!(out_results_CVAR, [test_aggregator.preferences.ϵ, res["numb_viol"], i,  res["cost_AGG"], res["max_UP"],  res["max_DOWN"], res["sum_UP"], res["sum_DOWN"]])
                else
                    println("######The result was infeasible at the $i-th iteration...####")
                end
            end
    end

    NEW_in_DICT_CVAR["SITE$s","MIX2"] = in_results_CVAR
    NEW_out_DICT_CVAR["SITE$s","MIX2"] = out_results_CVAR
end

##########FROM HERE

rels_CVAR = Dict()
rels_WCVAR = Dict()
rels_norm_CVAR = Dict()
rels_norm_WCVAR = Dict()
cost_scaled_CVAR = Dict()
cost_scaled_WCVAR = Dict()
mean_costs_CVAR = Dict()
mean_costs_WCVAR = Dict()

mean_viol_CVAR_dict = Dict()
mean_viol_WCVAR_dict = Dict()
perc95_viol_CVAR_dict = Dict()
perc95_viol_WCVAR_dict = Dict()
add_perc95_viol_WCVAR_dict = Dict()

max_viol_CVAR_dict = Dict()
max_viol_WCVAR_dict = Dict()
add_max_viol_WCVAR_dict = Dict()

add_rels_WCVAR = Dict()
add_mean_costs_WCVAR = Dict()

numb_viol_CVAR_dict = Dict()
numb_viol_WCVAR_dict = Dict()
add_numb_viol_WCVAR_dict = Dict()
add_mean_viol_WCVAR_dict = Dict()

epsilons = rng = range(0.01, 0.1, step = 0.01)

sites  = ["70", "17", "68", "29", "12", "40", "58"]
mixes = ["MIX2"]

using PyCall, PyPlot
sns = pyimport("seaborn")
pygui(true)
plt.style.use("default")

for s in sites
    for m in mixes
        out_results_CVAR = NEW_out_DICT_CVAR["SITE$s","$m"]
        in_results_CVAR = NEW_in_DICT_CVAR["SITE$s","$m"]
        out_results_WCVAR = NEW_out_DICT_WCVAR["SITE$s","$m"]
        in_results_WCVAR = NEW_in_DICT_WCVAR["SITE$s","$m"]

        add_out_results_WCVAR = ADD_out_DICT_WCVAR["SITE$s","$m"]
        add_in_results_WCVAR = ADD_in_DICT_WCVAR["SITE$s","$m"]

        mean_cost = []
        mean_viol = []
        reliability_CVAR = []
        reliability_CVAR_perc_up = []
        reliability_CVAR_perc_down = []
        numb_viol_CVAR = []
        perc95_viol_CVAR = []
        max_viol_CVAR = []

        mean_cost_WCVAR = []
        mean_viol_WCVAR = []
        reliability_WCVAR = []
        reliability_WCVAR_perc_up = []
        reliability_WCVAR_perc_down = []
        numb_viol_WCVAR = []
        perc95_viol_WCVAR = []
        max_viol_WCVAR = []

        add_mean_cost_WCVAR = []
        add_mean_viol_WCVAR = []
        add_reliability_WCVAR = []
        add_reliability_WCVAR_perc_up = []
        add_reliability_WCVAR_perc_down = []
        add_numb_viol_WCVAR = []
        add_perc95_viol_WCVAR = []
        add_max_viol_WCVAR = []

        for eps in rng

            CVAR = out_results_CVAR |>
              @filter(_.ϵ == eps) |>
              DataFrame

            push!(mean_cost, mean(CVAR.cost_AGG))
            rel = (1 - sum(CVAR.numb_viol)/(length(CVAR.ϵ).*5 .* 96))
            push!(reliability_CVAR, rel)
            push!(reliability_CVAR_perc_up, percentile((1 .- (CVAR.numb_viol)./(length(CVAR.ϵ).*5 .* 96)), 95))
            push!(reliability_CVAR_perc_down, percentile((1 .- (CVAR.numb_viol)./(length(CVAR.ϵ).*5 .* 96)), 5))

            push!(numb_viol_CVAR, sum(CVAR.numb_viol))
            push!(perc95_viol_CVAR, percentile((CVAR.sum_upper + CVAR.sum_lower), 99))

            push!(mean_viol, mean(CVAR.sum_upper + CVAR.sum_lower))
            push!(max_viol_CVAR, max(CVAR.max_upper..., CVAR.max_lower...))

        end

        for eps in rng

            WCVAR = out_results_WCVAR |>
              @filter(_.ϵ == eps) |>
              DataFrame

            push!(mean_cost_WCVAR, mean(WCVAR.cost_AGG))
            rel = (1 - sum(WCVAR.numb_viol)/(length(WCVAR.ϵ).*5 .* 96))
            push!(reliability_WCVAR, rel)
            push!(reliability_WCVAR_perc_up, percentile((1 .- (WCVAR.numb_viol)./(length(WCVAR.ϵ).*5 .* 96)), 95))
            push!(reliability_WCVAR_perc_down, percentile((1 .- (WCVAR.numb_viol)./(length(WCVAR.ϵ).*5 .* 96)), 5))

            push!(numb_viol_WCVAR, sum(WCVAR.numb_viol))
            push!(mean_viol_WCVAR, mean(WCVAR.sum_upper + WCVAR.sum_lower))
            push!(perc95_viol_WCVAR, percentile((WCVAR.sum_upper + WCVAR.sum_lower), 99))
            push!(max_viol_WCVAR, max(WCVAR.max_upper..., WCVAR.max_lower...))

        end

        for eps in range(0.1, 0.17; step =0.01)

            add_WCVAR = add_out_results_WCVAR |>
              @filter(_.ϵ == eps) |>
              DataFrame

              push!(add_mean_cost_WCVAR, mean(add_WCVAR.cost_AGG))
              rel = (1 - sum(add_WCVAR.numb_viol)/(length(add_WCVAR.ϵ).*5 .* 96))
              push!(add_reliability_WCVAR, rel)
              push!(add_reliability_WCVAR_perc_up, percentile((1 .- (add_WCVAR.numb_viol)./(length(add_WCVAR.ϵ).*5 .* 96)), 95))
              push!(add_reliability_WCVAR_perc_down, percentile((1 .- (add_WCVAR.numb_viol)./(length(add_WCVAR.ϵ).*5 .* 96)), 5))
              push!(add_mean_viol_WCVAR, mean(add_WCVAR.sum_upper + add_WCVAR.sum_lower))

              push!(add_numb_viol_WCVAR, sum(add_WCVAR.numb_viol))
              push!(add_perc95_viol_WCVAR, percentile((add_WCVAR.sum_upper + add_WCVAR.sum_lower), 99))
              push!(add_max_viol_WCVAR, max(add_WCVAR.max_upper..., add_WCVAR.max_lower...))

        end

        mean_costs_CVAR["SITE$s","$m"] = -mean_cost
        mean_costs_WCVAR["SITE$s","$m"] = -mean_cost_WCVAR
        add_mean_costs_WCVAR["SITE$s","$m"] = -add_mean_cost_WCVAR

        # cost_scaled_CVAR["SITE$s","$m"]  = scaled_cost_CVAR
        # cost_scaled_WCVAR["SITE$s","$m"]  = scaled_cost_WCVAR

        rels_CVAR["SITE$s","$m"] = reliability_CVAR
        rels_WCVAR["SITE$s","$m"] = reliability_WCVAR
        add_rels_WCVAR["SITE$s","$m"] = add_reliability_WCVAR

        numb_viol_CVAR_dict["SITE$s","$m"] = numb_viol_CVAR
        numb_viol_WCVAR_dict["SITE$s","$m"] = numb_viol_WCVAR
        add_numb_viol_WCVAR_dict["SITE$s","$m"] = add_numb_viol_WCVAR

        mean_viol_CVAR_dict["SITE$s","$m"] = mean_viol
        mean_viol_WCVAR_dict["SITE$s","$m"] = mean_viol_WCVAR
        add_mean_viol_WCVAR_dict["SITE$s","$m"] = add_mean_viol_WCVAR

        perc95_viol_CVAR_dict["SITE$s","$m"] = perc95_viol_CVAR
        perc95_viol_WCVAR_dict["SITE$s","$m"] = perc95_viol_WCVAR
        add_perc95_viol_WCVAR_dict["SITE$s","$m"] = add_perc95_viol_WCVAR

        max_viol_CVAR_dict["SITE$s","$m"] = max_viol_CVAR
        max_viol_WCVAR_dict["SITE$s","$m"] = max_viol_WCVAR
        add_max_viol_WCVAR_dict["SITE$s","$m"] = add_max_viol_WCVAR

        # rels_norm_CVAR["SITE$s","$m"] = rel_out_vs_in_CVAR
        # rels_norm_WCVAR["SITE$s","$m"] = rel_out_vs_in_WCVAR
    end
end

cvar = NEW_out_DICT_CVAR["SITE29","MIX2"] |>
  @filter(_.cost_AGG <= -440) |>
  DataFrame

wcvar = NEW_out_DICT_WCVAR["SITE29","MIX2"] |>
    @filter(_.cost_AGG <= -440) |>
    DataFrame

add_wcvar = ADD_out_DICT_WCVAR["SITE29","MIX2"] |>
    @filter(_.cost_AGG <= -440) |>
    DataFrame

#adding up the dataframes
wcvar = [wcvar; add_wcvar]

sns.set_context("paper", font_scale=1.8)
sns.set_style("whitegrid")
sns.set_style("ticks")
sns.color_palette()

p = sns.JointGrid()
sns.kdeplot(color = "tab:blue", x=-wcvar.cost_AGG[:], y=max((wcvar.max_upper, wcvar.max_lower)...),  ax=p.ax_joint, alpha = 0.8, stat = "density", common_norm = true, label = "WCVaR")
sns.kdeplot(color = "tab:orange", x=-cvar.cost_AGG[:], y=max((cvar.max_upper, cvar.max_lower)...), ax=p.ax_joint, alpha = 0.8, stat = "density", common_norm = true, label = "CVaR")

sns.histplot(color = "tab:blue", y=max((wcvar.max_upper, wcvar.max_lower)...),  ax=p.ax_marg_y, element="step", stat = "density", alpha = 0.5)
sns.histplot(color = "tab:orange", y=max((cvar.max_upper, cvar.max_lower)...),  ax=p.ax_marg_y, element="step", stat = "density", alpha = 0.5)
sns.histplot(color = "tab:blue", x=-wcvar.cost_AGG[:],  ax=p.ax_marg_x, element="step", stat = "density", alpha = 0.5)
sns.histplot(color = "tab:orange", x=-cvar.cost_AGG[:],  ax=p.ax_marg_x, element="step", stat = "density", alpha = 0.5)

# plt.legend()
plt.ylim(0.0, 1.5)
mpatches = pyimport("matplotlib.patches")
blue_patch = mpatches.Patch(color="tab:blue", alpha = 0.5, label="WCVaR")
red_patch = mpatches.Patch(color="tab:orange", alpha = 0.5, label="CVaR")

text = mpatches.Patch(label = "29, d=0.501",fc="w", fill=false, edgecolor="none", linewidth=0)

p.ax_joint.legend(handles=[blue_patch, red_patch, text], ncol =1, loc = "upper right")


module Libcint

using libcint_jll

#####
##### Functions names
#####

const dim = [
    "CINTgto_cart",
    "CINTgto_spheric",
    "CINTgto_spinor",
    ]

const f1e = [
    "cint1e_ovlp_cart",
    "cint1e_nuc_cart",
    "cint1e_kin_cart",
    "cint1e_ia01p_cart",
    "cint1e_irixp_cart",
    "cint1e_ircxp_cart",
    "cint1e_iking_cart",
    "cint1e_iovlpg_cart",
    "cint1e_inucg_cart",
    "cint1e_ipovlp_cart",
    "cint1e_ipkin_cart",
    "cint1e_ipnuc_cart",
    "cint1e_iprinv_cart",
    "cint1e_rinv_cart",
    "cint2e_cart",
    "cint2e_ig1_cart",
    "cint2e_ip1_cart",
    "cint1e_ovlp_sph",
    "cint1e_nuc_sph",
    "cint1e_kin_sph",
    "cint1e_ia01p_sph",
    "cint1e_irixp_sph",
    "cint1e_ircxp_sph",
    "cint1e_iking_sph",
    "cint1e_iovlpg_sph",
    "cint1e_inucg_sph",
    "cint1e_ipovlp_sph",
    "cint1e_ipkin_sph",
    "cint1e_ipnuc_sph",
    "cint1e_iprinv_sph",
    "cint1e_rinv_sph",
    "cint2e_sph",
    "cint2e_ig1_sph",
    "cint2e_ip1_sph",
    "cint1e_ovlp",
    "cint1e_nuc",
    "cint1e_nucg",
    "cint1e_srsr",
    "cint1e_sr",
    "cint1e_srsp",
    "cint1e_spsp",
    "cint1e_sp",
    "cint1e_spspsp",
    "cint1e_spnuc",
    "cint1e_spnucsp",
    "cint1e_srnucsr",
    "cint1e_sa10sa01",
    "cint1e_ovlpg",
    "cint1e_sa10sp",
    "cint1e_sa10nucsp",
    "cint1e_sa01sp",
    "cint1e_spgsp",
    "cint1e_spgnucsp",
    "cint1e_spgsa01",
    "cint1e_ipovlp",
    "cint1e_ipkin",
    "cint1e_ipnuc",
    "cint1e_iprinv",
    "cint1e_ipspnucsp",
    "cint1e_ipsprinvsp",
    "cint2e",
    "cint2e_spsp1",
    "cint2e_spsp1spsp2",
    "cint2e_srsr1",
    "cint2e_srsr1srsr2",
    "cint2e_sa10sp1",
    "cint2e_sa10sp1spsp2",
    "cint2e_g1",
    "cint2e_spgsp1",
    "cint2e_g1spsp2",
    "cint2e_spgsp1spsp2",
    "cint2e_ip1",
    "cint2e_ipspsp1",
    "cint2e_ip1spsp2",
    "cint2e_ipspsp1spsp2",
    "cint2e_ssp1ssp2",
    ]

#####
##### Function wrapper macros
#####

macro make_dim(name::String)
    fn = Symbol(name)
    _fn = Symbol('_'*name)
    quote
        function $(esc(fn))(
            i::Cint,
            bas::AbstractVector{Cint}
            )
            n = ccall(
                ($_fn, libcint),
                Cint,
                (Cint, Ref{Cint}),
                i, bas,
                )
        end
    end
end

macro make_fn(name::String)
    fn = Symbol(name)
    _fn = Symbol('_'*name)
    quote
        function $(esc(fn))(
            buf::AbstractArray{Cdouble},
            shls::AbstractVector{Cint},
            atm::AbstractVector{Cint},
            natm::Cint,
            bas::AbstractVector{Cint},
            nbas::Cint,
            env::AbstractVector{Cdouble},
            )
            not0 = ccall(
                ($_fn, libcint),
                Cint,
                (Ref{Cdouble}, Ref{Cint}, Ref{Cint}, Cint, Ref{Cint}, Cint, Ref{Cdouble}, Ptr{Cvoid}),
                buf, shls, atm, natm, bas, nbas, env, C_NULL,
                )
        end
    end
end

#####
##### Define functions
#####

for s in dim
    _s = Meta.parse('_'*s);
    @eval const $_s = $s
    @eval @make_dim($s)
end

for s in f1e
    _s = Meta.parse('_'*s);
    @eval const $_s = $s
    @eval @make_fn($s)
end

end # module


function fen_to_bitboard(fen_string::String)
    white = EMPTY
    R = zeros(UInt64, 0)
    N = zeros(UInt64, 0)
    B = zeros(UInt64, 0)
    Q = zeros(UInt64, 0)
    K = EMPTY
    P = zeros(UInt64, 0)

    black = EMPTY
    r = zeros(UInt64, 0)
    n = zeros(UInt64, 0)
    b = zeros(UInt64, 0)
    q = zeros(UInt64, 0)
    k = EMPTY
    p = zeros(UInt64, 0)

    free = EMPTY
    taken = EMPTY

    white_attacks = EMPTY
    black_attacks = EMPTY

    # board
    fen = split(fen_string, ' ')
    board = fen[1]
    square_i = 1
    fen_i = 1
    while true
        c = board[fen_i]
        if isnumeric(c)
            square_i += parse(Int64, c)
            fen_i += 1
        elseif c == '/'
            fen_i += 1
        else
            if isuppercase(c)
                # white
                white |= INT2UINT[square_i]
                if c == 'R'
                    push!(R, INT2UINT[square_i])
                elseif c == 'N'
                    push!(N, INT2UINT[square_i])
                elseif c == 'B'
                    push!(B, INT2UINT[square_i])
                elseif c == 'Q'
                    push!(Q, INT2UINT[square_i])
                elseif c == 'K'
                    K |= INT2UINT[square_i]
                elseif c == 'P'
                    push!(P, INT2UINT[square_i])
                end
            else
                # black
                black |= INT2UINT[square_i]
                if c == 'r'
                    push!(r, INT2UINT[square_i])
                elseif c == 'n'
                    push!(n, INT2UINT[square_i])
                elseif c == 'b'
                    push!(b, INT2UINT[square_i])
                elseif c == 'q'
                    push!(q, INT2UINT[square_i])
                elseif c == 'k'
                    k |= INT2UINT[square_i]
                elseif c == 'p'
                    push!(p, INT2UINT[square_i])
                end
            end
            square_i += 1
            fen_i += 1
        end

        if fen_i > length(board)

            if square_i != 65
                throw(ArgumentError("Invalid FEN string: too short/long"))
            end

            taken = white | black
            free = ~taken

            break
        end
    end

    if fen[2] == "w"
        player_color = "white"
    else
        player_color = "black"
    end

    white_king_moved = false
    black_king_moved = false
    if k != PGN2UINT["e8"]
        black_king_moved = true
    end
    if K != PGN2UINT["e1"]
        white_king_moved = true
    end
    white_can_castle_queenside = false
    white_can_castle_kingside = false
    black_can_castle_queenside = false
    black_can_castle_kingside = false
    for c in fen[3]
        if c == '-'
            break
        elseif c == 'K'
            white_can_castle_kingside = true
        elseif c == 'Q'
            white_can_castle_queenside = true
        elseif c == 'k'
            black_can_castle_kingside = true
        elseif c == 'q'
            black_can_castle_queenside = true
        end
    end

    enpassant_square = EMPTY
    enpassant_done = false
    if fen[4] != "-"
        enpassant_square = PGN2UINT[fen[4]]
    end

    if length(fen) >= 5
        halfmove_clock = parse(Int64, fen[5])
    else
        halfmove_clock = 1
        fullmove_clock = 0
    end
    if length(fen) == 6
        fullmove_clock = parse(Int64, fen[6])
    end

    game = Array{String,1}[]

    A = [EMPTY, EMPTY, EMPTY, EMPTY, EMPTY]
    a = [EMPTY, EMPTY, EMPTY, EMPTY, EMPTY]
    board = Bitboard(white, P, R, N, B, Q, K, A,
                    black, p, r, n, b, q, k, a,
                    free, taken,
                    white_attacks, black_attacks,
                    player_color,
                    white_can_castle_queenside,
                    white_can_castle_kingside,
                    black_can_castle_queenside,
                    black_can_castle_kingside,
                    white_king_moved, black_king_moved,
                    enpassant_square,
                    enpassant_done,
                    halfmove_clock, fullmove_clock,
                    fen_string, game)
    return update_attacked(board)
end


module DynamicsUtils

using NQCDynamics:
    AbstractSimulation,
    Simulation,
    RingPolymerSimulation,
    Calculators,
    masses

divide_by_mass!(dv, masses) = dv ./= masses'
velocity!(dr, v, r, sim, t) = dr .= v

"""
    apply_interbead_coupling!(du::DynamicalVariables, u::DynamicalVariables,
                              sim::RingPolymerSimulation)

Applies the force that arises from the harmonic springs between adjacent beads.

Only applies the force for atoms labelled as quantum within the `RingPolymerParameters`.
"""
function apply_interbead_coupling!(dr::AbstractArray{T,3}, r::AbstractArray{T,3}, sim::RingPolymerSimulation) where {T}
    for i in axes(dr, 3)
        iplus = mod1(i+1, sim.beads.n_beads)
        iminus = mod1(i-1, sim.beads.n_beads)
        for j in sim.beads.quantum_atoms
            for k in axes(dr, 1)
                dr[k,j,i] -= sim.beads.ω_n² * (2r[k,j,i] - r[k,j,iplus] - r[k,j,iminus])
            end
        end
    end
    return nothing
end

function classical_hamiltonian end

function classical_kinetic_energy(sim::Simulation, v::AbstractMatrix)
    kinetic = zero(eltype(v))
    for i in axes(v, 2)
        for j in axes(v, 1)
            kinetic += masses(sim, i) * v[j,i]^2
        end
    end
    return kinetic / 2
end

function classical_kinetic_energy(sim::RingPolymerSimulation, v::AbstractArray{T,3}) where {T}
    kinetic = zero(eltype(v))
    for k in axes(v, 3)
        for i in axes(v, 2)
            for j in axes(v, 1)
                kinetic += masses(sim, i) * v[j,i,k]^2
            end
        end
    end
    return kinetic / 2
end

function classical_potential_energy(sim::Simulation, r::AbstractMatrix)
    Calculators.evaluate_potential!(sim.calculator, r)
    sim.calculator.potential
end

function classical_potential_energy(sim::RingPolymerSimulation, r::AbstractArray{T,3}) where {T}
    Calculators.evaluate_potential!(sim.calculator, r)
    sum(sim.calculator.potential) + RingPolymers.get_spring_energy(sim.beads, masses(sim), r)
end

include("dynamics_variables.jl")
include("callbacks.jl")
export CellBoundaryCallback
export TerminatingCallback

include("density_matrix_dynamics.jl")
include("plot.jl")

end # module