################################################################################
#
#	NEW DEFINITIONS OF LATTICES AND UNITCELLS
#
################################################################################

# usings
using LinearAlgebra

################################################################################
#
#	ABSTRACT TYPES
#
################################################################################

# Bonds
abstract type AbstractBond{L,N} end

# Sites
abstract type AbstractSite{L,D} end

# Unitcells
abstract type AbstractUnitcell{D,N,L,B<:AbstractBond{L,N},S<:AbstractSite{L,D}} end

# Lattices
abstract type AbstractLattice{D,N,L,B<:AbstractBond{L,N},S<:AbstractSite{L,D},U<:AbstractUnitcell{D,N,L,B,S}} end

################################################################################
#
#	INTERFACING / ACCESSING
#	ABSTRACT TYPES
#	(have to be overwritten by concrete types)
#
################################################################################

# BONDS

# default constructor interface
function newBond(index_from::Int64, index_to::Int64, label::L, wrap::NTuple{N,Int64}, ::Type{B}) :: B where {L,N,B<:AbstractBond{L,N}} error("not implemented function 'newBond' for bond type $(B) with label type $(L) and wrap length $(N)") end

# from and to index
function indexFrom(b::AbstractBond{L,N}) :: Int64 where {L,N} error("not implemented interface function 'indexFrom' for bond type $(typeof(b))") end
function indexTo(b::AbstractBond{L,N})   :: Int64 where {L,N} error("not implemented interface function 'indexTo' for bond type $(typeof(b))") end

# label
function label(b::AbstractBond{L,N}) :: L where {L,N} error("not implemented interface function 'label' for bond type $(typeof(b))") end

# wrap
function wrap(b::AbstractBond{L,N}) :: NTuple{N,Int64} where {L,N} error("not implemented interface function 'wrap' for bond type $(typeof(b))") end

# TESTING THE BOND INTERFACE
function testInterface(::Type{T}) where {T<:AbstractBond}
	# get the parameterless constructor
	B = Base.typename(T).wrapper
	# get a new bond
	bond_1 = newBond(1, 1, 1.0, (1,1), B{Float64, 2})
	bond_2 = newBond(1, 1, "tx", (1,1), B{String, 2})
	bond_3 = newBond(1, 1, 2, (1,1,1), B{Int64, 3})
	# get the indices and labels and wraps
	for b in [bond_1, bond_2, bond_3]
		indexFrom(b)
		indexTo(b)
		label(b)
		wrap(b)
	end
	# return nothing
	return nothing
end

# SITES

# default constructor interface
function newSite(position::Vector{Float64}, label::L, ::Type{S}) :: S where {L,S<:AbstractSite{L,D} where D} error("not implemented function 'newSite' for site type $(S) with label type $(L)") end

# position
function point(s::AbstractSite{L,D}) :: Vector{Float64} where {L,D} error("not implemented interface function 'point' for site type $(typeof(s))") end

# label
function label(s::AbstractSite{L,D}) :: L where {L,D} error("not implemented interface function 'label' for site type $(typeof(s))") end

# TESTING THE SITE INTERFACE
function testInterface(::Type{T}) where {T<:AbstractSite}
	# get the parameterless constructor
	S = Base.typename(T).wrapper
	# get a new site
	site_1 = newSite([1.0, 0.0], "site_1", S{String,2})
	site_2 = newSite([1.0, 0.0, 0.0], 1.0, S{Float64,3})
	site_3 = newSite([1.0, 0.0], 1, S{Int64,2})
	# get the points and labels
	for s in [site_1, site_2, site_3]
		point(s)
		label(s)
	end
	# return nothing
	return nothing
end

# DEFINITION OF A BOND
# L - Label type
# N - dimension of the Bravais lattice (i.e. number of primitive directions to wrap)
mutable struct Bond{L,N} <: AbstractBond{L,N}

	# indices that the bond connects
	from	::Int64
	to		::Int64

	# label
	label	::L

	# wrap
	wrap	::NTuple{N, Int64}

end

# IMPLEMENTATION OF BOND RELATED FUNCTIONS

# creation
function newBond(index_from::Int64, index_to::Int64, label::L, wrap::NTuple{N,Int64}, ::Type{Bond{L,N}}) :: Bond{L,N} where {L,N} return Bond{L,N}(index_from, index_to, label, wrap) end

# from and to index
function indexFrom(b::Bond{L,N}) :: Int64 where {N,L} return b.from end
function indexTo(b::Bond{L,N})   :: Int64 where {N,L} return b.to end

# label
function label(b::Bond{L,N}) :: L where {N,L} return b.label end

# wrap
function wrap(b::Bond{L,N}) :: NTuple{N,Int64} where {N,L} return b.wrap end

# DEFINITION OF A SITE
# L - Label type
# D - dimension of the real space in which the site is located
mutable struct Site{L,D} <: AbstractSite{L,D}

	# label
	label	:: L

	# point
	point	:: Vector{Float64}

end

# IMPLEMENTATION OF SITE RELATED FUNCTIONS

# default constructor interface
function newSite(position::Vector{Float64}, label::L, ::Type{Site{L,D}}) :: Site{L,D} where {L,D} return Site{L,D}(label, position) end

# position
function point(s::Site{L,D}) :: Vector{Float64} where {L,D} return s.point end

# label
function label(s::Site{L,D}) :: L where {L,D} return s.label end

# DEFINITION OF A UNITCELL
# D - space dimension
# N - dimension of the Bravais lattice
# L - label type
mutable struct Unitcell{D,N,L,B,S} <: AbstractUnitcell{D,N,L,B,S}

    # FIELDS

    # basis vectors of the Bravais lattice
    lattice_vectors	::Vector{Vector{Float64}}

    # basis sites within the unitcell
    sites			::Vector{S}

    # list of bonds
    bonds			::Vector{B}

end

# Unitcell obtaining (2D - Square)
function getUnitcellSquare_New()::Unitcell_New{2,2,Float64}
	# return the unitcell
	return Unitcell_New{2,2,Float64}(
		# Bravais lattice vectors
	 	Vector{Float64}[
	    	[1.0, 0.0],
	    	[0.0, 1.0]
		],
	    # Basis Definition
	    Vector{Float64}[
	        [0.0, 0.0]
	    ],
	    # Bond Definition
	    Bond{Float64,2}[
	        Bond(1,1, 1.0, ( 0, 1) ),
	        Bond(1,1, 1.0, ( 0,-1) ),
	        Bond(1,1, 1.0, ( 1, 0) ),
	        Bond(1,1, 1.0, (-1, 0) )
	    ]
	)
end

# Unitcell obtaining (3D - SC)
function getUnitcellSC_New()::Unitcell_New{3,3,Float64}
	# return the unitcell
	return Unitcell_New{3,3,Float64}(
		# Bravais lattice vectors
		Vector{Float64}[
	        [1.0, 0.0, 0.0],
	        [0.0, 1.0, 0.0],
	        [0.0, 0.0, 1.0]
		],
        # Basis Definition
        Vector{Float64}[
            [0.0, 0.0, 0.0]
        ],
		# Bond Definition
	    Bond{Float64,3}[
            Bond(1, 1, 1.0, ( 1, 0, 0)),
            Bond(1, 1, 1.0, (-1, 0, 0)),
            Bond(1, 1, 1.0, (0,  1, 0)),
            Bond(1, 1, 1.0, (0, -1, 0)),
            Bond(1, 1, 1.0, (0, 0,  1)),
            Bond(1, 1, 1.0, (0, 0, -1)),
        ]
	)
end

# DEFINITION OF A LATTICE
# D - space dimension
# N - dimension of the Bravais lattice
# L - label type
mutable struct Lattice_New{D,N,L}

    # FIELDS

	# The unitcell of the lattice if applicable
    unitcell			::Unitcell_New{D,N,L}
	# how often the unitcell is repeated in the elementary directions
    unitcellRepetitions	::Array{Int64, 1}

    # basis vectors of the Bravais lattice
    lattice_vectors		::Array{Vector{Float64}, 1}

    # basis sites within the unitcell
    sites				::Array{Vector{Float64}, 1}
	# array of ints that give the index of the site in the original unit cell
    site_indices		::Array{Int64, 1}

    # list of bonds
    bonds				::Array{Bond{L,N}, 1}

end

# define the index function to get the index of an element in the array
function index(
			i1::Int64, i2::Int64,
			N1::Int64, N2::Int64
		) :: Int64
	return (i1-1)*N2 + i2
end
function index(
			i1::Int64, i2::Int64, i3::Int64,
			N1::Int64, N2::Int64, N3::Int64
		) :: Int64
	return N3*((i1-1)*N2+i2-1) + i3
end
function index(
			i1::Int64, i2::Int64, i3::Int64, i4::Int64,
			N1::Int64, N2::Int64, N3::Int64, N4::Int64
		) :: Int64
	return N4*(N3*((i1-1)*N2+i2-1) + i3 -1) + i4
end

# function to constrain an index
# returns (index_constrained, offset)
# s.t. index = index_constraint + offset*index_max
# To be replaced by ic = div(.,.) and offset = rem(.,.) ???
# apparently not faster...
function getConstrainedIndex(
		index		::Int64,
		index_max	::Int64
	) :: Tuple{Int64, Int64}
	# specify the initial offset as 0
	offset = 0 :: Int64
	# check that the index is bigger than the minimum value
	while index < 1
		index += index_max
		offset += -1
	end
	# check that the index is smaller than the maximum value
	while index > index_max
		index -= index_max
		offset += 1
	end
	# return both index and offset
	return (index, offset)
end

# Construction of periodic lattice (in 2D)
function getLatticePeriodic_New(
			unitcell			::Unitcell_New{D,2,L},
			repetition_array	::Array{Int64, 1}
		) :: Lattice_New{D,2,L} where {D,L}

    # extract the cardinal directions of the lattice from the array
    N_a1	= abs(repetition_array[1]) 	:: Int64
    N_a2	= abs(repetition_array[2]) 	:: Int64
	N_sites = length(unitcell.sites)	:: Int64
	N_bonds = length(unitcell.bonds)	:: Int64

    # GENERATE NEW POSITIONS
	sites 				= Array{Vector{Float64},1}(undef, N_a1*N_a2*N_sites)
    site_indices 		= Array{Int64,1}(undef, N_a1*N_a2*N_sites)

	# set all positions to their correct values
	for i in 1:N_a1
	for j in 1:N_a2
	    @simd for a in 1:N_sites
			# calculate the site index
			site_index 					= index(i,j,a, N_a1,N_a2,N_sites)
			# calculate the position
			sites[site_index]			= unitcell.sites[a] .+ (i.*unitcell.lattice_vectors[1]) .+ (j.*unitcell.lattice_vectors[2])
			# set the site index
	        site_indices[site_index]	= a
		end
	end
    end

    # GENERATE NEW CONNECTIONS
	bonds = Array{Bond{L,2},1}(undef, N_a1*N_a2*N_bonds)

	# iterate over all unit cells
	for i in 1:N_a1
	for j in 1:N_a2
		# add all connections for unitcell (i,j)
		@simd for b in 1:N_bonds
			# get the respective bond
			bond = unitcell.bonds[b] :: Bond{L,2}
			# calculate the index from where the connection goes
			index_from = index(i,j,bond.from, N_a1,N_a2,N_sites)
			# calculate the aimed unitcell
			i_to = i + bond.wrap[1]
			j_to = j + bond.wrap[2]
			# check if the connection goes around in a1 direction
            (i_to, offset_a1) = getConstrainedIndex(i_to, N_a1)
			# check if the connection goes around in a2 direction
            (j_to, offset_a2) = getConstrainedIndex(j_to, N_a2)
			# get the index to where the connection goes
			index_to = index(i_to, j_to, bond.to, N_a1,N_a2,N_sites)
            # generate a new connection and set it in the list
			bonds[index(i,j,b, N_a1,N_a2,N_bonds)] = Bond{L,2}(index_from, index_to, bond.label, (offset_a1, offset_a2))
		end
	end
	end

    # generate new lattice vectors, now spanning the WHOLE lattice
    lattice_vectors = Vector{Float64}[
			# multiply by extent of lattice
			unitcell.lattice_vectors[1] .* N_a1,
			unitcell.lattice_vectors[2] .* N_a2
		]

    # save everything to a Lattice object
    lattice = Lattice_New{D,2,L}(
        	unitcell,
        	Int64[N_a1, N_a2],
        	lattice_vectors,
        	sites,
        	site_indices,
        	bonds
        )

    # return the lattice
    return lattice

end

# Construction of periodic lattice (in 3D)
function getLatticePeriodic_New(
			unitcell			::Unitcell_New{D,3,L},
			repetition_array	::Array{Int64, 1}
		) :: Lattice_New{D,3,L} where {D,L}

    # extract the cardinal directions of the lattice from the array
    N_a1	= abs(repetition_array[1]) 	:: Int64
    N_a2	= abs(repetition_array[2]) 	:: Int64
    N_a3	= abs(repetition_array[3]) 	:: Int64
	N_sites = length(unitcell.sites)	:: Int64
	N_bonds = length(unitcell.bonds)	:: Int64

    # GENERATE NEW POSITIONS
	sites 				= Array{Vector{Float64},1}(undef, N_a1*N_a2*N_a3*N_sites)
    site_indices 		= Array{Int64,1}(undef, N_a1*N_a2*N_a3*N_sites)

	# set all positions to their correct values
	for i in 1:N_a1
	for j in 1:N_a2
	for k in 1:N_a3
	    @simd for a in 1:N_sites
			# calculate the site index
			site_index 					= index(i,j,k,a, N_a1,N_a2,N_a3,N_sites)
			# calculate the position
			sites[site_index]			= unitcell.sites[a] .+ (i.*unitcell.lattice_vectors[1]) .+ (j.*unitcell.lattice_vectors[2]) .+ (k.*unitcell.lattice_vectors[3])
			# set the site index
	        site_indices[site_index]	= a
		end
	end
	end
    end

    # GENERATE NEW CONNECTIONS
	bonds = Array{Bond{L,3},1}(undef, N_a1*N_a2*N_a3*N_bonds)

	# iterate over all unit cells
	for i in 1:N_a1
	for j in 1:N_a2
	for k in 1:N_a3
		# add all connections for unitcell (i,j)
		@simd for b in 1:N_bonds
			# get the respective bond
			bond = unitcell.bonds[b] :: Bond{L,3}
			# calculate the index from where the connection goes
			index_from = index(i,j,k,bond.from, N_a1,N_a2,N_a3,N_sites)
			# calculate the aimed unitcell
			i_to = i + bond.wrap[1] ::Int64
			j_to = j + bond.wrap[2] ::Int64
			k_to = k + bond.wrap[3] ::Int64
			# check if the connection goes around in a1 direction
            (i_to, offset_a1) = getConstrainedIndex(i_to, N_a1)
			# check if the connection goes around in a2 direction
            (j_to, offset_a2) = getConstrainedIndex(j_to, N_a2)
			# check if the connection goes around in a3 direction
            (k_to, offset_a3) = getConstrainedIndex(k_to, N_a3)
			# get the index to where the connection goes
			index_to = index(i_to, j_to, k_to, bond.to, N_a1,N_a2,N_a3,N_sites)
            # generate a new connection and set it in the list
			bonds[index(i,j,k,b, N_a1,N_a2,N_a3,N_bonds)] = Bond{L,3}(index_from, index_to, bond.label, (offset_a1, offset_a2, offset_a3))
		end
	end
	end
	end

	# generate new lattice vectors, now spanning the WHOLE lattice
    lattice_vectors = Vector{Float64}[
			# multiply by extent of lattice
			unitcell.lattice_vectors[1] .* N_a1,
			unitcell.lattice_vectors[2] .* N_a2,
			unitcell.lattice_vectors[3] .* N_a3
		]

    # save everything to a Lattice object
    lattice = Lattice_New{D,3,L}(
        	unitcell,
        	Int64[N_a1, N_a2, N_a3],
        	lattice_vectors,
        	sites,
        	site_indices,
        	bonds
        )

    # return the lattice
    return lattice

end


# version 0.1

module EcoEvo3D

using StatsBase

function getSampleFromArray(items)
  return sample(items)
end

function readLocations(filename)
	dat = readdlm(filename,'\t');
	@inbounds w = hcat(dat[:,2],dat[:,3]);
	w;
end

function readDistanceMatrix(filename)
	dat = readdlm(filename,' ');
	dat;
end

function changediagonal!(MA,nrows,value)
	@inbounds for(i in 1:nrows)
		@inbounds MA[i,i]=value;
	end
end

function createMRM(MRM,Sti,Sp,ts)
	MRM = [Sti Sp 1 ts];
	MRM;
end

function createMC(MC,Sti,Sp,ts)
	MC = [Sti Sp 1 ts];
	return MC;
end

function createMA(MA,Sti,Stj,Sp,anaG)
	MA = [Sti Stj Sp anaG];
	return MA;
end

function updateMC(MC)
	if length(MC)>0
		MC[:,3] = MC[:,3].+1;#event: -1 for extinction; 1 for speciation
	end
	return MC;
end

function updateMA(MA)
	if length(MA)>0
		MA[:,4] = MA[:,4] .- 1;
	end
	return MA;
end

function checkIfThereIsMRM(MRM,Sti,Sp,ts)
	if length(MRM)==0
		MRM = createMRM(MRM,Sti,Sp,ts);
	else
		pos = find( (MRM[:,1].==Sti) & (MRM[:,2].==Sp))
		if length(pos)==0
			MRM = cat(1,MRM,[Sti Sp 1 ts]);
		end
	end
	return MRM;
end

function checkIfThereIsMC(MC,Sti,Sp,ts)
	if length(MC)==0
		MC = createMC(MC,Sti,Sp,ts);
	else
		pos = find( (MC[:,1].==Sti) & (MC[:,2].==Sp))#position in the matrix MA referred to the presence of individuals of species 'Sp' coming from site 'Stj' to site 'Sti'
		if length(pos)==0
			MC = cat(1,MC,[Sti Sp 1 ts]);
		end
	end
	return MC;
end

function GetGammaRichness(R,S)
	richnessspeciesR = [];
	#%gamma richness
	for (i in 1:S)
		AR = sort(R[i])
		richnessspeciesR = [richnessspeciesR; unique(AR)];
	end;
	extantR = length(unique(sort(richnessspeciesR)));
	gamma = extantR;

	return gamma;
end

function OutputPerGeneration(outputfilepergen,ri,cost,J,G,S,k,anaG,retG,mr,ml,v,lakesIsolation,lakesArea,gamma,alpharich,SpecANA,SpecCLA,SpecMR,DispersalRich)
	for i in 1:S
		writedlm(outputfilepergen, [ri cost J G k anaG retG mr ml v gamma i lakesIsolation[i] lakesArea[i] alpharich[i] SpecANA[i] SpecCLA[i] SpecMR[i] DispersalRich[i]],' ');
	end
	flush(outputfilepergen);#To print in the output file each realization
	return;
end

function printPhylogeny(phylogenyfile,old,new,ts,ri)
	writedlm(phylogenyfile,[ri old new ts],' ');
        flush(phylogenyfile);
end

function UAB(MA,MC)
	MC = updateMC(MC);
	MA = updateMA(MA);
	return MC,MA;
end

function UAC(MA)
	MA = updateMA(MA);
	return MA;
end

function UARM(MA,MC)
	MC = updateMC(MC);
	MA = updateMA(MA);
	return MC,MA;
end

function UALM(MA,MC,R,Sti,Stj,Sp,anaG,retG,ts)
	MC = updateMC(MC);
	MA = updateMA(MA);

	if length(MA)==0 #Si no hay MA
  	MA = createMA(MA,Sti,Stj,Sp,anaG);#Create MA
	else#Si hay MA
		pos = find( (MA[:,1].==Sti) & (MA[:,2].==Stj) & (MA[:,3].==Sp))#position in the matrix MA referred to the presence of individuals of species 'Sp' coming from site 'Stj' to site 'Sti'
		if length(pos)==0 #No hay la linea
			indalive = length(find(R[Sti].==Sp))#Hay individuos de la specie sp vivos en el sitio Sti
			if (indalive == 0) #Checking if there are individuals of species 'Sp' alive in site 'Sti'
#        			println("<<<<<<<<<<< A D D    R O W   T O    A N A G E N E S I S   <<<<<<<<<<<<<<<<<");
				MA = cat(1,MA,[Sti Stj Sp anaG]);#Create the row in MA matrix, starting by 1
      			end
		else #Ya hay la linea
#			println("<<<<<<<<<<< R E T A R D    A N A G E N E S I S   <<<<<<<<<<<<<<<<<");
			MA[pos,4] = MA[pos,4] .+ retG;#retards anagenesis by increasing the remaining
		end
	end
  	return MC,MA,R;
end

########################

function checkAna(MA,R,anaG,lastspecies,listofanagenesis,ts,phylogenyfile,ri)
	pos = [];
	if length(MA)>0
		pos = find(MA[:,4] .== 0)#find the anagenesis events
		if length(pos)>0#is there was an anagenesis event
		  @inbounds for (p in pos)
  			Sti=round(Integer,MA[p,1]);#pos represents the rows of the matrix. p is one row. MA[p,1] is the target site
  			Stj=round(Integer,MA[p,2]);#pos represents the rows of the matrix. p is one row. MA[p,2] is the source site
		  	Sp=round(Integer,MA[p,3]);#pos represents the rows of the matrix. p is one row. MA[p,3] is the ancient species
			  MA, R, lastspecies,listofanagenesis = AnagenesisSpeciation(MA,R,Sti,Stj,Sp,lastspecies,listofanagenesis,ts,phylogenyfile,ri);#speciation in target site
#			  MA, R, lastspecies,listofanagenesis = MA, R, lastspecies, listofanagenesis;
		  end
	  end
  end
  return MA,R,lastspecies,listofanagenesis;
end

function AnagenesisSpeciation(MA,R,Sti,Stj,Sp,lastspecies,listofanagenesis,ts,phylogenyfile,ri)
	newspeciesAna = lastspecies + 1;#the id of the new species
	ancientindividuals = find( R[Sti].==Sp )#the position of all the individuals of the ancient species 'Sp' in the target site
	R[Sti][ancientindividuals] = newspeciesAna;#the speciation itself: all the individuals of former species 'Sp' in the target site are now from a new species 'newspeciesAna'
        println("new ANAgenesis Speciation : Species ",newspeciesAna," - time: ",ts);
 	pos = find( (MA[:,1].==Sti) & (MA[:,3].==Sp))#position in the matrix MA referred to the presence of individuals of species 'Sp' in site 'Sti'
	MA = MA[1:size(MA,1).!=pos,:];#Borra la linea 'pos' de la matriz MA!!
	printPhylogeny(phylogenyfile,Sp,newspeciesAna,ts,ri);

  	if length(listofanagenesis)>0
		listofanagenesis = cat(1,listofanagenesis,[Sti newspeciesAna]);
	else
		listofanagenesis = cat(1,[Sti newspeciesAna]);
	end

	return MA,R,newspeciesAna,listofanagenesis;
end

function CladogenesisEvent(MC,R,Sti,Individual,lastspecies,ts,phylogenyfile,ri)
	newspeciesClado = lastspecies + 1;
	printPhylogeny(phylogenyfile,R[Sti][Individual],newspeciesClado,ts,ri);
        println("new CLADOgenesis Speciation : Species ",newspeciesClado," - time: ",ts);
   	R[Sti][Individual] = newspeciesClado;
	MC = checkIfThereIsMC(MC,Sti,newspeciesClado,ts);
	return MC,R,newspeciesClado;
end

function LocalMigrationEvent(R,KillHab,MigrantHab,KillInd,Dc,Ji,S)
	MigrantSpecies = -1;

	allpos = find(Dc[KillHab,:] .>= MigrantHab);#All the sites at a distance lower than the threshold 'MigrantHab'
	kr = minimum(allpos[find(allpos .!= KillHab)]);
	MigrantInd = rand(1:Ji[kr]);
	MigrantSpecies = R[kr][MigrantInd];
#        println("LOCAL Migration");
	R[KillHab][KillInd] = R[kr][MigrantInd];
	return kr,R,MigrantSpecies;
end;

function RegionalMigrationEvent(MRM,R,Sti,Individual,ts,lastspecies)
  newspeciesMR = lastspecies + 1;
  R[Sti][Individual] = newspeciesMR;
  println("new RegionalMig Speciation: Species ",newspeciesMR," - time: ",ts)
  MRM = checkIfThereIsMRM(MRM,Sti,newspeciesMR,ts);

  return MRM,R,newspeciesMR;
end;

function BirthEvent(R,BirthLocal,KillInd,KillHab)
	R[KillHab][KillInd] = R[KillHab][BirthLocal];
	return R;
end

function calculateSpeciationMA(listofanagenesis,R)
	S = length(R);#number of sites equals the number of rows in R
	speciatedMA = zeros(S);
	if (length(listofanagenesis) > 0)
		@inbounds for i in 1:S
			@inbounds speciesR = unique(sort(R[i]))';
			pos = find(listofanagenesis[:,1].== i);
			if length(pos)>0
				speciesMA = listofanagenesis[pos,2];
				speciatedMA[i] = length(setdiff(speciesMA,setdiff(speciesMA,speciesR))');
			end
		end
	end
	return speciatedMA;
end

function calculateSpeciationMC(MC,R,S,k,Ji)
	speciatedMC = zeros(S);
	if (length(MC) > 0)
		@inbounds for i in 1:S
		@inbounds speciesR = unique(sort(R[i]))';
			pos = find( (MC[:,1].== i) & (MC[:,3].>= k) );#Only the species that are in the system for more than k time steps (Protracted)
			if length(pos)>0
				speciesMC = unique(sort(MC[pos,2]));
				speciatedMC[i] = length(setdiff(speciesMC,setdiff(speciesMC,speciesR))');
			end
		end
	end
	return speciatedMC;
end

function calculateSpeciationMR(MRM,R,S,Ji)
	speciatedMRM = zeros(S);
	if (length(MRM) > 0)
		@inbounds for i in 1:S
      @inbounds speciesR = unique(sort(R[i]))';
      pos = find(MRM[:,1].== i);#check for site i
      if length(pos)>0
        speciesMRM = unique(sort(MRM[pos,2]));
        @inbounds speciatedMRM[i] = length(setdiff(speciesMRM,setdiff(speciesMRM,speciesR))');
      end
		end
	end
#	writedlm("MRMmatrix.dat",MRM,' ');
	return speciatedMRM;
end

function richnessanalysis!(S,R,Ji,richnessspeciesR,alpharich)
	#%gamma richness
	@inbounds for (i in 1:S)
		@inbounds AR = sort(R[i]);
		richnessspeciesR = [richnessspeciesR; unique(AR)];
		@inbounds alpharich[i] = length(unique(AR));
	end;
	gamma = length(unique(sort(richnessspeciesR)));
	return gamma,alpharich;
end

function normalizeDc!(Dc,S)
  for(i in 1:S)
    Dc[i,:] = Dc[i,:]/maximum(Dc[i,:]);
  end
end

function repeatval(x,N)
  return map(v -> v=x,1:N);
end

#Each site starts with one different species
function initialPopulation(R,Ji)
  lastspecies = 0;
  for(iJ in Ji)#iterating over columns
     lastspecies = lastspecies+1;
     popi = repeatval(lastspecies,iJ);
     push!(R,popi);
  end
  return R,lastspecies;
end

function dynamic(seed,nreal,Gmax,J,v,mr,ml,anaG,retG,distmatfile,verticesdata,model)
	DI = readDistanceMatrix(distmatfile);#the location of the points of the landscape.
	Dc = cumsum(DI,2);#Should we divide
  dT = sum(DI,2);
	const S = length(DI[:,1]);#Number of sites
  normalizeDc!(Dc,S);
#### To get the cost
        substring = distmatfile[search(distmatfile,'_')+1:end];
        modelstop = search(substring,'_');
        coststop = search(substring,'t');
	const cost = float(substring[coststop+1:modelstop-1]);

	sitesdata = readdlm(verticesdata,' ',header=true)[1];#Proportion of individuals of each site
#	Pj = ones(S);#Proportion of individuals of each site - if an input file is not defined, the proportion is the same for each site
        lakesVolume = sitesdata[:,2];
        lakesArea = sitesdata[:,3];
	Pj = lakesVolume;#In case we define different carrying capacities for each site we consider the volume of the lakes (third column)
	mins = find(sitesdata.==minimum(sitesdata[:,1]));#the first column represents the height of the lakes
	entrypoint = mins[rand(1:length(mins))];
	t=1;#Sites have different sizes and are located at different height.
	Ji=round(Integer,J * Pj/sum(Pj));

	outputfilepergen = open(string("RichnessPerGen_AnaG_",anaG,"_retG_",retG,"_cost_",cost,"_MR_",signif(mr,3),"_VR_",signif(v,3),".txt"),"a")
	writedlm(outputfilepergen, ["Real Cost J G Gi anaG retG mr ml v gamma Site dT lakeArea alpharich SpecANA SpecCLA SpecMR DispersalRich"]);

	outputfile = open(string("RichnessPerSite_AnaG_",anaG,"_retG_",retG,"_cost_",cost,"_MR_",signif(mr,3),"_VR_",signif(v,3),".txt"),"a")
	writedlm(outputfile,["Real Cost Model J G anaG retG Site lakeArea Ji dT mr ml v gamma alpharich SpecANA SpecCLA SpecMR DispersalRich"]);

	phylogenyfile = open(string("Phylogeny_AnaG_",anaG,"_retG_",retG,"_cost_",cost,"_MR_",signif(mr,3),"_VR_",signif(v,3),".txt"),"a")
	writedlm(phylogenyfile,["Repl Ancestral Derived Age"]);

  @inbounds for (ri in 1:nreal)#realizations
 	  lastspecies = 0;
  	ts=0;
    R=[];
    R,lastspecies = initialPopulation(R,Ji)
	  MA = Array(Number,0,0);#Matrix to calculate Anagenesis speciation (MA = [Sti, Stj, S, C])
	  MC = Array(Number,0,0);#Matrix to control Cladogenesis speciation (MC = [Sti, S, E])
	  MRM = Array(Number,0,0);#Matrix to control events of Regional Migration (MRM = [Sti, S, ts])
		srand(seed+(7*ri));
		listofanagenesis = Array(Number,0,0);

		#%Resources
#		G = rand(15:Gmax);#Minimum of 15 generations
		G = Gmax;#Minimum of 15 generations

		@inbounds for (k = 1:G)#%population-metapopulation-metacommunity dynamics (not-tracking multitrophic metacommunity dynamics!)
			@inbounds for (j = 1:J)#For each individual in each site
				ts = ts+1;#Time step increases
				mvb = rand();
		       		#Demography - Resources
    				KillHab = rand(1:S);#which site to kill
				KillInd = rand(1:Ji[KillHab]);#which individual to kill
        			KillIndEntryPoint = rand(1:Ji[entrypoint]);
				MigrantHab = rand()*maximum(Dc[KillHab,:]);
				BirthLocal = getSampleFromArray(1:length(R[KillHab]));#which individual to born

				if mvb <= ml;#Local Migration event
					kr,R,MigrantSpecies = LocalMigrationEvent(R,KillHab,MigrantHab,KillInd,Dc,Ji,S);
					MC,MA,R = UALM(MA,MC,R,KillHab,kr,MigrantSpecies,anaG,retG,ts);#Update Anagenesis after Local Migration event
				elseif (ml < mvb <= (ml+mr));#Regional Migration event
					MRM,R,lastspecies = RegionalMigrationEvent(MRM,R,entrypoint,KillIndEntryPoint,ts,lastspecies);#Speciation through the entry point
					MC,MA = UARM(MA,MC);#Update Anagenesis after Regional Migration event
				elseif ((ml+mr) < mvb <= (ml+mr+v))
					if(v > 0)#we only simulate Cladogenesis when the probability is higher than 0
						MC,R,lastspecies = CladogenesisEvent(MC,R,KillHab,KillInd,lastspecies,ts,phylogenyfile,ri);
						MA = UAC(MA);#Update Anagenesis after Cladogenesis Speciation
					end
				else #Birth event
					R = BirthEvent(R,BirthLocal,KillInd,KillHab);#Birth event
					MC,MA = UAB(MA,MC);#Update MA after Birth event
		       		end;
####ANAGENESIS
				lengthlistbefore = length(listofanagenesis);#Size of Anagenesis Matrix before checking for Anagenesis
				MA,R,lastspecies,listofanagenesis = checkAna(MA,R,anaG,lastspecies,listofanagenesis,ts,phylogenyfile,ri); #After the update of matrix MA, we check for events of Anagenesis
				lengthlistafter = length(listofanagenesis);#Size of Anagenesis Matrix after checking for Anagenesis
####CLADOGENESIS
#				if ((lengthlistafter - lengthlistbefore) > 0)#If there is at least one Anagenesis Speciation Event
#					mvc = rand();
#					if (mvc <= v);#Probability to occur a Cladogenesis Speciation event
#						allInd = find(R[St,:].==lastspecies);#All individuals from the new species
#						if (length(allInd)>0)
#							KillInd = allInd[rand(1:length(allInd))];#Randomly choose an individual from the new species
#							MC,R,lastspecies = CladogenesisEvent(MC,R,KillHab,KillInd,lastspecies,ts,phylogenyfile,ri);#Cladogenesis event
#							MA = UAC(MA);#Update Anagenesis after Cladogenesis Speciation event
#						end #if allInd
#					end #if mvc
#				end #if lengthlistafter
			end;#end S*Ji
			richnessspeciesR = [];
			alpharich = zeros(S);
			gamma,alpharich = richnessanalysis!(S,R,Ji,richnessspeciesR,alpharich);
			SpecANA = calculateSpeciationMA(listofanagenesis,R);
			SpecCLA = calculateSpeciationMC(MC,R,S,anaG,Ji);
			SpecMR = calculateSpeciationMR(MRM,R,S,Ji);
			DispersalRich = alpharich - (SpecANA + SpecCLA + SpecMR);
			OutputPerGeneration(outputfilepergen,ri,cost,J,G,S,k,anaG,retG,mr,ml,v,dT,lakesArea,gamma,alpharich,SpecANA,SpecCLA,SpecMR,DispersalRich);
		end;#end Gmax

		#To analyze the resulting richness
		richnessspeciesR = [];
		alpharich = zeros(S);
		gamma,alpharich = richnessanalysis!(S,R,Ji,richnessspeciesR,alpharich);
		SpecANA = calculateSpeciationMA(listofanagenesis,R);
		SpecCLA = calculateSpeciationMC(MC,R,S,anaG,Ji);
		SpecMR = calculateSpeciationMR(MRM,R,S,Ji);
		DispersalRich = alpharich - (SpecANA + SpecCLA + SpecMR);
#		matrixfile = open("Rmatrix.txt","a");
		for i in 1:S
			writedlm(outputfile,[ri cost model J G anaG retG i lakesArea[i] Ji[i] dT[i] mr ml v gamma alpharich[i] SpecANA[i] SpecCLA[i] SpecMR[i] DispersalRich[i]],' ');
#                        for j in 1:length(R[i])
#	                        print(matrixfile,R[i][j],' ');
#			end
#	                print(matrixfile,'\n');
		end
		flush(outputfile);
#		ANAfile = open("ANAmatrix.txt","a");
#		MRMfile = open("MRMmatrix.txt","a");
#		rowsANA = size(listofanagenesis,1);rowsMRM = size(MRM,1);
#		colsANA = size(listofanagenesis,2);colsMRM = size(MRM,2);
#		for i in 1:rowsANA #number of rows
#			for j in 1:colsANA
#	                        print(ANAfile,listofanagenesis[i,j],' ');
#			end
#	                print(ANAfile,'\n');
#		end
#		for i in 1:rowsMRM #number of rows
#			for j in 1:colsMRM
#	                        print(MRMfile,MRM[i,j],' ');
#			end
#	                print(MRMfile,'\n');
#		end

	end#%ri
	close(outputfile);
	close(outputfilepergen);
	close(phylogenyfile);
#	close(logfile);
end#ecoevo3d function

end #module


using CUDA
import Zygote
using Base: issingletontype
using .NeuralNetwork

const use_cuda = Ref{Union{Nothing,Bool}}(nothing)

function check_use_cuda()
  if use_cuda[] === nothing
    use_cuda[] = CUDA.functional()
    if use_cuda[] && !CUDA.has_cudnn()
      @warn "CUDA.jl found cuda, but did not find libcudnn. Some functionality will not be available."
    end
    if !(use_cuda[])
      @info """The GPU function is being called but the GPU is not accessible.
               Defaulting back to the CPU. (No action is required if you want to run on the CPU).""" maxlog=1
    end
  end
end
Zygote.@nograd check_use_cuda

@generated new_struct(DT, args...) = Expr(:call, nameof(DT), :(args...))
new_struct(t::T, args...) where {T<:Tuple} = ntuple(i->args[i], length(args))

isleaf(t::T) where {T}  = issingletontype(T), !isstructtype(T), false
isleaf(t::AbstractArray{T}) where {T} = false, isprimitivetype(T), true

adapt(x) = CUDA.cu(x)
adapt(x::Zygote.FillArrays.AbstractFill) = CUDA.cu(collect(x))
adapt(x::Zygote.OneElement) = CUDA.cu(collect(x))

cpu(x::AbstractArray) = Array(x)

for processor in (:gpu, :cpu)
    @eval begin
        $(Symbol("array_$(processor)_obj"))(objs::AbstractArray{T}) where {T} = collect([$(processor)(objs[i]) for i in 1 : length(objs)])
    end

    @eval begin
        function $(processor)(obj::T) where {T}
            is_singleton, is_leaf, is_array = isleaf(obj)
            is_leaf && return $((processor==:gpu) ? :adapt : :cpu)(obj)
            if is_array
                return $(Symbol("array_$(processor)_obj"))(obj)
            elseif is_singleton
                return obj
            else
                fields = fieldnames(T)
                return new_struct(obj, [$(processor)(getfield(obj, f)) for f in fields]...)
            end
        end
    end
end

"""
    gpu(model)
Transform the model so that it can be trained on the GPU. When called in an environment without a GPU, it does nothing and returns the original model.
!!! note
    This function is included in the HorseML module and can only be used with `using HorseML`.

# Example
```jldoctest gpu
julia> model = NetWork(Dense(10=>5, relu), Dense(5=>1, tanh)) |> gpu
Layer1 : Dense(IO:10 => 5, σ:relu)
Layer2 : Dense(IO:5 => 1, σ:tanh)

julia> model[1].w |> typeof
CUDA.CuArray{Float32, 2, CUDA.Mem.DeviceBuffer}
```
"""
function gpu(model::NetWork)
    check_use_cuda()
    if !use_cuda[]
        @warn "your computer doesn't have a GPU, or couldn't recognized. So a GPU isn't used."
        return model
    end
    N = NetWork()
    n = length(model.net)
    for i in 1 : n
        add_layer!(N, gpu(model[i]))
    end
    return N
end

"""
    cpu(model)
Put the model trained on the GPU back on the CPU.
!!! note
    This function is included in the HorseML module and can only be used with `using HorseML`.

# Example
```jldoctest
julia> model_on_cpu = model |> cpu #This model is made with description of gpu function
Layer1 : Dense(IO:10 => 5, σ:relu)
Layer2 : Dense(IO:5 => 1, σ:tanh)

julia> model_on_cpu[1].w |> typeof
Matrix{Float32} (alias for Array{Float32, 2})
```
"""
function cpu(model::NetWork)
    N = NetWork()
    n = length(model.net)
    for i in 1 : n
        add_layer!(N, cpu(model[i]))
    end
    return N
end


"""
    NodeType

Each node in the directed graph can be classified into the following types
- VARIABLE: Denotes a decision variable.
- PARAMETER: An adjustable parameter value (not a decision variable).
- CONSTANT: A constant value
- EXPRESSION: Any other expression that isn't a subexpression
- SUBEXPRESSION: Any expression referencing a different graph representation.
"""
@enum(NodeClass, CONSTANT, PARAMETER, VARIABLE, EXPRESSION, SUBEXPRESSION)
#=
- SELECT: These nodes do not store values rather they reference the output
          stored in the buffer assocatied with an EXPRESSION. These nodes
          only occur after a multiple output function. In the mimo JuMP
          extension they correspond to intermediate variables introduced
          by the user.
=#

struct Variable end
struct Parameter end
struct Constant end
struct Expression end
struct Subexpression end
struct User end

function user end
function usern end

const ATOM_TYPE_INSTANCES = instances(AtomType)

abstract type AbstractNode end

"""
Describes connectivity and expression represented by node.
"""
struct Node <: AbstractNode
    node_class::NodeClass
    ex_type::AtomType
    first_index::Int
    second_index::Int
    arity::Int
    children::Vector{Int}
end

for (t, s, a) in ((Variable, VARIABLE, VAR_ATOM),
                  (Parameter, PARAMETER, PARAM_ATOM),
                  (Constant, CONSTANT, CONST_ATOM),
                  (Subexpression, SUBEXPRESSION, SUBEXPR),)
    @eval Node(::$t, i::Int) = Node($s, $a, i, 0, 0, Int[])
end

for v in (PLUS, MINUS, MULT, POW, DIV, MAX, MIN)
    @eval Node(::Val{true}, ::Val{$v}, c::Vector{Int}) = Node(EXPRESSION, $v, 0, 0, length(c), c)
end
for d in ALL_ATOM_TYPES
    @eval Node(::Val{false}, ::Val{$d}, c::Vector{Int}) = Node(EXPRESSION, $d, 0, 0, 1, c)
end
Node(::Val{true}, ::Val{USER}, i::Int, c::Vector{Int}) = Node(EXPRESSION, USER, i, 0, 1, c)
Node(::Val{true}, ::Val{USERN}, i::Int, c::Vector{Int}) = Node(EXPRESSION, USERN, i, 0, length(c), c)

Node(::Val{:first_index}, n::Node, i::Int) = Node(node_class(n), ex_type(n), i, second_index(n), arity(n), children(n))

node_class(n::Node)   = n.node_class
ex_type(n::Node)      = n.ex_type
first_index(n::Node)  = n.first_index
second_index(n::Node) = n.node_second_index
arity(n::Node)        = n.arity
children(n::Node)     = n.children
child(n::Node, i)     = @inbounds getindex(n.children, i)

node_is_class(::Variable, n::Node) = node_class(n) == VARIABLE
node_is_class(::Parameter, n::Node) = node_class(n) == PARAMETER
node_is_class(::Constant, n::Node) = node_class(n) == CONSTANT

mv_eago_not_jump = setdiff(JuMP._Derivatives.operators,
                           union(Symbol[k for k in keys(REV_BIVARIATE_ATOM_DICT)],
                                 Symbol[k for k in keys(REV_NARITY_ATOM_DICT)]))
eago_mv_switch = quote end
for s in mv_eago_not_jump
    global eago_mv_switch = quote
        $eago_mv_switch
        (d == $s) && (return Node(Val(true), Val($s), v[c]))
    end
end
@eval function _create_call_node(i, v, c::UnitRange{Int}, op::OperatorRegistry)
    if i == 1
        return Node(Val(true), Val(PLUS), v[c])
    elseif i == 2
        return Node(Val(true), Val(MINUS), v[c])
    elseif i == 3
        return Node(Val(true), Val(MULT), v[c])
    elseif i == 4
        return Node(Val(true), Val(POW), v[c])
    elseif i == 5
        return Node(Val(true), Val(DIV), v[c])
    elseif i == 6
        error("If-else currently unsupported...")
    elseif i == 7
        return Node(Val(true), Val(MAX), v[c])
    elseif i == 8
        return Node(Val(true), Val(MIN), v[c])
    elseif i >= JuMP._Derivatives.USER_OPERATOR_ID_START
        i_mv = i - JuMP._Derivatives.USER_OPERATOR_ID_START + 1
        d = op.multivariate_id[i_mv]
        $eago_mv_switch
        return Node(Val(true), Val(USERN), i_mv, v[c])
    end
end

function binary_switch_typ(ids, exprs)
    if length(exprs) <= 3
        out = Expr(:if, Expr(:call, :(==), :i, ids[1]),
                   :(Node(Val(false), Val($(exprs[1])), v[c])))
        if length(exprs) > 1
            push!(out.args, binary_switch_typ(ids[2:end], exprs[2:end]))
        end
        return out
    else
        mid = length(exprs) >>> 1
        return Expr(:if, Expr(:call, :(<=), :i, ids[mid]),
                         binary_switch_typ(ids[1:mid], exprs[1:mid]),
                         binary_switch_typ(ids[mid+1:end], exprs[mid+1:end]))
    end
end

indx_JuMP = Int[]
indx_EAGO = AtomType[]
for k in univariate_operators
    if haskey(REV_UNIVARIATE_ATOM_DICT, k)
        k_EAGO = REV_UNIVARIATE_ATOM_DICT[k]
        push!(indx_JuMP, univariate_operator_to_id[k])
        push!(indx_EAGO, k_EAGO)
    end
end

uni_eago_not_jump = setdiff(univariate_operators, Symbol[k for k in keys(REV_UNIVARIATE_ATOM_DICT)])
uni_eago_not_jump = push!(uni_eago_not_jump, :-)
eago_uni_switch = quote end
for s in uni_eago_not_jump
    global eago_uni_switch = quote
        $eago_uni_switch
        (d == $s) && (return Node(Val(false), Val($s), v[c]))
    end
end
atom_switch = binary_switch_typ(indx_JuMP, indx_EAGO)
@eval function _create_call_node_uni(i::Int, v, c::UnitRange{Int}, op::OperatorRegistry)

    if i >= JuMP._Derivatives.USER_UNIVAR_OPERATOR_ID_START
        j = i - JuMP._Derivatives.USER_UNIVAR_OPERATOR_ID_START + 1
        dop = op.univariate_operator_id[j]
        d = op.univariate_operator_to_id[dop]
        $eago_uni_switch
        return Node(Val(true), Val(USER), j, v[c])
    end
    $atom_switch
end


# -----------------------------------------------------------
# GridLayout
# -----------------------------------------------------------

"Simple grid, indices are row-major."
immutable GridLayout <: SubplotLayout
    nr::Int
    nc::Int
end

Base.length(layout::GridLayout) = layout.nr * layout.nc
Base.start(layout::GridLayout) = 1
Base.done(layout::GridLayout, state) = state > length(layout)
function Base.next(layout::GridLayout, state)
  r = div(state-1, layout.nc) + 1
  c = mod1(state, layout.nc)
  (r,c), state + 1
end

nrows(layout::GridLayout) = layout.nr
ncols(layout::GridLayout) = layout.nc
ncols(layout::GridLayout, row::Int) = layout.nc

# get the plot index given row and column
Base.getindex(layout::GridLayout, r::Int, c::Int) = (r-1) * layout.nc + c

# -----------------------------------------------------------
# RowsLayout
# -----------------------------------------------------------

"Number of plots per row"
immutable RowsLayout <: SubplotLayout
    numplts::Int
    rowcounts::AbstractVector{Int}
end

Base.length(layout::RowsLayout) = layout.numplts
Base.start(layout::RowsLayout) = 1
Base.done(layout::RowsLayout, state) = state > length(layout)
function Base.next(layout::RowsLayout, state)
  r = 1
  c = 0
  for i = 1:state
    c += 1
    if c > layout.rowcounts[r]
      r += 1
      c = 1
    end
  end
  (r,c), state + 1
end

nrows(layout::RowsLayout) = length(layout.rowcounts)
ncols(layout::RowsLayout, row::Int) = row < 1 ? 0 : (row > nrows(layout) ? 0 : layout.rowcounts[row])

# get the plot index given row and column
Base.getindex(layout::RowsLayout, r::Int, c::Int) = sum(layout.rowcounts[1:r-1]) + c

# -----------------------------------------------------------
# FlexLayout
# -----------------------------------------------------------

"Flexible, nested layout with optional size percentages."
immutable FlexLayout <: SubplotLayout
    n::Int
    grid::Matrix # Nested layouts. Each position
                 # can be a plot index or another FlexLayout
    widths::Vector{Float64}
    heights::Vector{Float64}
end

typealias IntOrFlex Union{Int,FlexLayout}

Base.length(layout::FlexLayout) = layout.n
Base.start(layout::FlexLayout) = 1
Base.done(layout::FlexLayout, state) = state > length(layout)
function Base.next(layout::FlexLayout, state)
    # TODO: change this method to return more info
    # TODO: might consider multiple iterator types.. some backends might have an easier time row-by-row for example
    error()
    r = 1
    c = 0
    for i = 1:state
        c += 1
        if c > layout.rowcounts[r]
            r += 1
            c = 1
        end
    end
    (r,c), state + 1
end

nrows(layout::FlexLayout)           = size(layout.grid, 1)
ncols(layout::FlexLayout, row::Int) = size(layout.grid, 2)

# get the plot index given row and column
Base.getindex(layout::FlexLayout, r::Int, c::Int) = layout.grid[r,c]

# -----------------------------------------------------------

# we're taking in a nested structure of some kind... parse it out and build a FlexLayout
function subplotlayout(mat::AbstractVecOrMat; widths = nothing, heights = nothing)
    n = 0
    nr, nc = size(mat)
    grid = Array(IntOrFlex, nr, nc)
    for i=1:nr, j=1:nc
        v = mat[i,j]

        if isa(v, Integer)
            grid[i,j] = Int(v)
            n += 1

        elseif isa(v, Tuple)
            warn("need to handle tuples somehow... (idx, sizepct)")
            grid[i,j] = nothing

        elseif v == nothing
            grid[i,j] = nothing

        elseif isa(v, AbstractVecOrMat)
            grid[i,j] = layout(v)
            n += grid[i,j].n

        else
            error("How do we process? $v")
        end
    end

    if widths == nothing
        widths = ones(nc) ./ nc
    end
    if heights == nothing
        heights = ones(nr) ./ nr
    end

    FlexLayout(n, grid, widths, heights)
end

function subplotlayout(sz::Tuple{Int,Int})
  GridLayout(sz...)
end

function subplotlayout(rowcounts::AVec{Int})
  RowsLayout(sum(rowcounts), rowcounts)
end

function subplotlayout(numplts::Int, nr::Int, nc::Int)

  # figure out how many rows/columns we need
  if nr == -1
    if nc == -1
      nr = round(Int, sqrt(numplts))
      nc = ceil(Int, numplts / nr)
    else
      nr = ceil(Int, numplts / nc)
    end
  else
    nc = ceil(Int, numplts / nr)
  end

  # if it's a perfect rectangle, just create a grid
  if numplts == nr * nc
    return GridLayout(nr, nc)
  end

  # create the rowcounts vector
  i = 0
  rowcounts = Int[]
  for r in 1:nr
    cnt = min(nc, numplts - i)
    push!(rowcounts, cnt)
    i += cnt
  end

  RowsLayout(numplts, rowcounts)
end


using ForwardDiff: value

"""
    AbstractObsScheme

Types inheriting from abstract type `AbstractObsScheme` define the scheme
according to which a stochastic process has been observed
"""
abstract type AbstractObsScheme end

"""
    PartObs <: AbstractObsScheme

Type acting as a flag for partially observed diffusions
"""
struct PartObs <: AbstractObsScheme end

"""
    FPT <: AbstractObsScheme

Observation scheme in which only first passage times are observed
"""
struct FPT <: AbstractObsScheme end

"""
    ParamUpdateType

Types inheriting from abstract type `ParamUpdateType` define the way in which
parameters are to be updated by the MCMC sampler
"""
abstract type ParamUpdateType end

"""
    ConjugateUpdt <: ParamUpdateType

Type acting as a flag for update from full conditional (conjugate to a prior)
"""
struct ConjugateUpdt <: ParamUpdateType end

"""
    MetropolisHastingsUpdt <: ParamUpdateType

Flag for performing update according to Metropolis Hastings step
"""
struct MetropolisHastingsUpdt <: ParamUpdateType end

"""
    setBlocking(𝔅::NoBlocking, ::Any, ::Any, ::Any, ::Any)

No blocking is to be done, do nothing
"""
setBlocking(𝔅::NoBlocking, ::Any, ::Any, ::Any, ::Any) = 𝔅

"""
    setBlocking(::ChequeredBlocking, blockingParams, P, WW, XX)

Blocking pattern is chosen to be a chequerboard.
"""
function setBlocking(::ChequeredBlocking, blockingParams, P, WW, XX)
    ChequeredBlocking(blockingParams..., P, WW, XX)
end

"""
    FPTInfo{S,T}

The struct
```
struct FPTInfo{S,T}
    condCoord::NTuple{N,S}
    upCrossing::NTuple{N,Bool}
    autoRenewed::NTuple{N,Bool}
    reset::NTuple{N,T}
end
```
serves as a container for the information regarding first passage time
observations. `condCoord` is an NTuple of coordinates that are conditioned on
the first passage time nature of the observations. `upCrossing` indicates
whether observations of the corresponding coordinate are up-crossings or
down-crossings. `autoRenewed` indicates whether process starts from the
renewed state (i.e. normally the process is unconstrained until it hits level
`reset` for the first time, however `autoRenewed` process is constrained on the
first passage time from the very beginnig). `reset` level is the level that
needs to be reached before the process starts to be conditioned on the first
passage time.
"""
struct FPTInfo{S,T,N}
    condCoord::NTuple{N,S}
    upCrossing::NTuple{N,Bool}
    autoRenewed::NTuple{N,Bool}
    reset::NTuple{N,T}

    FPTInfo(condCoord::NTuple{N,S}, upCrossing::NTuple{N,Bool},
            reset::NTuple{N,T},
            autoRenewed::NTuple{N,Bool} = Tuple(fill(false,length(condCoord)))
            ) where {S,T,N} = new{S,T,N}(condCoord, upCrossing,
                                         autoRenewed, reset)
end

"""
    checkSingleCoordFpt(XXᵒ, c, cidx, fpt)

Verify whether coordinate `c` (with index number `cidx`) of path `XXᵒ`.yy
adheres to the first passage time observation scheme specified by the object
`fpt`.
"""
function checkSingleCoordFpt(XXᵒ, c, cidx, fpt)
    k = length(XXᵒ.yy)
    thrsd = XXᵒ.yy[end][c]
    renewed = fpt.autoRenewed[cidx]
    s = fpt.upCrossing[cidx] ? 1 : -1
    for i in 1:k
        if !renewed && (s*XXᵒ.yy[i][c] <= s*fpt.reset[cidx])
            renewed = true
        elseif renewed && (s*XXᵒ.yy[i][c] > s*thrsd)
            return false
        end
    end
    return renewed
end

"""
    checkFpt(::PartObs, XXᵒ, fpt)

First passage time constrains are automatically satisfied for the partially
observed scheme
"""
checkFpt(::PartObs, XXᵒ, fpt) = true

"""
    checkFpt(::FPT, XXᵒ, fpt)

Verify whether path `XXᵒ`.yy adheres to the first passage time observation
scheme specified by the object `fpt`.
"""
function checkFpt(::FPT, XXᵒ, fpt)
    for (cidx, c) in enumerate(fpt.condCoord)
        if !checkSingleCoordFpt(XXᵒ, c, cidx, fpt)
            return false
        end
    end
    return true
end

"""
    checkFullPathFpt(::PartObs, ::Any, ::Any, ::Any)

First passage time constrains are automatically satisfied for the partially
observed scheme
"""
checkFullPathFpt(::PartObs, ::Any, ::Any, ::Any) = true

"""
    checkFullPathFpt(::PartObs, XXᵒ, m, fpt)

Verify whether all paths in the range `iRange`, i.e. `XXᵒ`[i].yy, i in `iRange`
adhere to the first passage time observation scheme specified by the object
`fpt`
"""
function checkFullPathFpt(::FPT, XXᵒ, iRange, fpt)
    for i in iRange
        if !checkFpt(FPT(), XXᵒ[i], fpt[i])
            return false
        end
    end
    return true
end

"""
    findProposalLaw(xx, tt, P˟, P̃, Ls, Σs; dt=1/5000, timeChange=true,
                    solver::ST=Ralston3())

Initialise the object with proposal law and all the necessary containers needed
for the simulation of the guided proposals
"""
function findProposalLaw(::Type{K}, xx, tt, P˟, P̃, Ls, Σs, τ; dt=1/5000,
                         solver::ST=Ralston3(),
                         changePt::ODEChangePt=NoChangePt()) where {K,ST}
    m = length(xx) - 1
    P = Array{ContinuousTimeProcess,1}(undef,m)
    for i in m:-1:1
        numPts = Int64(ceil((tt[i+1]-tt[i])/dt))+1
        t = τ(tt[i], tt[i+1]).( range(tt[i], stop=tt[i+1], length=numPts) )
        P[i] = ( (i==m) ? GuidPropBridge(K, t, P˟, P̃[i], Ls[i], xx[i+1], Σs[i];
                                         changePt=changePt, solver=ST()) :
                          GuidPropBridge(K, t, P˟, P̃[i], Ls[i], xx[i+1], Σs[i],
                                         P[i+1].H[1], P[i+1].Hν[1], P[i+1].c[1];
                                         changePt=changePt, solver=ST()) )
    end
    P
end

"""
    initialise(::ObsScheme, P, m, y::StartingPtPrior{T}, ::S, fpt)

Initialise the workspace for MCMC algorithm. Initialises containers for driving
Wiener processes `WWᵒ` & `WW`, for diffusion processes `XXᵒ` & `XX`, for
diffusion Law `Pᵒ` (parametetrised by proposal parameters) and defines the type
of Wiener process `Wnr`.
"""
function initialise(::ObsScheme, P, m, yPr::StartingPtPrior{T}, ::S,
                    fpt) where {ObsScheme <: AbstractObsScheme,T,S}
    y = startPt(yPr)
    Pᵒ = deepcopy(P)
    TW = typeof(sample([0], Wiener{S}()))
    TX = typeof(SamplePath([], zeros(T, 0)))
    XXᵒ = Vector{TX}(undef,m)
    WWᵒ = Vector{TW}(undef,m)
    Wnr = Wiener{S}()
    for i in 1:m
        WWᵒ[i] = Bridge.samplepath(P[i].tt, zero(S))
        sample!(WWᵒ[i], Wnr)
        XXᵒ[i] = solve(Euler(), y, WWᵒ[i], P[i])
        while !checkFpt(ObsScheme(), XXᵒ[i], fpt[i])
            sample!(WWᵒ[i], Wnr)
            solve!(Euler(), XXᵒ[i], y, WWᵒ[i], P[i])
        end
        y = XXᵒ[i].yy[end]
    end
    y = startPt(yPr)
    ll = logpdf(yPr, y)
    ll += pathLogLikhd(ObsScheme(), XXᵒ, P, 1:m, fpt, skipFPT=true)
    ll += lobslikelihood(P[1], y)

    XX = deepcopy(XXᵒ)
    WW = deepcopy(WWᵒ)
    # needed for proper initialisation of the Crank-Nicolson scheme
    yPr = invStartPt(y, yPr, P[1])

    Wnr, WWᵒ, WW, XXᵒ, XX, Pᵒ, ll, yPr
end

"""
    savePath!(Paths, XX, saveMe, skip)

If `saveMe` flag is true, then save the entire path spanning all segments in
`XX`. Only 1 in  every `skip` points is saved to reduce storage space.
"""
function savePath!(Paths, XX, saveMe, skip)
    if saveMe
        push!(Paths, collect(Iterators.flatten(XX[i].yy[1:skip:end-1]
                                               for i in 1:length(XX))))
    end
end

"""
    acceptSample(logThreshold, verbose=false)

Make a random MCMC decision for whether to accept a sample or reject it.
"""
function acceptSample(logThreshold, verbose=false)
    if rand(Exponential(1.0)) > -logThreshold # Reject if NaN
        verbose && print("\t ✓\n")
        return true
    else
        verbose && print("\t .\n")
        return false
    end
end

"""
    solveBackRec!(P, solver::ST=Ralston3()) where ST

Solve backward recursion to find H, Hν, c and Q, which together define r̃(t,x)
and p̃(x, 𝓓) under the auxiliary law, when no blocking is done
"""
function solveBackRec!(::NoBlocking, P, solver::ST=Ralston3()) where ST
    m = length(P)
    gpupdate!(P[m]; solver=ST())
    for i in (m-1):-1:1
        gpupdate!(P[i], P[i+1].H[1], P[i+1].Hν[1], P[i+1].c[1]; solver=ST())
    end
end

"""
    solveBackRec!(P, solver::ST=Ralston3()) where ST

Solve backward recursion to find H, Hν, c and Q, which together define r̃(t,x)
and p̃(x, 𝓓) under the auxiliary law, when blocking is done
"""
function solveBackRec!(𝔅::BlockingSchedule, P, solver::ST=Ralston3()) where ST
    for block in reverse(𝔅.blocks[𝔅.idx])
        gpupdate!(P[block[end]]; solver=ST())
        for i in reverse(block[1:end-1])
            gpupdate!(P[i], P[i+1].H[1], P[i+1].Hν[1], P[i+1].c[1]; solver=ST())
        end
    end
end

"""
    proposalStartPt(::BlockingSchedule, ::Val{1}, ::Any, yPr, P, ρ)

Set a new starting point for the proposal path when sampling the first block in
a blocking scheme.

...
# Arguments
- `::BlockingSchedule`: indicator that a blocking scheme is used
- `::Val{1}`: indicator that it's the first block, so starting point needs updating
- `yPr`: prior over the starting point
- `P`: diffusion law
- `ρ`: memory parameter in the Crank-Nicolson scheme
...
"""
function proposalStartPt(::BlockingSchedule, ::Val{1}, ::Any, yPr, P, ρ)
    proposalStartPt(NoBlocking(), nothing, nothing, yPr, P, ρ)
end

"""
    proposalStartPt(::BlockingSchedule, ::Any, y₀, yPr, ::Any, ::Any)

Default behaviour of dealing with a starting point in the blocking scheme is
to do nothing
"""
function proposalStartPt(::BlockingSchedule, ::Any, y₀, yPr, ::Any, ::Any)
    y₀, yPr
end

"""
    proposalStartPt(::NoBlocking, ::Any, y₀, yPr, P, ρ)

Set a new starting point for the proposal path when no blocking is done
...
# Arguments
- `::NoBlocking`: indicator that no blocking is done
- `yPr`: prior over the starting point
- `P`: diffusion law
- `ρ`: memory parameter in the Crank-Nicolson scheme
...
"""
function proposalStartPt(::NoBlocking, ::Any, ::Any, yPr, P, ρ)
    yPrᵒ = rand(yPr, ρ)
    y = startPt(yPrᵒ, P)
    y, yPrᵒ
end

"""
    printInfo(verbose::Bool, it::Integer, ll, llᵒ, msg="update")

Print information to the console about current likelihood values

...
# Arguments
- `verbose`: flag for whether to print anything at all
- `it`: iteration of the Markov chain
- `ll`: likelihood of the previous, accepted sample
- `llᵒ`: likelihood of the proposal sample
- `msg`: message to start with
...
"""
function printInfo(verbose::Bool, it::Integer, ll, llᵒ, msg="update")
    verbose && print(msg, ": ", it, " ll ", round(ll, digits=3), " ",
                     round(llᵒ, digits=3), " diff_ll: ", round(llᵒ-ll,digits=3))
end

"""
    pathLogLikhd(::ObsScheme, XX, P, iRange, fpt; skipFPT=false)

Compute likelihood for path `XX` to be observed under `P`. Only segments with
index numbers in `iRange` are considered. `fpt` contains relevant info about
checks regarding adherence to first passage time pattern. `skipFPT` if set to
`true` can skip the step of checking adherence to fpt pattern (used for
conjugate updates, or any updates that keep `XX` unchanged)
"""
function pathLogLikhd(::ObsScheme, XX, P, iRange, fpt; skipFPT=false
                      ) where ObsScheme <: AbstractObsScheme
    ll = 0.0
    for i in iRange
        ll += llikelihood(LeftRule(), XX[i], P[i])
    end
    !skipFPT && (ll = checkFullPathFpt(ObsScheme(), XX, iRange, fpt) ? ll : -Inf)
    ll
end

"""
    swap!(A, Aᵒ, iRange)

Swap contents between containers A & Aᵒ in the index range iRange
"""
function swap!(A, Aᵒ, iRange)
    for i in iRange
        A[i], Aᵒ[i] = Aᵒ[i], A[i]
    end
end

"""
    swap!(A, Aᵒ, B, Bᵒ, iRange)

Swap contents between containers A & Aᵒ in the index range iRange, do the same
for containers B & Bᵒ
"""
function swap!(A, Aᵒ, B, Bᵒ, iRange)
    swap!(A, Aᵒ, iRange)
    swap!(B, Bᵒ, iRange)
end

"""
    crankNicolson!(yᵒ, y, ρ)

Preconditioned Crank Nicolson update with memory parameter `ρ`, previous vector
`y` and new vector `yᵒ`
"""
crankNicolson!(yᵒ, y, ρ) = (yᵒ .= √(1-ρ)*yᵒ + √(ρ)*y)

"""
    sampleSegment!(i, Wnr, WW, WWᵒ, P, y, XX, ρ)

Sample `i`th path segment using preconditioned Crank-Nicolson scheme
...
# Arguments
- `i`: index of the segment to be sampled
- `Wnr`: type of the Wiener process
- `WW`: containers with old Wiener paths
- `WWᵒ`: containers where proposal Wiener paths will be stored
- `P`: laws of the diffusion to be sampled
- `y`: starting point of the segment
- `XX`: containers for proposal diffusion path
- `ρ`: memory parameter for the Crank-Nicolson scheme
...
"""
function sampleSegment!(i, Wnr, WW, WWᵒ, P, y, XX, ρ)
    sample!(WWᵒ[i], Wnr)
    crankNicolson!(WWᵒ[i].yy, WW[i].yy, ρ)
    solve!(Euler(), XX[i], y, WWᵒ[i], P[i])
    XX[i].yy[end]
end

"""
    sampleSegments!(iRange, Wnr, WW, WWᵒ, P, y, XX, ρ)

Sample paths segments in index range `iRange` using preconditioned
Crank-Nicolson scheme
...
# Arguments
- `iRange`: range of indices of the segments that need to be sampled
- `Wnr`: type of the Wiener process
- `WW`: containers with old Wiener paths
- `WWᵒ`: containers where proposal Wiener paths will be stored
- `P`: laws of the diffusion to be sampled
- `y`: starting point of the segment
- `XX`: containers for proposal diffusion path
- `ρ`: memory parameter for the Crank-Nicolson scheme
...
"""
function sampleSegments!(iRange, Wnr, WW, WWᵒ, P, y, XX, ρ)
    for i in iRange
        y = sampleSegment!(i, Wnr, WW, WWᵒ, P, y, XX, ρ)
    end
end

"""
    impute!(::ObsScheme, 𝔅::NoBlocking, Wnr, yPr, WWᵒ, WW, XXᵒ, XX, P, ll, fpt;
            ρ=0.0, verbose=false, it=NaN, headStart=false) where
            ObsScheme <: AbstractObsScheme -> acceptedLogLikhd, acceptDecision

Imputation step of the MCMC scheme (without blocking).
...
# Arguments
- `::ObsScheme`: observation scheme---first-passage time or partial observations
- `Wnr`: type of the Wiener process
- `yPr`: prior over the starting point of the diffusion path
- `WWᵒ`: containers for proposal Wiener paths
- `WW`: containers with old Wiener paths
- `XXᵒ`: containers for proposal diffusion paths
- `XX`: containers with old diffusion paths
- `P`: laws of the diffusion path (proposal and target)
- `11`: log-likelihood of the old (previously accepted) diffusion path
- `fpt`: info about first-passage time conditioning
- `ρ`: memory parameter for the Crank-Nicolson scheme
- `verbose`: whether to print updates info while sampling
- `it`: iteration index of the MCMC algorithm
- `headStart`: flag for whether to 'ease into' fpt conditions
...
"""
function impute!(::ObsScheme, 𝔅::NoBlocking, Wnr, yPr, WWᵒ, WW, XXᵒ, XX, P, ll,
                 fpt; ρ=0.0, verbose=false, it=NaN, headStart=false,
                 solver::ST=Ralston3()) where
                 {ObsScheme <: AbstractObsScheme, ST}
    # sample proposal starting point
    yᵒ, yPrᵒ = proposalStartPt(𝔅, nothing, nothing, yPr, P[1], ρ)

    # sample proposal path
    m = length(WWᵒ)
    yᵗᵉᵐᵖ = copy(yᵒ)
    for i in 1:m
        sampleSegment!(i, Wnr, WW, WWᵒ, P, yᵗᵉᵐᵖ, XXᵒ, ρ)
        if headStart
            while !checkFpt(ObsScheme(), XXᵒ[i], fpt[i])
                sampleSegment!(i, Wnr, WW, WWᵒ, P, yᵗᵉᵐᵖ, XXᵒ, ρ)
            end
        end
        yᵗᵉᵐᵖ = XXᵒ[i].yy[end]
    end

    llᵒ = logpdf(yPrᵒ, yᵒ)
    llᵒ += pathLogLikhd(ObsScheme(), XXᵒ, P, 1:m, fpt)
    llᵒ += lobslikelihood(P[1], yᵒ)

    printInfo(verbose, it, value(ll), value(llᵒ), "impute")

    if acceptSample(llᵒ-ll, verbose)
        swap!(XX, XXᵒ, WW, WWᵒ, 1:m)
        return llᵒ, true, 𝔅, yPrᵒ
    else
        return ll, false, 𝔅, yPr
    end
end

#NOTE deprecated
"""
    swapXX!(𝔅::ChequeredBlocking, XX)

Swap containers between `XX` and `𝔅.XX`
"""
function swapXX!(𝔅::BlockingSchedule, XX)
    for block in 𝔅.blocks[𝔅.idx]
        swap!(XX, 𝔅.XX, block)
    end
end

#NOTE deprecated
"""
    swapXX!(𝔅::NoBlocking, XX)

nothing to do
"""
swapXX!(𝔅::NoBlocking, XX) = nothing

"""
    noiseFromPath!(𝔅::BlockingSchedule, XX, WW, P)

Compute driving Wiener noise `WW` from path `XX` drawn under law `P`
"""
function noiseFromPath!(𝔅::BlockingSchedule, XX, WW, P)
    for block in 𝔅.blocks[𝔅.idx]
        for i in block
            invSolve!(Euler(), XX[i], WW[i], P[i])
        end
    end
end

"""
    startPtLogPdf(::Val{1}, yPr::StartingPtPrior, y)

Compute the log-likelihood contribution of the starting point for a given prior
under a blocking scheme (intended to be used with a first block only)
"""
startPtLogPdf(::Val{1}, yPr::StartingPtPrior, y) = logpdf(yPr, y)

"""
    startPtLogPdf(::Any, yPr::StartingPtPrior, y)

Default contribution to log-likelihood from the startin point under blocking
"""
startPtLogPdf(::Any, yPr::StartingPtPrior, y) = 0.0

#NOTE deprecated
"""
    impute!(::ObsScheme, 𝔅::ChequeredBlocking, Wnr, y, WWᵒ, WW, XXᵒ, XX, P, ll,
            fpt; ρ=0.0, verbose=false, it=NaN, headStart=false) where
            ObsScheme <: AbstractObsScheme -> acceptedLogLikhd, acceptDecision

Imputation step of the MCMC scheme (without blocking).
...
# Arguments
- `::ObsScheme`: observation scheme---first-passage time or partial observations
- `𝔅`: object with relevant information about blocking
- `Wnr`: type of the Wiener process
- `yPr`: prior over the starting point of the diffusion path
- `WWᵒ`: containers for proposal Wiener paths
- `WW`: containers with old Wiener paths
- `XXᵒ`: containers for proposal diffusion paths
- `XX`: containers with old diffusion paths
- `P`: laws of the diffusion path (proposal and target)
- `11`: log-likelihood of the old (previously accepted) diffusion path
- `fpt`: info about first-passage time conditioning
- `ρ`: memory parameter for the Crank-Nicolson scheme
- `verbose`: whether to print updates info while sampling
- `it`: iteration index of the MCMC algorithm
- `headStart`: flag for whether to 'ease into' fpt conditions
...
"""
function impute!_deprecated(::ObsScheme, 𝔅::ChequeredBlocking, Wnr, yPr, WWᵒ, WW, XXᵒ, XX,
                 P, ll, fpt; ρ=0.0, verbose=false, it=NaN, headStart=false,
                 solver::ST=Ralston3()) where
                 {ObsScheme <: AbstractObsScheme, ST}
    θ = params(P[1].Target)             # current parameter
    𝔅 = next(𝔅, XX, θ)
    solveBackRec!(𝔅, 𝔅.P, ST())         # compute (H, Hν, c) for given blocks

    swapXX!(𝔅, XX)                      # move current path to object 𝔅
    noiseFromPath!(𝔅, 𝔅.XX, 𝔅.WW, 𝔅.P) # find noise WW that generates XX under 𝔅

    # compute white noise generating starting point under 𝔅
    yPr𝔅 = invStartPt(𝔅.XX[1].yy[1], yPr, 𝔅.P[1])

    for (blockIdx, block) in enumerate(𝔅.blocks[𝔅.idx])
        blockFlag = Val{block[1]}()
        y = 𝔅.XX[block[1]].yy[1]       # current starting point

        # set the starting point for the block
        yᵒ, yPrᵒ = proposalStartPt(𝔅, blockFlag, y, yPr𝔅, 𝔅.P[block[1]], ρ)

        # sample path in block
        sampleSegments!(block, Wnr, 𝔅.WW, 𝔅.WWᵒ, 𝔅.P, yᵒ, 𝔅.XXᵒ, ρ)
        setEndPtManually!(𝔅, blockIdx, block)

        # loglikelihoods
        llᵒ = startPtLogPdf(blockFlag, yPrᵒ, yᵒ)
        llᵒ += pathLogLikhd(ObsScheme(), 𝔅.XXᵒ, 𝔅.P, block, fpt)
        llᵒ += lobslikelihood(𝔅.P[block[1]], yᵒ)

        llPrev = startPtLogPdf(blockFlag, yPr𝔅, y)
        llPrev += pathLogLikhd(ObsScheme(), 𝔅.XX, 𝔅.P, block, fpt; skipFPT=true)
        llPrev += lobslikelihood(𝔅.P[block[1]], y)

        printInfo(verbose, it, value(llPrev), value(llᵒ), "impute")
        if acceptSample(llᵒ-llPrev, verbose)
            swap!(𝔅.XX, 𝔅.XXᵒ, block)
            registerAccpt!(𝔅, blockIdx, true)
            yPr𝔅 = yPrᵒ # can do something non-trivial only for the first block
        else
            registerAccpt!(𝔅, blockIdx, false)
        end
    end
    swapXX!(𝔅, XX) # move accepted path from object 𝔅 to general container XX
    noiseFromPath!(𝔅, XX, WW, P) # compute noise WW that generated XX under law P
    # compute white noise generating starting point under P
    y = XX[1].yy[1]
    yPr = invStartPt(y, yPr𝔅, P[1])

    ll = logpdf(yPr, y) # starting point contribution
    ll += pathLogLikhd(ObsScheme(), XX, P, 1:length(P), fpt; skipFPT=true)
    ll += lobslikelihood(P[1], y)

    # acceptance indicator does not matter for sampling with blocking
    return ll, true, 𝔅, yPr
end

"""
    impute!(::ObsScheme, 𝔅::ChequeredBlocking, Wnr, y, WWᵒ, WW, XXᵒ, XX, P, ll,
            fpt; ρ=0.0, verbose=false, it=NaN, headStart=false) where
            ObsScheme <: AbstractObsScheme -> acceptedLogLikhd, acceptDecision

Imputation step of the MCMC scheme (without blocking).
...
# Arguments
- `::ObsScheme`: observation scheme---first-passage time or partial observations
- `𝔅`: object with relevant information about blocking
- `Wnr`: type of the Wiener process
- `yPr`: prior over the starting point of the diffusion path
- `WWᵒ`: containers for proposal Wiener paths
- `WW`: containers with old Wiener paths
- `XXᵒ`: containers for proposal diffusion paths
- `XX`: containers with old diffusion paths
- `P`: laws of the diffusion path (proposal and target)
- `11`: log-likelihood of the old (previously accepted) diffusion path
- `fpt`: info about first-passage time conditioning
- `ρ`: memory parameter for the Crank-Nicolson scheme
- `verbose`: whether to print updates info while sampling
- `it`: iteration index of the MCMC algorithm
- `headStart`: flag for whether to 'ease into' fpt conditions
...
"""
function impute!(::ObsScheme, 𝔅::ChequeredBlocking, Wnr, yPr, WWᵒ, WW, XXᵒ, XX,
                 P, ll, fpt; ρ=0.0, verbose=false, it=NaN, headStart=false,
                 solver::ST=Ralston3()
                 ) where {ObsScheme <: AbstractObsScheme, ST}
    θ = params(𝔅.P[1].Target)             # current parameter
    𝔅 = next(𝔅, 𝔅.XX, θ)
    solveBackRec!(𝔅, 𝔅.P, ST())         # compute (H, Hν, c) for given blocks
    noiseFromPath!(𝔅, 𝔅.XX, 𝔅.WW, 𝔅.P) # find noise WW that generates XX under 𝔅.P

    # compute white noise generating starting point under 𝔅
    yPr = invStartPt(𝔅.XX[1].yy[1], yPr, 𝔅.P[1])

    ll_total = 0.0
    for (blockIdx, block) in enumerate(𝔅.blocks[𝔅.idx])
        blockFlag = Val{block[1]}()
        y = 𝔅.XX[block[1]].yy[1]       # accepted starting point

        # proposal starting point for the block (can be non-y only for the first block)
        yᵒ, yPrᵒ = proposalStartPt(𝔅, blockFlag, y, yPr, 𝔅.P[block[1]], ρ)

        # sample path in block
        sampleSegments!(block, Wnr, 𝔅.WW, 𝔅.WWᵒ, 𝔅.P , yᵒ, 𝔅.XXᵒ, ρ)
        setEndPtManually!(𝔅, blockIdx, block)

        # starting point, path and observations contribution
        llᵒ = startPtLogPdf(blockFlag, yPrᵒ, yᵒ)
        llᵒ += pathLogLikhd(ObsScheme(), 𝔅.XXᵒ, 𝔅.P, block, fpt)
        llᵒ += lobslikelihood(𝔅.P[block[1]], yᵒ)

        llPrev = startPtLogPdf(blockFlag, yPr, y)
        llPrev += pathLogLikhd(ObsScheme(), 𝔅.XX, 𝔅.P, block, fpt; skipFPT=true)
        llPrev += lobslikelihood(𝔅.P[block[1]], y)

        printInfo(verbose, it, value(llPrev), value(llᵒ), "impute")
        if acceptSample(llᵒ-llPrev, verbose)
            swap!(𝔅.XX, 𝔅.XXᵒ, block)
            registerAccpt!(𝔅, blockIdx, true)
            yPr = yPrᵒ # can do something non-trivial only for the first block
            ll_total += llᵒ
        else
            registerAccpt!(𝔅, blockIdx, false)
            ll_total += llPrev
        end
    end
    # acceptance indicator does not matter for sampling with blocking
    return ll_total, true, 𝔅, yPr
end

"""
    updateLaws!(Ps, θᵒ)

Set new parameter `θᵒ` for the laws in vector `Ps`
"""
function updateLaws!(Ps, θᵒ)
    m = length(Ps)
    for i in 1:m
        Ps[i] = GuidPropBridge(Ps[i], θᵒ)
    end
end

"""
    updateTargetLaws!(𝔅::NoBlocking, θᵒ)

Nothing to do
"""
updateTargetLaws!(𝔅::NoBlocking, θᵒ) = nothing

"""
    updateTargetLaws!(𝔅::BlockingSchedule, θᵒ)

Set new parameter `θᵒ` for the target laws in blocking object `𝔅`
"""
function updateTargetLaws!(𝔅::BlockingSchedule, θᵒ)
    for block in 𝔅.blocks[𝔅.idx]
        for i in block
            𝔅.P[i] = GuidPropBridge(𝔅.P[i], θᵒ)
        end
    end
end

"""
    updateProposalLaws!(𝔅::BlockingSchedule, θᵒ)

Set new parameter `θᵒ` for the proposal laws inside blocking object `𝔅`
"""
function updateProposalLaws!(𝔅::BlockingSchedule, θᵒ)
    for block in 𝔅.blocks[𝔅.idx]
        for i in block
            𝔅.Pᵒ[i] = GuidPropBridge(𝔅.Pᵒ[i], θᵒ)
        end
    end
end

"""
    findPathFromWiener!(XX, y, WW, P, iRange)

Find path `XX` (that starts from `y`) that is generated under law `P` from the
Wiener process `WW`. Only segments with indices in range `iRange` are considered
"""
function findPathFromWiener!(XX, y, WW, P, iRange)
    for i in iRange
        solve!(Euler(), XX[i], y, WW[i], P[i])
        y = XX[i].yy[end]
    end
end

"""
    priorKernelContrib(tKern, priors, θ, θᵒ)

Contribution to the log-likelihood ratio from transition kernel `tKernel` and
`priors`.
"""
function priorKernelContrib(tKern, priors, θ, θᵒ)
    llr = logpdf(tKern, θᵒ, θ) - logpdf(tKern, θ, θᵒ)
    for prior in priors
        llr += logpdf(prior, θᵒ) - logpdf(prior, θ)
    end
    llr
end

"""
    setEndPtManually!(𝔅::BlockingSchedule, blockIdx, block)

Manually set the end-point of the proposal path under blocking so that it agrees
with the end-point of the previously accepted path. If it is the last block,
then do nothing
"""
function setEndPtManually!(𝔅::BlockingSchedule, blockIdx, block)
    if blockIdx < length(𝔅.blocks[𝔅.idx])
        𝔅.XXᵒ[block[end]].yy[end] = 𝔅.XX[block[end]].yy[end]
    end
end

"""
    updateParam!(::ObsScheme, ::MetropolisHastingsUpdt, tKern, θ, ::UpdtIdx,
                 yPr, WW, Pᵒ, P, XXᵒ, XX, ll, prior, fpt, recomputeODEs;
                 solver::ST=Ralston3(), verbose=false,
                 it=NaN) where {ObsScheme <: AbstractObsScheme, ST, UpdtIdx}
                 -> acceptedLogLikhd, acceptDecision
Update parameters
...
# Arguments
- `::ObsScheme`: observation scheme---first-passage time or partial observations
- `::MetropolisHastingsUpdt()`: type of the parameter update
- `tKern`: transition kernel
- `θ`: current value of the parameter
- `updtIdx`: object declaring indices of the updated parameter
- `yPr`: prior over the starting point of the diffusion path
- `WW`: containers with Wiener paths
- `Pᵒ`: container for the laws of the diffusion path with new parametrisation
- `P`: laws of the diffusion path with old parametrisation
- `XXᵒ`: containers for proposal diffusion paths
- `XX`: containers with old diffusion paths
- `11`: likelihood of the old (previously accepted) parametrisation
- `priors`: list of priors
- `fpt`: info about first-passage time conditioning
- `recomputeODEs`: whether auxiliary law depends on the updated params
- `verbose`: whether to print updates info while sampling
- `it`: iteration index of the MCMC algorithm
...
"""
function updateParam!(::ObsScheme, ::MetropolisHastingsUpdt, 𝔅::NoBlocking,
                      tKern, θ, ::UpdtIdx, yPr, WW, Pᵒ, P, XXᵒ, XX, ll, priors,
                      fpt, recomputeODEs; solver::ST=Ralston3(), verbose=false,
                      it=NaN) where {ObsScheme <: AbstractObsScheme, ST, UpdtIdx}
    m = length(WW)
    θᵒ = rand(tKern, θ, UpdtIdx())               # sample new parameter
    updateLaws!(Pᵒ, θᵒ)
    recomputeODEs && solveBackRec!(NoBlocking(), Pᵒ, ST()) # compute (H, Hν, c)

    # find white noise which for a given θᵒ gives a correct starting point
    y = XX[1].yy[1]
    yPrᵒ = invStartPt(y, yPr, Pᵒ[1])

    findPathFromWiener!(XXᵒ, y, WW, Pᵒ, 1:m)

    llᵒ = logpdf(yPrᵒ, y)
    llᵒ += pathLogLikhd(ObsScheme(), XXᵒ, Pᵒ, 1:m, fpt)
    llᵒ += lobslikelihood(Pᵒ[1], y)

    printInfo(verbose, it, ll, llᵒ)

    llr = ( llᵒ - ll + priorKernelContrib(tKern, priors, θ, θᵒ))

    # Accept / reject
    if acceptSample(llr, verbose)
        swap!(XX, XXᵒ, P, Pᵒ, 1:m)
        return llᵒ, true, θᵒ, yPrᵒ
    else
        return ll, false, θ, yPr
    end
end

"""
    updateParam!(::ObsScheme, ::MetropolisHastingsUpdt, tKern, θ, ::UpdtIdx,
                 yPr, WW, Pᵒ, P, XXᵒ, XX, ll, prior, fpt, recomputeODEs;
                 solver::ST=Ralston3(), verbose=false,
                 it=NaN) where {ObsScheme <: AbstractObsScheme, ST, UpdtIdx}
                 -> acceptedLogLikhd, acceptDecision
Update parameters
...
# Arguments
- `::ObsScheme`: observation scheme---first-passage time or partial observations
- `::MetropolisHastingsUpdt()`: type of the parameter update
- `tKern`: transition kernel
- `θ`: current value of the parameter
- `updtIdx`: object declaring indices of the updated parameter
- `yPr`: prior over the starting point of the diffusion path
- `WW`: containers with Wiener paths
- `Pᵒ`: container for the laws of the diffusion path with new parametrisation
- `P`: laws of the diffusion path with old parametrisation
- `XXᵒ`: containers for proposal diffusion paths
- `XX`: containers with old diffusion paths
- `11`: likelihood of the old (previously accepted) parametrisation
- `priors`: list of priors
- `fpt`: info about first-passage time conditioning
- `recomputeODEs`: whether auxiliary law depends on the updated params
- `verbose`: whether to print updates info while sampling
- `it`: iteration index of the MCMC algorithm
...
"""
function updateParam!(::ObsScheme, ::MetropolisHastingsUpdt,
                      𝔅::ChequeredBlocking, tKern, θ, ::UpdtIdx,
                      yPr, WW, Pᵒ, P, XXᵒ, XX, ll, priors, fpt, recomputeODEs;
                      solver::ST=Ralston3(), verbose=false,
                      it=NaN) where {ObsScheme <: AbstractObsScheme, ST, UpdtIdx}
    m = length(WW)
    θᵒ = rand(tKern, θ, UpdtIdx())               # sample new parameter
    updateProposalLaws!(𝔅, θᵒ)                   # update law `Pᵒ` accordingly
    solveBackRec!(𝔅, 𝔅.Pᵒ, ST())                 # compute (H, Hν, c)

    llᵒ = logpdf(yPr, 𝔅.XX[1].yy[1])
    for (blockIdx, block) in enumerate(𝔅.blocks[𝔅.idx])
        y = 𝔅.XX[block[1]].yy[1]
        findPathFromWiener!(𝔅.XXᵒ, y, 𝔅.WW, 𝔅.Pᵒ, block)
        setEndPtManually!(𝔅, blockIdx, block)

        # Compute log-likelihood ratio
        llᵒ += pathLogLikhd(ObsScheme(), 𝔅.XXᵒ, 𝔅.Pᵒ, block, fpt)
        llᵒ += lobslikelihood(𝔅.Pᵒ[block[1]], y)
    end
    printInfo(verbose, it, ll, llᵒ)

    llr = ( llᵒ - ll + priorKernelContrib(tKern, priors, θ, θᵒ))

    # Accept / reject
    if acceptSample(llr, verbose)
        swap!(𝔅.XX, 𝔅.XXᵒ, 𝔅.P, 𝔅.Pᵒ, 1:m)
        return llᵒ, true, θᵒ, yPr
    else
        return ll, false, θ, yPr
    end
end

fetchTargetLaw(𝔅::NoBlocking, P) = P[1].Target

fetchTargetLaw(𝔅::BlockingSchedule, P) = 𝔅.P[1].Target

"""
    updateParam!(::PartObs, ::ConjugateUpdt, tKern, θ, ::UpdtIdx, yPr, WW, Pᵒ,
                 P, XXᵒ, XX, ll, priors, fpt, recomputeODEs;
                 solver::ST=Ralston3(), verbose=false, it=NaN
                 ) -> acceptedLogLikhd, acceptDecision
Update parameters
see the definition of  updateParam!(…, ::MetropolisHastingsUpdt, …) for the
explanation of the arguments.
"""
function updateParam!(::ObsScheme, ::ConjugateUpdt, 𝔅::NoBlocking,
                      tKern, θ, ::UpdtIdx, yPr, WW, Pᵒ, P, XXᵒ, XX, ll, priors,
                      fpt, recomputeODEs; solver::ST=Ralston3(), verbose=false,
                      it=NaN) where {ObsScheme <: AbstractObsScheme, ST, UpdtIdx}
    m = length(P)
    ϑ = conjugateDraw(θ, XX, P[1].Target, priors[1], UpdtIdx())   # sample new parameter
    θᵒ = moveToProperPlace(ϑ, θ, UpdtIdx())     # align so that dimensions agree

    updateLaws!(P, θᵒ)
    recomputeODEs && solveBackRec!(NoBlocking(), P, ST()) # compute (H, Hν, c)

    for i in 1:m    # compute wiener path WW that generates XX
        invSolve!(Euler(), XX[i], WW[i], P[i])
    end
    # compute white noise that generates starting point
    y = XX[1].yy[1]
    yPr = invStartPt(y, yPr, P[1])

    llᵒ = logpdf(yPr, y)
    llᵒ += pathLogLikhd(ObsScheme(), XX, P, 1:m, fpt; skipFPT=true)
    llᵒ += lobslikelihood(P[1], y)
    printInfo(verbose, it, value(ll), value(llᵒ))
    return llᵒ, true, θᵒ, yPr
end

"""
    updateParam!(::PartObs, ::ConjugateUpdt, tKern, θ, ::UpdtIdx, yPr, WW, Pᵒ,
                 P, XXᵒ, XX, ll, priors, fpt, recomputeODEs;
                 solver::ST=Ralston3(), verbose=false, it=NaN
                 ) -> acceptedLogLikhd, acceptDecision
Update parameters
see the definition of  updateParam!(…, ::MetropolisHastingsUpdt, …) for the
explanation of the arguments.
"""
function updateParam!(::ObsScheme, ::ConjugateUpdt, 𝔅::BlockingSchedule,
                      tKern, θ, ::UpdtIdx, yPr, WW, Pᵒ, P, XXᵒ, XX, ll, priors,
                      fpt, recomputeODEs; solver::ST=Ralston3(), verbose=false,
                      it=NaN) where {ObsScheme <: AbstractObsScheme, ST, UpdtIdx}
    m = length(P)
    ϑ = conjugateDraw(θ, 𝔅.XX, 𝔅.P[1].Target, priors[1], UpdtIdx())   # sample new parameter
    θᵒ = moveToProperPlace(ϑ, θ, UpdtIdx())     # align so that dimensions agree

    updateTargetLaws!(𝔅, θᵒ)
    recomputeODEs && solveBackRec!(𝔅, 𝔅.P, ST())
    for i in 1:m    # compute wiener path WW that generates XX
        invSolve!(Euler(), 𝔅.XX[i], 𝔅.WW[i], 𝔅.P[i])
    end
    # compute white noise that generates starting point
    y = 𝔅.XX[1].yy[1]
    yPr = invStartPt(y, yPr, 𝔅.P[1])
    llᵒ = logpdf(yPr, y)
    for block in 𝔅.blocks[𝔅.idx]
        llᵒ += pathLogLikhd(ObsScheme(), 𝔅.XX, 𝔅.P, block, fpt; skipFPT=true)
        llᵒ += lobslikelihood(𝔅.P[block[1]], 𝔅.XX[block[1]].yy[1])
    end
    printInfo(verbose, it, value(ll), value(llᵒ))
    return llᵒ, true, θᵒ, yPr
end

"""
    mcmc(::ObsScheme, obs, obsTimes, yPr::StartingPtPrior, w, P˟, P̃, Ls, Σs,
         numSteps, tKernel, priors; fpt=fill(NaN, length(obsTimes)-1), ρ=0.0,
         dt=1/5000, timeChange=true, saveIter=NaN, verbIter=NaN,
         updtCoord=(Val((true,)),), paramUpdt=true, skipForSave=1,
         updtType=(MetropolisHastingsUpdt(),), solver::ST=Ralston3(), warmUp=0)

Gibbs sampler alternately imputing unobserved parts of the path and updating
unknown coordinates of the parameter vector (the latter only if paramUpdt==true)
...
# Arguments
- `::ObsScheme`: observation scheme---first-passage time or partial observations
- `obs`: vector with observations
- `obsTimes`: times of the observations
- `yPr`: prior over the starting point of the diffusion path
- `w`: dummy variable whose type must agree with the type of the Wiener process
- `P˟`: law of the target diffusion (with initial θ₀ set)
- `P̃`: law of the auxiliary process (with initial θ₀ set)
- `Ls`: vector of observation operators (one per each observation)
- `Σs`: vector of covariance matrices of the noise (one per each observaiton)
- `numSteps`: number of mcmc iterations
- `tKernel`: transition kernel (also with initial θ₀ set)
- `priors`: a list of lists of priors
- `τ`: time-change transformation
- `fpt`: info about first-passage time conditioning
- `ρ`: memory parameter for the Crank-Nicolson scheme
- `dt`: time-distance for the path imputation
- `saveIter`: save path `XX` once every `saveIter` many iterations
- `verbIter`: print out progress info once every `verbIter` many iterations
- `updtCoord`: list of objects declaring indices of to-be-updated parameters
- `paramUpdt`: flag for whether to update parameters at all
- `skipForSave`: when saving paths, save only one in every `skipForSave` points
- `updtType`: list of types of updates to cycle through
- `solver`: numerical solver used for computing backward ODEs
- `warmUp`: number of steps for which no parameter update is to be made
...
"""
function mcmc(::Type{K}, ::ObsScheme, obs, obsTimes, yPr::StartingPtPrior, w,
              P˟, P̃, Ls, Σs, numSteps, tKernel, priors, τ;
              fpt=fill(NaN, length(obsTimes)-1), ρ=0.0, dt=1/5000, saveIter=NaN,
              verbIter=NaN, updtCoord=(Val((true,)),), paramUpdt=true,
              skipForSave=1, updtType=(MetropolisHastingsUpdt(),),
              blocking::Blocking=NoBlocking(),
              blockingParams=([], 0.1, NoChangePt()),
              solver::ST=Ralston3(), changePt::CP=NoChangePt(), warmUp=0
              ) where {K, ObsScheme <: AbstractObsScheme, ST, Blocking, CP}
    P = findProposalLaw( K, obs, obsTimes, P˟, P̃, Ls, Σs, τ; dt=dt, solver=ST(),
                         changePt=CP(getChangePt(blockingParams[3])) )
    m = length(obs)-1
    updtLen = length(updtCoord)
    Wnr, WWᵒ, WW, XXᵒ, XX, Pᵒ, ll, yPr = initialise(ObsScheme(), P, m, yPr, w,
                                                    fpt)
    Paths = []
    accImpCounter = 0
    accUpdtCounter = [0 for i in 1:updtLen]
    θ = params(P˟)
    θchain = Vector{typeof(θ)}(undef, (numSteps-warmUp)*updtLen+1)
    θchain[1] = copy(θ)
    recomputeODEs = [any([e in dependsOnParams(P[1].Pt) for e
                         in idx(uc)]) for uc in updtCoord]

    updtStepCounter = 1
    𝔅 = setBlocking(blocking, blockingParams, P, WW, XX)
    display(𝔅)
    for i in 1:numSteps
        verbose = (i % verbIter == 0)
        i > warmUp && savePath!(Paths, blocking == NoBlocking() ? XX : 𝔅.XX,
                                (i % saveIter == 0), skipForSave)
        ll, acc, 𝔅, yPr = impute!(ObsScheme(), 𝔅, Wnr, yPr, WWᵒ, WW, XXᵒ, XX,
                                  P, ll, fpt, ρ=ρ, verbose=verbose, it=i,
                                  solver=ST())
        accImpCounter += 1*acc
        if paramUpdt && i > warmUp
            for j in 1:updtLen
                (ll, acc, θ,
                 yPr) = updateParam!(ObsScheme(), updtType[j], 𝔅, tKernel, θ,
                                     updtCoord[j], yPr, WW, Pᵒ, P, XXᵒ, XX, ll,
                                     priors[j], fpt, recomputeODEs[j];
                                     solver=ST(), verbose=verbose, it=i)
                accUpdtCounter[j] += 1*acc
                updtStepCounter += 1
                θchain[updtStepCounter] = copy(θ)
                verbose && print("\n")
            end
            verbose && print("------------------------------------------------",
                             "------\n")
        end
    end
    displayAcceptanceRate(𝔅)
    Time = collect(Iterators.flatten(p.tt[1:skipForSave:end-1] for p in P))
    θchain, accImpCounter/numSteps, accUpdtCounter./numSteps, Paths, Time
end


#using Queryverse
using DataFrames
using DataFramesMeta
using Dates
using DelimitedFiles
using JuMP
using Logging
using CSV
using Query
using Statistics

const FILE_ACCESS_OVER = "w+";
const FILE_ACCESS_APPEND = "a+";
#include("import_post_data.jl")

DF_Gen_Sch = CSV.read(".//inputs//UnitSchedule_(DEC-DEP 2019).csv",
                  dateformat="mm-dd-yyyy THH:MM:SS.000Z", header=true, copycols=true, DataFrame);

DF_Gens = CSV.read(".//inputs//data_generators.csv", header=true, DataFrame);
DF_Peakers = CSV.read(".//inputs//data_peakers.csv", header=true, DataFrame);

select!(DF_Gens, Not(:Cogen));

#=
cycles_units_header    = ["Section", "Time", "Note1", "Note2", "Note3", "Note4"]
open(".//outputs//postproc_gens_cycle_units.csv", FILE_ACCESS_OVER) do io
    writedlm(io, permutedims(cycles_units_header), ',')
end; # closes file
=#

function calc_gens_cyclesbyID(df_results::DataFrame, df_gen_sch::DataFrame,
    df_gens_all::DataFrame, ini_date::String, end_date::String)
    #Selects a subset of schedules that fall into the months that users specify.
    # @where could be extended to include other month using the OR operation (||)
    ini_date = DateTime(ini_date, "mm/dd/yyyy HH:MM:SS");
    end_date = DateTime(end_date, "mm/dd/yyyy HH:MM:SS");

    # add UNIT_ID
    insertcols!(df_gen_sch, 4, :UNIT_ID => df_gen_sch.UNIT_NAME .* "_" .* string.(df_gen_sch.CC_KEY));
    #df_gen_sch = df_gen_sch

    df_gens_sch_drange = @from i in df_gen_sch begin
                @where DateTime(i.BEGIN_DATE) >= ini_date && DateTime(i.BEGIN_DATE) <= end_date && i.EDITION_NAME=="Actual"
                @select {i.PORTFOLIO_NAME, i.EDITION_NAME, i.UNIT_NAME, i.CC_KEY, i.UNIT_ID, i.UNIT_TYPE, i.BEGIN_DATE, i.MW}
                @collect DataFrame
           end

    df_convgen_sch = @from i in df_gens_sch_drange begin
                @where  i.UNIT_TYPE=="Steam" || i.UNIT_TYPE=="Steam Dual Boiler" || i.UNIT_TYPE=="Peaker" || i.UNIT_TYPE=="Nuclear" || i.UNIT_TYPE=="Combined-Cycle"
                @select {i.PORTFOLIO_NAME, i.EDITION_NAME, i.UNIT_NAME, i.CC_KEY, i.UNIT_ID, i.UNIT_TYPE, i.BEGIN_DATE, i.MW}
                @collect DataFrame
           end
    insertcols!(df_convgen_sch, size(df_convgen_sch,2), :On_Off => Int64.(df_convgen_sch.MW .> 0));
    insertcols!(df_convgen_sch, size(df_convgen_sch,2), :ShutDown => zeros(Int64));
    insertcols!(df_convgen_sch, size(df_convgen_sch,2), :StartUp => zeros(Int64));

    for unit in eachrow(df_gens_all)
        #unit_name = unit.UNIT_NAME;
        unit_id = unit.UNIT_ID;
        df_unit_sch = @where(df_convgen_sch, (:UNIT_ID .== unit_id))
        insertcols!(df_unit_sch, 1, :UNIT => unit.UNIT);
        #df_unit_sch = @where(df_convgen_sch, (:UNIT_NAME .== "ALLE_UN01"))

        sort!(df_unit_sch, :BEGIN_DATE)

        was_on = zero(Int64);
        for row in eachrow(df_unit_sch)
            if rownumber(row) == 1  #First row, save initial ON/OFF value
                was_on = row.On_Off;
                row.StartUp = row.On_Off;
            elseif ((row.On_Off == 1)  & (was_on == 0))
                was_on = 1;
                row.StartUp = 1;
            elseif ((row.On_Off == 0)  & (was_on == 1))
                was_on = 0;
                row.ShutDown = 1;
            end;
        end;

        if isempty(df_results)
            df_results = copy(df_unit_sch);
        else
            df_results = vcat(df_results, df_unit_sch);
        end;
        #println("Number of cycles for unit:", df_unit_sch.UNIT_NAME, "is: ", n_cycles)
    end; # for loop all units in dataframe df_gens_all

    return df_results;
end; # function

#Combines the slow generators and peakers specs into one dataframe
DF_Gens_All = vcat(DF_Gens, DF_Peakers);

# Filtering the Actual generation outcomes from UNIT_Schedules.csv
# Reformats the date-time data given in column BEGIN_DATE
DF_Gen_Sch.BEGIN_DATE = DateTime.(DF_Gen_Sch.BEGIN_DATE, "mm/dd/yyyy HH:MM:SS p");

Start_date = "01/01/2019 00:00:00"
Final_date = "01/02/2019 02:00:00"
DF_Results = DataFrame();

DF_Results = calc_gens_cycles(DF_Results, DF_Gen_Sch, DF_Gens_All, Start_date, Final_date);

Start_date = replace(Start_date, "/" => "-")
Final_date = replace(Final_date, "/" => "-")
Start_date = Start_date[1:findfirst(isequal(' '), Start_date)-1]
Final_date = Final_date[1:findfirst(isequal(' '), Final_date)-1]

CSV.write(".//outputs//postprocess//postproc_gens_cycles_ByID_$(Start_date)_$(Final_date).csv", DF_Results)

# Get Summary of results by unit and save them
#Reference: https://julia-data-query.readthedocs.io/en/latest/dplyr.html
DF_Results_ByID = groupby(DF_Results, [:UNIT_ID, :UNIT_TYPE]);

DF_ResultsID_Summary = @combine(DF_Results_ByID,
  Sum_On_Off = sum(:On_Off),
  Avg_On_Off = mean(:On_Off),
  Sum_StartUp = sum(:StartUp),
  Sum_ShutDown = sum(:ShutDown))

CSV.write(".//outputs//postprocess//postproc_gens_cycles_SUMMARYbyID_$(Start_date)_$(Final_date).csv", DF_ResultsID_Summary)

#=TODO:
Emissions calculations for 358 days
Shutdown is 1, one hour after the unit has been shutoff.
Compare the number of shutdowns and startup.
startup if at time 0 the units are on.
=#

#include(".//postprocessing//calc_input_gens_cyclesbyID.jl")


# Thu 08 Nov 2012 05:07:44 PM EST
#
# NeHe Tut 6 - Rotate a textured cube
#
# Q - quit

# load necessary GLUT/OpenGL routines

global OpenGLver="1.0"
using OpenGL
using GLUT

### auxiliary functions

function cube(size)
  glBegin(GL_QUADS)
    # Front Face
    glTexCoord(0.0, 0.0)
    glVertex(-size, -size, size)
    glTexCoord(1.0, 0.0)
    glVertex(size, -size, size)
    glTexCoord(1.0, 1.0)
    glVertex(size, size, size)
    glTexCoord(0.0, 1.0)
    glVertex(-size, size, size)

    # Back Face
    glTexCoord(1.0, 0.0)
    glVertex(-size, -size, -size)
    glTexCoord(1.0, 1.0)
    glVertex(-size, size, -size)
    glTexCoord(0.0, 1.0)
    glVertex(size, size, -size)
    glTexCoord(0.0, 0.0)
    glVertex(size, -size, -size)

    # Top Face
    glTexCoord(0.0, 1.0)
    glVertex(-size, size, -size)
    glTexCoord(0.0, 0.0)
    glVertex(-size, size, size)
    glTexCoord(1.0, 0.0)
    glVertex(size, size, size)
    glTexCoord(1.0, 1.0)
    glVertex(size, size, -size)

    # Bottom Face
    glTexCoord(1.0, 1.0)
    glVertex(-size, -size, -size)
    glTexCoord(0.0, 1.0)
    glVertex(size, -size, -size)
    glTexCoord(0.0, 0.0)
    glVertex(size, -size, size)
    glTexCoord(1.0, 0.0)
    glVertex(-size, -size, size)

    # Right Face
    glTexCoord(1.0, 0.0)
    glVertex(size, -size, -size)
    glTexCoord(1.0, 1.0)
    glVertex(size, size, -size)
    glTexCoord(0.0, 1.0)
    glVertex(size, size, size)
    glTexCoord(0.0, 0.0)
    glVertex(size, -size, size)

    # Left Face
    glTexCoord(0.0, 0.0)
    glVertex(-size, -size, -size)
    glTexCoord(1.0, 0.0)
    glVertex(-size, -size, size)
    glTexCoord(1.0, 1.0)
    glVertex(-size, size, size)
    glTexCoord(0.0, 1.0)
    glVertex(-size, size, -size)
  glEnd()
end

### end of auxiliary functions

# initialize variables

global window

global xrot      = 0.0
global yrot      = 0.0
global zrot      = 0.0

global tex       = Array(Uint32,1) # generating 1 texture

global cube_size = 1.0

width            = 640
height           = 480

# load textures from images

function LoadGLTextures()
    global tex

    img, w, h = glimread(expanduser("~/.julia/GLUT/Examples/NeHe/tut6/NeHe.bmp"))

    glGenTextures(1,tex)
    glBindTexture(GL_TEXTURE_2D,tex[1])
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
    glTexImage2D(GL_TEXTURE_2D, 0, 3, w, h, 0, GL_RGB, GL_UNSIGNED_BYTE, img)
end

# function to init OpenGL context

function initGL(w::Integer,h::Integer)
    glViewport(0,0,w,h)
    LoadGLTextures()
    glClearColor(0.0, 0.0, 0.0, 0.0)
    glClearDepth(1.0)
    glDepthFunc(GL_LESS)
    glEnable(GL_DEPTH_TEST)
    glShadeModel(GL_SMOOTH)

    # enable texture mapping
    glEnable(GL_TEXTURE_2D)

    glMatrixMode(GL_PROJECTION)
    glLoadIdentity()

    gluPerspective(45.0,w/h,0.1,100.0)

    glMatrixMode(GL_MODELVIEW)
end

# prepare Julia equivalents of C callbacks that are typically used in GLUT code

function ReSizeGLScene(w::Int32,h::Int32)
    if h == 0
        h = 1
    end

    glViewport(0,0,w,h)

    glMatrixMode(GL_PROJECTION)
    glLoadIdentity()

    gluPerspective(45.0,w/h,0.1,100.0)

    glMatrixMode(GL_MODELVIEW)

    return nothing
end

_ReSizeGLScene = cfunction(ReSizeGLScene, Void, (Int32, Int32))

function DrawGLScene()
    global tex
    global xrot
    global yrot
    global zrot

    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glLoadIdentity()

    glTranslate(0.0, 0.0, -5.0)

    glRotate(xrot,1.0,0.0,0.0)
    glRotate(yrot,0.0,1.0,0.0)
    glRotate(zrot,0.0,0.0,1.0)

    glBindTexture(GL_TEXTURE_2D,tex[1])
    cube(cube_size)

    xrot +=0.2
    yrot +=0.3
    zrot +=0.4

    glutSwapBuffers()

    return nothing
end

_DrawGLScene = cfunction(DrawGLScene, Void, ())

function keyPressed(the_key::Char,x::Int32,y::Int32)
    if the_key == int('q')
        glutDestroyWindow(window)
    end

    return nothing # keyPressed returns "void" in C. this is a workaround for Julia's "automatically return the value of the last expression in a function" behavior.
end

_keyPressed = cfunction(keyPressed, Void, (Char, Int32, Int32))

# run GLUT routines

glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_ALPHA | GLUT_DEPTH)
glutInitWindowSize(width, height)
glutInitWindowPosition(0, 0)

window = glutCreateWindow("NeHe Tut 6")

glutDisplayFunc(_DrawGLScene)
glutFullScreen()

glutIdleFunc(_DrawGLScene)
glutReshapeFunc(_ReSizeGLScene)
glutKeyboardFunc(_keyPressed)

initGL(width, height)

glutMainLoop()


"""
@axpy
@ascal
@dot

BLAS Level 1 functions.
"""

macro axpy(a, x, y)
    expr = quote
        let u = $(esc(a))
            for i in eachindex($(esc(x)))
                @inbounds $(esc(y))[i] += u * $(esc(x))[i]
            end
        end
    end
    expr
end

macro scal(a, x)
    expr = quote
        let u = $(esc(a))
            for i in eachindex($(esc(x)))
                @inbounds $(esc(x))[i] *= u
            end
        end
    end
    expr
end

macro dot(x, y)
    expr = quote
        s = 0
        for i in eachindex($(esc(x)))
            @inbounds s += $(esc(x))[i] * $(esc(y))[i]
        end
        s
    end
    expr
end

"""
itime(t)

Get interval time from a given cumulative time vector t.
The first element is t[1]

Retuen value:
dt: interval time vector
maxt: maximum interval time
"""

function itime(t::AbstractVector{Tv}) where Tv
    dt = similar(t)
    prev = Tv(0)
    maxt = Tv(0)
    @inbounds for i = eachindex(t)
        dt[i] = t[i] - prev
        prev = t[i]
        if dt[i] > maxt
            maxt = dt[i]
        end
    end
    return dt, maxt
end


### A Pluto.jl notebook ###
# v0.17.3

using Markdown
using InteractiveUtils

# ╔═╡ 0069ec00-f42b-40bb-a82e-c91ec78583e4
begin
	import Pkg
	Pkg.develop("PoreMatMod")
end

# ╔═╡ 37939a7a-0651-11ec-11c1-6b5ef0a19ec2
# load required packages (Pluto.jl will automatically install them)
using PoreMatMod, PlutoUI

# ╔═╡ 2889e30d-442d-4620-b5f6-0216e21c623b
using PoreMatMod.ExampleHelpers

# ╔═╡ 8d523993-6e85-443a-9949-12030552b457
md"""
## Example: append missing hydrogen atoms to linkers of a MOF
"""

# ╔═╡ ab9b556e-5082-48fa-a7ca-b37ef895d703
input_file_message()

# ╔═╡ 9a9bd9fd-5fea-4c01-a87a-5a5a0d332c1a
xtal_folder_message()

# ╔═╡ fcebd06c-9607-42a5-ba1b-e7683b0924ab
rc[:paths][:crystals]

# ╔═╡ af6f9a09-17ad-45dd-b1fd-50f9cc5a692e
moiety_folder_message()

# ╔═╡ 62439d74-601c-40fc-a488-c0838b9ada69
rc[:paths][:moieties]

# ╔═╡ 026100b5-0708-48bb-840d-931605524874
md"""
!!! example \"the task\"
	We have an IRMOF-1 crystal structure with hydrogen atoms missing on the linkers, presumably owing to artifacts of X-ray structure determination. We wish to append hydrogen atoms onto the missing positions on the linkers.

**Parent crystal structure**: first, we read in the `.cif` file describing the parent structure, which is not simulation-ready owing to the missing hydrogen atoms.
"""

# ╔═╡ 0433da26-4f59-424f-9603-875d904c0fd5
begin
	# read in the parent xtal
	parent = Crystal("IRMOF-1_noH.cif") # load .cif file
	infer_bonds!(parent, true)          # infer bonds
	view_structure(parent)              # view structure
end

# ╔═╡ 4ad53cf5-d063-4bb6-ae20-1b6cc29902c8
fragment_construction_note()

# ╔═╡ 64b95411-07da-44af-a06e-9e6676328ffd
md"""
**Query fragment**: next, we construct a query fragment to match the corrupted fragment in the IRMOF-1 parent structure.
"""

# ╔═╡ 83532414-6471-4002-b23c-1600243318d1
query = moiety("1,4-C-phenylene_noH.xyz");

# ╔═╡ fc9e8e21-02c0-43ca-980f-55496526d7f3
view_query_or_replacement("1,4-C-phenylene_noH.xyz")

# ╔═╡ f68335d7-b4e0-40b3-b10d-bf406ab42c1c
with_terminal() do
	display_query_or_replacement_file("1,4-C-phenylene_noH.xyz")
end

# ╔═╡ c53f896d-fa27-4290-aa6d-aa8c0c467f3b
md"""
**Replacement fragment**: then, we construct a replacement fragment as a corrected version of the query fragment (with hydrogen atoms appropriately appended).
"""

# ╔═╡ 0d8ac0fd-c7b1-4781-aaa2-33cc8c1c08ae
replacement = moiety("1,4-C-phenylene.xyz");

# ╔═╡ 836f6d08-c507-44c6-b927-c9ea240f40f8
view_query_or_replacement("1,4-C-phenylene.xyz")

# ╔═╡ b172c36b-80bf-4620-b2e4-5c39d719962e
with_terminal() do
	display_query_or_replacement_file("1,4-C-phenylene.xyz")
end

# ╔═╡ 5b71d14a-be80-4ac3-8983-62571d0d4e7d
md"""
**Find and replace**: finally, we search the parent MOF for the query fragment and effect the replacements. Voila; we have a simulation-ready IRMOF-1 structure. 🚀
"""

# ╔═╡ 74aa19d2-b1a4-4333-9ff9-e6ea74e7d989
begin
	child = replace(parent, query => replacement)
	view_structure(child)
end

# ╔═╡ 5869bf7a-958e-4e51-997a-18497e7deaba
write_cif_message()

# ╔═╡ dbf3a196-dd4d-4ac8-8396-42ac7c7e0ba1
write_cif(child, "simulation_ready_IRMOF-1.cif")

# ╔═╡ Cell order:
# ╟─0069ec00-f42b-40bb-a82e-c91ec78583e4
# ╟─8d523993-6e85-443a-9949-12030552b457
# ╠═37939a7a-0651-11ec-11c1-6b5ef0a19ec2
# ╠═2889e30d-442d-4620-b5f6-0216e21c623b
# ╟─ab9b556e-5082-48fa-a7ca-b37ef895d703
# ╟─9a9bd9fd-5fea-4c01-a87a-5a5a0d332c1a
# ╠═fcebd06c-9607-42a5-ba1b-e7683b0924ab
# ╟─af6f9a09-17ad-45dd-b1fd-50f9cc5a692e
# ╠═62439d74-601c-40fc-a488-c0838b9ada69
# ╟─026100b5-0708-48bb-840d-931605524874
# ╠═0433da26-4f59-424f-9603-875d904c0fd5
# ╟─4ad53cf5-d063-4bb6-ae20-1b6cc29902c8
# ╟─64b95411-07da-44af-a06e-9e6676328ffd
# ╠═83532414-6471-4002-b23c-1600243318d1
# ╟─fc9e8e21-02c0-43ca-980f-55496526d7f3
# ╟─f68335d7-b4e0-40b3-b10d-bf406ab42c1c
# ╟─c53f896d-fa27-4290-aa6d-aa8c0c467f3b
# ╠═0d8ac0fd-c7b1-4781-aaa2-33cc8c1c08ae
# ╟─836f6d08-c507-44c6-b927-c9ea240f40f8
# ╟─b172c36b-80bf-4620-b2e4-5c39d719962e
# ╟─5b71d14a-be80-4ac3-8983-62571d0d4e7d
# ╠═74aa19d2-b1a4-4333-9ff9-e6ea74e7d989
# ╟─5869bf7a-958e-4e51-997a-18497e7deaba
# ╠═dbf3a196-dd4d-4ac8-8396-42ac7c7e0ba1


using AlphaZero
using Test

include("../games/tictactoe/main.jl")
include("../games/connect-four/main.jl")
include("../scripts/lib/dummy_run.jl")

include("test_game.jl")

@testset "Testing Games" begin
  test_symmetries(Tictactoe.Game)
  test_symmetries(ConnectFour.Game)
end

@testset "Dummy Runs" begin
  @test dummy_run(Tictactoe)
  #@test dummy_run(ConnectFour) # Takes a bit too long for Travis
end