# This file is a part of Julia. License is MIT: https://julialang.org/license

# Matrix-matrix multiplication

AdjOrTransStridedMat{T} = Union{Adjoint{T, <:StridedMatrix}, Transpose{T, <:StridedMatrix}}
StridedMaybeAdjOrTransMat{T} = Union{StridedMatrix{T}, Adjoint{T, <:StridedMatrix}, Transpose{T, <:StridedMatrix}}

# matmul.jl: Everything to do with dense matrix multiplication

matprod(x, y) = x*y + x*y

# dot products

dot(x::Union{DenseArray{T},StridedVector{T}}, y::Union{DenseArray{T},StridedVector{T}}) where {T<:BlasReal} = BLAS.dot(x, y)
dot(x::Union{DenseArray{T},StridedVector{T}}, y::Union{DenseArray{T},StridedVector{T}}) where {T<:BlasComplex} = BLAS.dotc(x, y)

function dot(x::Vector{T}, rx::AbstractRange{TI}, y::Vector{T}, ry::AbstractRange{TI}) where {T<:BlasReal,TI<:Integer}
    if length(rx) != length(ry)
        throw(DimensionMismatch(lazy"length of rx, $(length(rx)), does not equal length of ry, $(length(ry))"))
    end
    if minimum(rx) < 1 || maximum(rx) > length(x)
        throw(BoundsError(x, rx))
    end
    if minimum(ry) < 1 || maximum(ry) > length(y)
        throw(BoundsError(y, ry))
    end
    GC.@preserve x y BLAS.dot(length(rx), pointer(x)+(first(rx)-1)*sizeof(T), step(rx), pointer(y)+(first(ry)-1)*sizeof(T), step(ry))
end

function dot(x::Vector{T}, rx::AbstractRange{TI}, y::Vector{T}, ry::AbstractRange{TI}) where {T<:BlasComplex,TI<:Integer}
    if length(rx) != length(ry)
        throw(DimensionMismatch(lazy"length of rx, $(length(rx)), does not equal length of ry, $(length(ry))"))
    end
    if minimum(rx) < 1 || maximum(rx) > length(x)
        throw(BoundsError(x, rx))
    end
    if minimum(ry) < 1 || maximum(ry) > length(y)
        throw(BoundsError(y, ry))
    end
    GC.@preserve x y BLAS.dotc(length(rx), pointer(x)+(first(rx)-1)*sizeof(T), step(rx), pointer(y)+(first(ry)-1)*sizeof(T), step(ry))
end

function *(transx::Transpose{<:Any,<:StridedVector{T}}, y::StridedVector{T}) where {T<:BlasComplex}
    x = transx.parent
    return BLAS.dotu(x, y)
end

# Matrix-vector multiplication
function (*)(A::StridedMatrix{T}, x::StridedVector{S}) where {T<:BlasFloat,S<:Real}
    TS = promote_op(matprod, T, S)
    y = isconcretetype(TS) ? convert(AbstractVector{TS}, x) : x
    mul!(similar(x, TS, size(A,1)), A, y)
end
function (*)(A::AbstractMatrix{T}, x::AbstractVector{S}) where {T,S}
    TS = promote_op(matprod, T, S)
    mul!(similar(x,TS,axes(A,1)),A,x)
end

# these will throw a DimensionMismatch unless B has 1 row (or 1 col for transposed case):
(*)(a::AbstractVector, tB::Transpose{<:Any,<:AbstractMatrix}) = reshape(a, length(a), 1) * tB
(*)(a::AbstractVector, adjB::Adjoint{<:Any,<:AbstractMatrix}) = reshape(a, length(a), 1) * adjB
(*)(a::AbstractVector, B::AbstractMatrix) = reshape(a, length(a), 1) * B

@inline mul!(y::StridedVector{T}, A::StridedVecOrMat{T}, x::StridedVector{T},
             alpha::Number, beta::Number) where {T<:BlasFloat} =
    gemv!(y, 'N', A, x, alpha, beta)

# Complex matrix times real vector. Reinterpret the matrix as a real matrix and do real matvec compuation.
for elty in (Float32, Float64)
    @eval begin
        @inline function mul!(y::StridedVector{Complex{$elty}}, A::StridedVecOrMat{Complex{$elty}}, x::StridedVector{$elty},
                alpha::Real, beta::Real)
            Afl = reinterpret($elty, A)
            yfl = reinterpret($elty, y)
            mul!(yfl, Afl, x, alpha, beta)
            return y
        end
    end
end

# Real matrix times complex vector.
# Multiply the matrix with the real and imaginary parts separately
@inline mul!(y::StridedVector{Complex{T}}, A::StridedMaybeAdjOrTransMat{T}, x::StridedVector{Complex{T}},
        alpha::Number, beta::Number) where {T<:BlasFloat} =
    gemv!(y, A isa StridedArray ? 'N' : 'T', A isa StridedArray ? A : parent(A), x, alpha, beta)

@inline mul!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector,
             alpha::Number, beta::Number) =
    generic_matvecmul!(y, 'N', A, x, MulAddMul(alpha, beta))

function *(tA::Transpose{<:Any,<:StridedMatrix{T}}, x::StridedVector{S}) where {T<:BlasFloat,S}
    TS = promote_op(matprod, T, S)
    mul!(similar(x, TS, size(tA, 1)), tA, convert(AbstractVector{TS}, x))
end
function *(tA::Transpose{<:Any,<:AbstractMatrix{T}}, x::AbstractVector{S}) where {T,S}
    TS = promote_op(matprod, T, S)
    mul!(similar(x, TS, size(tA, 1)), tA, x)
end
@inline mul!(y::StridedVector{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, x::StridedVector{T},
                      alpha::Number, beta::Number) where {T<:BlasFloat} =
    gemv!(y, 'T', tA.parent, x, alpha, beta)
@inline mul!(y::AbstractVector, tA::Transpose{<:Any,<:AbstractVecOrMat}, x::AbstractVector,
                      alpha::Number, beta::Number) =
    generic_matvecmul!(y, 'T', tA.parent, x, MulAddMul(alpha, beta))

function *(adjA::Adjoint{<:Any,<:StridedMatrix{T}}, x::StridedVector{S}) where {T<:BlasFloat,S}
    TS = promote_op(matprod, T, S)
    mul!(similar(x, TS, size(adjA, 1)), adjA, convert(AbstractVector{TS}, x))
end
function *(adjA::Adjoint{<:Any,<:AbstractMatrix{T}}, x::AbstractVector{S}) where {T,S}
    TS = promote_op(matprod, T, S)
    mul!(similar(x, TS, size(adjA, 1)), adjA, x)
end

@inline mul!(y::StridedVector{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, x::StridedVector{T},
                      alpha::Number, beta::Number) where {T<:BlasReal} =
    mul!(y, transpose(adjA.parent), x, alpha, beta)
@inline mul!(y::StridedVector{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, x::StridedVector{T},
                      alpha::Number, beta::Number) where {T<:BlasComplex} =
    gemv!(y, 'C', adjA.parent, x, alpha, beta)
@inline mul!(y::AbstractVector, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, x::AbstractVector,
                      alpha::Number, beta::Number) =
    generic_matvecmul!(y, 'C', adjA.parent, x, MulAddMul(alpha, beta))

# Vector-Matrix multiplication
(*)(x::AdjointAbsVec,   A::AbstractMatrix) = (A'*x')'
(*)(x::TransposeAbsVec, A::AbstractMatrix) = transpose(transpose(A)*transpose(x))

_parent(A) = A
_parent(A::Adjoint) = parent(A)
_parent(A::Transpose) = parent(A)

"""
    *(A::AbstractMatrix, B::AbstractMatrix)

Matrix multiplication.

# Examples
```jldoctest
julia> [1 1; 0 1] * [1 0; 1 1]
2×2 Matrix{Int64}:
 2  1
 1  1
```
"""
function (*)(A::AbstractMatrix, B::AbstractMatrix)
    TS = promote_op(matprod, eltype(A), eltype(B))
    mul!(similar(B, TS, (size(A, 1), size(B, 2))), A, B)
end
# optimization for dispatching to BLAS, e.g. *(::Matrix{Float32}, ::Matrix{Float64})
# but avoiding the case *(::Matrix{<:BlasComplex}, ::Matrix{<:BlasReal})
# which is better handled by reinterpreting rather than promotion
function (*)(A::StridedMaybeAdjOrTransMat{<:BlasReal}, B::StridedMaybeAdjOrTransMat{<:BlasReal})
    TS = promote_type(eltype(A), eltype(B))
    mul!(similar(B, TS, (size(A, 1), size(B, 2))),
         wrapperop(A)(convert(AbstractArray{TS}, _parent(A))),
         wrapperop(B)(convert(AbstractArray{TS}, _parent(B))))
end
function (*)(A::StridedMaybeAdjOrTransMat{<:BlasComplex}, B::StridedMaybeAdjOrTransMat{<:BlasComplex})
    TS = promote_type(eltype(A), eltype(B))
    mul!(similar(B, TS, (size(A, 1), size(B, 2))),
         wrapperop(A)(convert(AbstractArray{TS}, _parent(A))),
         wrapperop(B)(convert(AbstractArray{TS}, _parent(B))))
end

@inline function mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T},
                      alpha::Number, beta::Number) where {T<:BlasFloat}
    return gemm_wrapper!(C, 'N', 'N', A, B, MulAddMul(alpha, beta))
end
# Complex Matrix times real matrix: We use that it is generally faster to reinterpret the
# first matrix as a real matrix and carry out real matrix matrix multiply
function (*)(A::StridedMatrix{<:BlasComplex}, B::StridedMaybeAdjOrTransMat{<:BlasReal})
    TS = promote_type(eltype(A), eltype(B))
    mul!(similar(B, TS, (size(A, 1), size(B, 2))),
         convert(AbstractArray{TS}, A),
         wrapperop(B)(convert(AbstractArray{real(TS)}, _parent(B))))
end
function (*)(A::AdjOrTransStridedMat{<:BlasComplex}, B::StridedMaybeAdjOrTransMat{<:BlasReal})
    TS = promote_type(eltype(A), eltype(B))
    mul!(similar(B, TS, (size(A, 1), size(B, 2))),
         copy_oftype(A, TS), # remove AdjOrTrans to use reinterpret trick below
         wrapperop(B)(convert(AbstractArray{real(TS)}, _parent(B))))
end
# the following case doesn't seem to benefit from the translation A*B = (B' * A')'
function (*)(A::StridedMatrix{<:BlasReal}, B::StridedMatrix{<:BlasComplex})
    temp = real(B)
    R = A * temp
    temp .= imag.(B)
    I = A * temp
    Complex.(R, I)
end
(*)(A::AdjOrTransStridedMat{<:BlasReal}, B::StridedMatrix{<:BlasComplex}) = copy(transpose(transpose(B) * parent(A)))
(*)(A::StridedMaybeAdjOrTransMat{<:BlasReal}, B::AdjOrTransStridedMat{<:BlasComplex}) = copy(wrapperop(B)(parent(B) * transpose(A)))

for elty in (Float32,Float64)
    @eval begin
        @inline function mul!(C::StridedMatrix{Complex{$elty}}, A::StridedVecOrMat{Complex{$elty}}, B::StridedVecOrMat{$elty},
                              alpha::Real, beta::Real)
            Afl = reinterpret($elty, A)
            Cfl = reinterpret($elty, C)
            mul!(Cfl, Afl, B, alpha, beta)
            return C
        end
    end
end

"""
    muladd(A, y, z)

Combined multiply-add, `A*y .+ z`, for matrix-matrix or matrix-vector multiplication.
The result is always the same size as `A*y`, but `z` may be smaller, or a scalar.

!!! compat "Julia 1.6"
     These methods require Julia 1.6 or later.

# Examples
```jldoctest
julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; z=[0, 100];

julia> muladd(A, B, z)
2×2 Matrix{Float64}:
   3.0    3.0
 107.0  107.0
```
"""
function Base.muladd(A::AbstractMatrix, y::AbstractVecOrMat, z::Union{Number, AbstractArray})
    Ay = A * y
    for d in 1:ndims(Ay)
        # Same error as Ay .+= z would give, to match StridedMatrix method:
        size(z,d) > size(Ay,d) && throw(DimensionMismatch("array could not be broadcast to match destination"))
    end
    for d in ndims(Ay)+1:ndims(z)
        # Similar error to what Ay + z would give, to match (Any,Any,Any) method:
        size(z,d) > 1 && throw(DimensionMismatch(string("dimensions must match: z has dims ",
            axes(z), ", must have singleton at dim ", d)))
    end
    Ay .+ z
end

function Base.muladd(u::AbstractVector, v::AdjOrTransAbsVec, z::Union{Number, AbstractArray})
    if size(z,1) > length(u) || size(z,2) > length(v)
        # Same error as (u*v) .+= z:
        throw(DimensionMismatch("array could not be broadcast to match destination"))
    end
    for d in 3:ndims(z)
        # Similar error to (u*v) + z:
        size(z,d) > 1 && throw(DimensionMismatch(string("dimensions must match: z has dims ",
            axes(z), ", must have singleton at dim ", d)))
    end
    (u .* v) .+ z
end

Base.muladd(x::AdjointAbsVec, A::AbstractMatrix, z::Union{Number, AbstractVecOrMat}) =
    muladd(A', x', z')'
Base.muladd(x::TransposeAbsVec, A::AbstractMatrix, z::Union{Number, AbstractVecOrMat}) =
    transpose(muladd(transpose(A), transpose(x), transpose(z)))

function Base.muladd(A::StridedMaybeAdjOrTransMat{<:Number}, y::AbstractVector{<:Number}, z::Union{Number, AbstractVector})
    T = promote_type(eltype(A), eltype(y), eltype(z))
    C = similar(A, T, axes(A,1))
    C .= z
    mul!(C, A, y, true, true)
end

function Base.muladd(A::StridedMaybeAdjOrTransMat{<:Number}, B::StridedMaybeAdjOrTransMat{<:Number}, z::Union{Number, AbstractVecOrMat})
    T = promote_type(eltype(A), eltype(B), eltype(z))
    C = similar(A, T, axes(A,1), axes(B,2))
    C .= z
    mul!(C, A, B, true, true)
end

"""
    mul!(Y, A, B) -> Y

Calculates the matrix-matrix or matrix-vector product ``AB`` and stores the result in `Y`,
overwriting the existing value of `Y`. Note that `Y` must not be aliased with either `A` or
`B`.

# Examples
```jldoctest
julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; Y = similar(B); mul!(Y, A, B);

julia> Y
2×2 Matrix{Float64}:
 3.0  3.0
 7.0  7.0
```

# Implementation
For custom matrix and vector types, it is recommended to implement
5-argument `mul!` rather than implementing 3-argument `mul!` directly
if possible.
"""
@inline function mul!(C, A, B)
    return mul!(C, A, B, true, false)
end

"""
    mul!(C, A, B, α, β) -> C

Combined inplace matrix-matrix or matrix-vector multiply-add ``A B α + C β``.
The result is stored in `C` by overwriting it.  Note that `C` must not be
aliased with either `A` or `B`.

!!! compat "Julia 1.3"
    Five-argument `mul!` requires at least Julia 1.3.

# Examples
```jldoctest
julia> A=[1.0 2.0; 3.0 4.0]; B=[1.0 1.0; 1.0 1.0]; C=[1.0 2.0; 3.0 4.0];

julia> mul!(C, A, B, 100.0, 10.0) === C
true

julia> C
2×2 Matrix{Float64}:
 310.0  320.0
 730.0  740.0
```
"""
@inline mul!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat,
             alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'N', 'N', A, B, MulAddMul(alpha, beta))

"""
    rmul!(A, B)

Calculate the matrix-matrix product ``AB``, overwriting `A`, and return the result.
Here, `B` must be of special matrix type, like, e.g., [`Diagonal`](@ref),
[`UpperTriangular`](@ref) or [`LowerTriangular`](@ref), or of some orthogonal type,
see [`QR`](@ref).

# Examples
```jldoctest
julia> A = [0 1; 1 0];

julia> B = LinearAlgebra.UpperTriangular([1 2; 0 3]);

julia> LinearAlgebra.rmul!(A, B);

julia> A
2×2 Matrix{Int64}:
 0  3
 1  2

julia> A = [1.0 2.0; 3.0 4.0];

julia> F = qr([0 1; -1 0]);

julia> rmul!(A, F.Q)
2×2 Matrix{Float64}:
 2.0  1.0
 4.0  3.0
```
"""
rmul!(A, B)

"""
    lmul!(A, B)

Calculate the matrix-matrix product ``AB``, overwriting `B`, and return the result.
Here, `A` must be of special matrix type, like, e.g., [`Diagonal`](@ref),
[`UpperTriangular`](@ref) or [`LowerTriangular`](@ref), or of some orthogonal type,
see [`QR`](@ref).

# Examples
```jldoctest
julia> B = [0 1; 1 0];

julia> A = LinearAlgebra.UpperTriangular([1 2; 0 3]);

julia> LinearAlgebra.lmul!(A, B);

julia> B
2×2 Matrix{Int64}:
 2  1
 3  0

julia> B = [1.0 2.0; 3.0 4.0];

julia> F = qr([0 1; -1 0]);

julia> lmul!(F.Q, B)
2×2 Matrix{Float64}:
 3.0  4.0
 1.0  2.0
```
"""
lmul!(A, B)

@inline function mul!(C::StridedMatrix{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, B::StridedVecOrMat{T},
                 alpha::Number, beta::Number) where {T<:BlasFloat}
    A = tA.parent
    if A === B
        return syrk_wrapper!(C, 'T', A, MulAddMul(alpha, beta))
    else
        return gemm_wrapper!(C, 'T', 'N', A, B, MulAddMul(alpha, beta))
    end
end
@inline mul!(C::AbstractMatrix, tA::Transpose{<:Any,<:AbstractVecOrMat}, B::AbstractVecOrMat,
                 alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'T', 'N', tA.parent, B, MulAddMul(alpha, beta))

@inline function mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, tB::Transpose{<:Any,<:StridedVecOrMat{T}},
                 alpha::Number, beta::Number) where {T<:BlasFloat}
    B = tB.parent
    if A === B
        return syrk_wrapper!(C, 'N', A, MulAddMul(alpha, beta))
    else
        return gemm_wrapper!(C, 'N', 'T', A, B, MulAddMul(alpha, beta))
    end
end
# Complex matrix times transposed real matrix. Reinterpret the first matrix to real for efficiency.
for elty in (Float32,Float64)
    @eval begin
        @inline function mul!(C::StridedMatrix{Complex{$elty}}, A::StridedVecOrMat{Complex{$elty}}, tB::Transpose{<:Any,<:StridedVecOrMat{$elty}},
                         alpha::Real, beta::Real)
            Afl = reinterpret($elty, A)
            Cfl = reinterpret($elty, C)
            mul!(Cfl, Afl, tB, alpha, beta)
            return C
        end
    end
end
# collapsing the following two defs with C::AbstractVecOrMat yields ambiguities
@inline mul!(C::AbstractVector, A::AbstractVecOrMat, tB::Transpose{<:Any,<:AbstractVecOrMat},
             alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'N', 'T', A, tB.parent, MulAddMul(alpha, beta))
@inline mul!(C::AbstractMatrix, A::AbstractVecOrMat, tB::Transpose{<:Any,<:AbstractVecOrMat},
             alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'N', 'T', A, tB.parent, MulAddMul(alpha, beta))

@inline mul!(C::StridedMatrix{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, tB::Transpose{<:Any,<:StridedVecOrMat{T}},
                 alpha::Number, beta::Number) where {T<:BlasFloat} =
    gemm_wrapper!(C, 'T', 'T', tA.parent, tB.parent, MulAddMul(alpha, beta))
@inline mul!(C::AbstractMatrix, tA::Transpose{<:Any,<:AbstractVecOrMat}, tB::Transpose{<:Any,<:AbstractVecOrMat},
                 alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'T', 'T', tA.parent, tB.parent, MulAddMul(alpha, beta))

@inline mul!(C::StridedMatrix{T}, tA::Transpose{<:Any,<:StridedVecOrMat{T}}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}},
                 alpha::Number, beta::Number) where {T<:BlasFloat} =
    gemm_wrapper!(C, 'T', 'C', tA.parent, adjB.parent, MulAddMul(alpha, beta))
@inline mul!(C::AbstractMatrix, tA::Transpose{<:Any,<:AbstractVecOrMat}, tB::Adjoint{<:Any,<:AbstractVecOrMat},
                 alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'T', 'C', tA.parent, tB.parent, MulAddMul(alpha, beta))

@inline mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, B::StridedVecOrMat{T},
                 alpha::Real, beta::Real) where {T<:BlasReal} =
    mul!(C, transpose(adjA.parent), B, alpha, beta)
@inline function mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, B::StridedVecOrMat{T},
                 alpha::Number, beta::Number) where {T<:BlasComplex}
    A = adjA.parent
    if A === B
        return herk_wrapper!(C, 'C', A, MulAddMul(alpha, beta))
    else
        return gemm_wrapper!(C, 'C', 'N', A, B, MulAddMul(alpha, beta))
    end
end
@inline mul!(C::AbstractMatrix, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, B::AbstractVecOrMat,
                 alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'C', 'N', adjA.parent, B, MulAddMul(alpha, beta))

@inline mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, adjB::Adjoint{<:Any,<:StridedVecOrMat{<:BlasReal}},
                 alpha::Number, beta::Number) where {T<:BlasFloat} =
    mul!(C, A, transpose(adjB.parent), alpha, beta)
@inline function mul!(C::StridedMatrix{T}, A::StridedVecOrMat{T}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}},
                 alpha::Number, beta::Number) where {T<:BlasComplex}
    B = adjB.parent
    if A === B
        return herk_wrapper!(C, 'N', A, MulAddMul(alpha, beta))
    else
        return gemm_wrapper!(C, 'N', 'C', A, B, MulAddMul(alpha, beta))
    end
end
@inline mul!(C::AbstractMatrix, A::AbstractVecOrMat, adjB::Adjoint{<:Any,<:AbstractVecOrMat},
                 alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'N', 'C', A, adjB.parent, MulAddMul(alpha, beta))

@inline mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, adjB::Adjoint{<:Any,<:StridedVecOrMat{T}},
                 alpha::Number, beta::Number) where {T<:BlasFloat} =
    gemm_wrapper!(C, 'C', 'C', adjA.parent, adjB.parent, MulAddMul(alpha, beta))
@inline mul!(C::AbstractMatrix, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, adjB::Adjoint{<:Any,<:AbstractVecOrMat},
                 alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'C', 'C', adjA.parent, adjB.parent, MulAddMul(alpha, beta))

@inline mul!(C::StridedMatrix{T}, adjA::Adjoint{<:Any,<:StridedVecOrMat{T}}, tB::Transpose{<:Any,<:StridedVecOrMat{T}},
                 alpha::Number, beta::Number) where {T<:BlasFloat} =
    gemm_wrapper!(C, 'C', 'T', adjA.parent, tB.parent, MulAddMul(alpha, beta))
@inline mul!(C::AbstractMatrix, adjA::Adjoint{<:Any,<:AbstractVecOrMat}, tB::Transpose{<:Any,<:AbstractVecOrMat},
                 alpha::Number, beta::Number) =
    generic_matmatmul!(C, 'C', 'T', adjA.parent, tB.parent, MulAddMul(alpha, beta))

# Supporting functions for matrix multiplication

# copy transposed(adjoint) of upper(lower) side-digonals. Optionally include diagonal.
@inline function copytri!(A::AbstractMatrix, uplo::AbstractChar, conjugate::Bool=false, diag::Bool=false)
    n = checksquare(A)
    off = diag ? 0 : 1
    if uplo == 'U'
        for i = 1:n, j = (i+off):n
            A[j,i] = conjugate ? adjoint(A[i,j]) : transpose(A[i,j])
        end
    elseif uplo == 'L'
        for i = 1:n, j = (i+off):n
            A[i,j] = conjugate ? adjoint(A[j,i]) : transpose(A[j,i])
        end
    else
        throw(ArgumentError(lazy"uplo argument must be 'U' (upper) or 'L' (lower), got $uplo"))
    end
    A
end

function gemv!(y::StridedVector{T}, tA::AbstractChar, A::StridedVecOrMat{T}, x::StridedVector{T},
               α::Number=true, β::Number=false) where {T<:BlasFloat}
    mA, nA = lapack_size(tA, A)
    if nA != length(x)
        throw(DimensionMismatch(lazy"second dimension of A, $nA, does not match length of x, $(length(x))"))
    end
    if mA != length(y)
        throw(DimensionMismatch(lazy"first dimension of A, $mA, does not match length of y, $(length(y))"))
    end
    if mA == 0
        return y
    end
    if nA == 0
        return _rmul_or_fill!(y, β)
    end

    alpha, beta = promote(α, β, zero(T))
    if alpha isa Union{Bool,T} && beta isa Union{Bool,T} && stride(A, 1) == 1 && stride(A, 2) >= size(A, 1)
        return BLAS.gemv!(tA, alpha, A, x, beta, y)
    else
        return generic_matvecmul!(y, tA, A, x, MulAddMul(α, β))
    end
end

function gemv!(y::StridedVector{Complex{T}}, tA::AbstractChar, A::StridedVecOrMat{T}, x::StridedVector{Complex{T}},
    α::Number = true, β::Number = false) where {T<:BlasFloat}
    mA, nA = lapack_size(tA, A)
    nA != length(x) &&
        throw(DimensionMismatch(lazy"second dimension of A, $nA, does not match length of x, $(length(x))"))
    mA != length(y) &&
        throw(DimensionMismatch(lazy"first dimension of A, $mA, does not match length of y, $(length(y))"))
    mA == 0 && return y
    nA == 0 && return _rmul_or_fill!(y, β)
    alpha, beta = promote(α, β, zero(T))
    @views if alpha isa Union{Bool,T} && beta isa Union{Bool,T} && stride(A, 1) == 1 && stride(A, 2) >= size(A, 1)
        xfl = reinterpret(reshape, T, x) # Use reshape here.
        yfl = reinterpret(reshape, T, y)
        BLAS.gemv!(tA, alpha, A, xfl[1, :], beta, yfl[1, :])
        BLAS.gemv!(tA, alpha, A, xfl[2, :], beta, yfl[2, :])
        return y
    else
        return generic_matvecmul!(y, tA, A, x, MulAddMul(α, β))
    end
end

function syrk_wrapper!(C::StridedMatrix{T}, tA::AbstractChar, A::StridedVecOrMat{T},
        _add = MulAddMul()) where {T<:BlasFloat}
    nC = checksquare(C)
    if tA == 'T'
        (nA, mA) = size(A,1), size(A,2)
        tAt = 'N'
    else
        (mA, nA) = size(A,1), size(A,2)
        tAt = 'T'
    end
    if nC != mA
        throw(DimensionMismatch(lazy"output matrix has size: $(nC), but should have size $(mA)"))
    end
    if mA == 0 || nA == 0 || iszero(_add.alpha)
        return _rmul_or_fill!(C, _add.beta)
    end
    if mA == 2 && nA == 2
        return matmul2x2!(C, tA, tAt, A, A, _add)
    end
    if mA == 3 && nA == 3
        return matmul3x3!(C, tA, tAt, A, A, _add)
    end

    # BLAS.syrk! only updates symmetric C
    # alternatively, make non-zero β a show-stopper for BLAS.syrk!
    if iszero(_add.beta) || issymmetric(C)
        alpha, beta = promote(_add.alpha, _add.beta, zero(T))
        if (alpha isa Union{Bool,T} &&
            beta isa Union{Bool,T} &&
            stride(A, 1) == stride(C, 1) == 1 &&
            stride(A, 2) >= size(A, 1) &&
            stride(C, 2) >= size(C, 1))
            return copytri!(BLAS.syrk!('U', tA, alpha, A, beta, C), 'U')
        end
    end
    return gemm_wrapper!(C, tA, tAt, A, A, _add)
end

function herk_wrapper!(C::Union{StridedMatrix{T}, StridedMatrix{Complex{T}}}, tA::AbstractChar, A::Union{StridedVecOrMat{T}, StridedVecOrMat{Complex{T}}},
        _add = MulAddMul()) where {T<:BlasReal}
    nC = checksquare(C)
    if tA == 'C'
        (nA, mA) = size(A,1), size(A,2)
        tAt = 'N'
    else
        (mA, nA) = size(A,1), size(A,2)
        tAt = 'C'
    end
    if nC != mA
        throw(DimensionMismatch(lazy"output matrix has size: $(nC), but should have size $(mA)"))
    end
    if mA == 0 || nA == 0 || iszero(_add.alpha)
        return _rmul_or_fill!(C, _add.beta)
    end
    if mA == 2 && nA == 2
        return matmul2x2!(C, tA, tAt, A, A, _add)
    end
    if mA == 3 && nA == 3
        return matmul3x3!(C, tA, tAt, A, A, _add)
    end

    # Result array does not need to be initialized as long as beta==0
    #    C = Matrix{T}(undef, mA, mA)

    if iszero(_add.beta) || issymmetric(C)
        alpha, beta = promote(_add.alpha, _add.beta, zero(T))
        if (alpha isa Union{Bool,T} &&
            beta isa Union{Bool,T} &&
            stride(A, 1) == stride(C, 1) == 1 &&
            stride(A, 2) >= size(A, 1) &&
            stride(C, 2) >= size(C, 1))
            return copytri!(BLAS.herk!('U', tA, alpha, A, beta, C), 'U', true)
        end
    end
    return gemm_wrapper!(C, tA, tAt, A, A, _add)
end

function gemm_wrapper(tA::AbstractChar, tB::AbstractChar,
                      A::StridedVecOrMat{T},
                      B::StridedVecOrMat{T}) where {T<:BlasFloat}
    mA, nA = lapack_size(tA, A)
    mB, nB = lapack_size(tB, B)
    C = similar(B, T, mA, nB)
    gemm_wrapper!(C, tA, tB, A, B)
end

function gemm_wrapper!(C::StridedVecOrMat{T}, tA::AbstractChar, tB::AbstractChar,
                       A::StridedVecOrMat{T}, B::StridedVecOrMat{T},
                       _add = MulAddMul()) where {T<:BlasFloat}
    mA, nA = lapack_size(tA, A)
    mB, nB = lapack_size(tB, B)

    if nA != mB
        throw(DimensionMismatch(lazy"A has dimensions ($mA,$nA) but B has dimensions ($mB,$nB)"))
    end

    if C === A || B === C
        throw(ArgumentError("output matrix must not be aliased with input matrix"))
    end

    if mA == 0 || nA == 0 || nB == 0 || iszero(_add.alpha)
        if size(C) != (mA, nB)
            throw(DimensionMismatch(lazy"C has dimensions $(size(C)), should have ($mA,$nB)"))
        end
        return _rmul_or_fill!(C, _add.beta)
    end

    if mA == 2 && nA == 2 && nB == 2
        return matmul2x2!(C, tA, tB, A, B, _add)
    end
    if mA == 3 && nA == 3 && nB == 3
        return matmul3x3!(C, tA, tB, A, B, _add)
    end

    alpha, beta = promote(_add.alpha, _add.beta, zero(T))
    if (alpha isa Union{Bool,T} &&
        beta isa Union{Bool,T} &&
        stride(A, 1) == stride(B, 1) == stride(C, 1) == 1 &&
        stride(A, 2) >= size(A, 1) &&
        stride(B, 2) >= size(B, 1) &&
        stride(C, 2) >= size(C, 1))
        return BLAS.gemm!(tA, tB, alpha, A, B, beta, C)
    end
    generic_matmatmul!(C, tA, tB, A, B, _add)
end

# blas.jl defines matmul for floats; other integer and mixed precision
# cases are handled here

lapack_size(t::AbstractChar, M::AbstractVecOrMat) = (size(M, t=='N' ? 1 : 2), size(M, t=='N' ? 2 : 1))

function copyto!(B::AbstractVecOrMat, ir_dest::AbstractUnitRange{Int}, jr_dest::AbstractUnitRange{Int}, tM::AbstractChar, M::AbstractVecOrMat, ir_src::AbstractUnitRange{Int}, jr_src::AbstractUnitRange{Int})
    if tM == 'N'
        copyto!(B, ir_dest, jr_dest, M, ir_src, jr_src)
    else
        LinearAlgebra.copy_transpose!(B, ir_dest, jr_dest, M, jr_src, ir_src)
        tM == 'C' && conj!(@view B[ir_dest, jr_dest])
    end
    B
end

function copy_transpose!(B::AbstractMatrix, ir_dest::AbstractUnitRange{Int}, jr_dest::AbstractUnitRange{Int}, tM::AbstractChar, M::AbstractVecOrMat, ir_src::AbstractUnitRange{Int}, jr_src::AbstractUnitRange{Int})
    if tM == 'N'
        LinearAlgebra.copy_transpose!(B, ir_dest, jr_dest, M, ir_src, jr_src)
    else
        copyto!(B, ir_dest, jr_dest, M, jr_src, ir_src)
        tM == 'C' && conj!(@view B[ir_dest, jr_dest])
    end
    B
end

# TODO: It will be faster for large matrices to convert to float,
# call BLAS, and convert back to required type.

# NOTE: the generic version is also called as fallback for
#       strides != 1 cases

function generic_matvecmul!(C::AbstractVector{R}, tA, A::AbstractVecOrMat, B::AbstractVector,
                            _add::MulAddMul = MulAddMul()) where R
    require_one_based_indexing(C, A, B)
    mB = length(B)
    mA, nA = lapack_size(tA, A)
    if mB != nA
        throw(DimensionMismatch(lazy"matrix A has dimensions ($mA,$nA), vector B has length $mB"))
    end
    if mA != length(C)
        throw(DimensionMismatch(lazy"result C has length $(length(C)), needs length $mA"))
    end

    Astride = size(A, 1)

    @inbounds begin
    if tA == 'T'  # fastest case
        if nA == 0
            for k = 1:mA
                _modify!(_add, false, C, k)
            end
        else
            for k = 1:mA
                aoffs = (k-1)*Astride
                s = zero(A[aoffs + 1]*B[1] + A[aoffs + 1]*B[1])
                for i = 1:nA
                    s += transpose(A[aoffs+i]) * B[i]
                end
                _modify!(_add, s, C, k)
            end
        end
    elseif tA == 'C'
        if nA == 0
            for k = 1:mA
                _modify!(_add, false, C, k)
            end
        else
            for k = 1:mA
                aoffs = (k-1)*Astride
                s = zero(A[aoffs + 1]*B[1] + A[aoffs + 1]*B[1])
                for i = 1:nA
                    s += A[aoffs + i]'B[i]
                end
                _modify!(_add, s, C, k)
            end
        end
    else # tA == 'N'
        for i = 1:mA
            if !iszero(_add.beta)
                C[i] *= _add.beta
            elseif mB == 0
                C[i] = false
            else
                C[i] = zero(A[i]*B[1] + A[i]*B[1])
            end
        end
        for k = 1:mB
            aoffs = (k-1)*Astride
            b = _add(B[k], false)
            for i = 1:mA
                C[i] += A[aoffs + i] * b
            end
        end
    end
    end # @inbounds
    C
end

function generic_matmatmul(tA, tB, A::AbstractVecOrMat{T}, B::AbstractMatrix{S}) where {T,S}
    mA, nA = lapack_size(tA, A)
    mB, nB = lapack_size(tB, B)
    C = similar(B, promote_op(matprod, T, S), mA, nB)
    generic_matmatmul!(C, tA, tB, A, B)
end

const tilebufsize = 10800  # Approximately 32k/3

function generic_matmatmul!(C::AbstractMatrix, tA, tB, A::AbstractMatrix, B::AbstractMatrix,
                            _add::MulAddMul=MulAddMul())
    mA, nA = lapack_size(tA, A)
    mB, nB = lapack_size(tB, B)
    mC, nC = size(C)

    if iszero(_add.alpha)
        return _rmul_or_fill!(C, _add.beta)
    end
    if mA == nA == mB == nB == mC == nC == 2
        return matmul2x2!(C, tA, tB, A, B, _add)
    end
    if mA == nA == mB == nB == mC == nC == 3
        return matmul3x3!(C, tA, tB, A, B, _add)
    end
    _generic_matmatmul!(C, tA, tB, A, B, _add)
end

generic_matmatmul!(C::AbstractVecOrMat, tA, tB, A::AbstractVecOrMat, B::AbstractVecOrMat, _add::MulAddMul) =
    _generic_matmatmul!(C, tA, tB, A, B, _add)

function _generic_matmatmul!(C::AbstractVecOrMat{R}, tA, tB, A::AbstractVecOrMat{T}, B::AbstractVecOrMat{S},
                             _add::MulAddMul) where {T,S,R}
    require_one_based_indexing(C, A, B)
    mA, nA = lapack_size(tA, A)
    mB, nB = lapack_size(tB, B)
    if mB != nA
        throw(DimensionMismatch(lazy"matrix A has dimensions ($mA,$nA), matrix B has dimensions ($mB,$nB)"))
    end
    if size(C,1) != mA || size(C,2) != nB
        throw(DimensionMismatch(lazy"result C has dimensions $(size(C)), needs ($mA,$nB)"))
    end

    if iszero(_add.alpha) || isempty(A) || isempty(B)
        return _rmul_or_fill!(C, _add.beta)
    end

    tile_size = 0
    if isbitstype(R) && isbitstype(T) && isbitstype(S) && (tA == 'N' || tB != 'N')
        tile_size = floor(Int, sqrt(tilebufsize / max(sizeof(R), sizeof(S), sizeof(T), 1)))
    end
    @inbounds begin
    if tile_size > 0
        sz = (tile_size, tile_size)
        Atile = Array{T}(undef, sz)
        Btile = Array{S}(undef, sz)

        z1 = zero(A[1, 1]*B[1, 1] + A[1, 1]*B[1, 1])
        z = convert(promote_type(typeof(z1), R), z1)

        if mA < tile_size && nA < tile_size && nB < tile_size
            copy_transpose!(Atile, 1:nA, 1:mA, tA, A, 1:mA, 1:nA)
            copyto!(Btile, 1:mB, 1:nB, tB, B, 1:mB, 1:nB)
            for j = 1:nB
                boff = (j-1)*tile_size
                for i = 1:mA
                    aoff = (i-1)*tile_size
                    s = z
                    for k = 1:nA
                        s += Atile[aoff+k] * Btile[boff+k]
                    end
                    _modify!(_add, s, C, (i,j))
                end
            end
        else
            Ctile = Array{R}(undef, sz)
            for jb = 1:tile_size:nB
                jlim = min(jb+tile_size-1,nB)
                jlen = jlim-jb+1
                for ib = 1:tile_size:mA
                    ilim = min(ib+tile_size-1,mA)
                    ilen = ilim-ib+1
                    fill!(Ctile, z)
                    for kb = 1:tile_size:nA
                        klim = min(kb+tile_size-1,mB)
                        klen = klim-kb+1
                        copy_transpose!(Atile, 1:klen, 1:ilen, tA, A, ib:ilim, kb:klim)
                        copyto!(Btile, 1:klen, 1:jlen, tB, B, kb:klim, jb:jlim)
                        for j=1:jlen
                            bcoff = (j-1)*tile_size
                            for i = 1:ilen
                                aoff = (i-1)*tile_size
                                s = z
                                for k = 1:klen
                                    s += Atile[aoff+k] * Btile[bcoff+k]
                                end
                                Ctile[bcoff+i] += s
                            end
                        end
                    end
                    if isone(_add.alpha) && iszero(_add.beta)
                        copyto!(C, ib:ilim, jb:jlim, Ctile, 1:ilen, 1:jlen)
                    else
                        C[ib:ilim, jb:jlim] .= @views _add.(Ctile[1:ilen, 1:jlen], C[ib:ilim, jb:jlim])
                    end
                end
            end
        end
    else
        # Multiplication for non-plain-data uses the naive algorithm
        if tA == 'N'
            if tB == 'N'
                for i = 1:mA, j = 1:nB
                    z2 = zero(A[i, 1]*B[1, j] + A[i, 1]*B[1, j])
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += A[i, k]*B[k, j]
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            elseif tB == 'T'
                for i = 1:mA, j = 1:nB
                    z2 = zero(A[i, 1]*transpose(B[j, 1]) + A[i, 1]*transpose(B[j, 1]))
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += A[i, k] * transpose(B[j, k])
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            else
                for i = 1:mA, j = 1:nB
                    z2 = zero(A[i, 1]*B[j, 1]' + A[i, 1]*B[j, 1]')
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += A[i, k]*B[j, k]'
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            end
        elseif tA == 'T'
            if tB == 'N'
                for i = 1:mA, j = 1:nB
                    z2 = zero(transpose(A[1, i])*B[1, j] + transpose(A[1, i])*B[1, j])
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += transpose(A[k, i]) * B[k, j]
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            elseif tB == 'T'
                for i = 1:mA, j = 1:nB
                    z2 = zero(transpose(A[1, i])*transpose(B[j, 1]) + transpose(A[1, i])*transpose(B[j, 1]))
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += transpose(A[k, i]) * transpose(B[j, k])
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            else
                for i = 1:mA, j = 1:nB
                    z2 = zero(transpose(A[1, i])*B[j, 1]' + transpose(A[1, i])*B[j, 1]')
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += transpose(A[k, i]) * adjoint(B[j, k])
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            end
        else
            if tB == 'N'
                for i = 1:mA, j = 1:nB
                    z2 = zero(A[1, i]'*B[1, j] + A[1, i]'*B[1, j])
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += A[k, i]'B[k, j]
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            elseif tB == 'T'
                for i = 1:mA, j = 1:nB
                    z2 = zero(A[1, i]'*transpose(B[j, 1]) + A[1, i]'*transpose(B[j, 1]))
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += adjoint(A[k, i]) * transpose(B[j, k])
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            else
                for i = 1:mA, j = 1:nB
                    z2 = zero(A[1, i]'*B[j, 1]' + A[1, i]'*B[j, 1]')
                    Ctmp = convert(promote_type(R, typeof(z2)), z2)
                    for k = 1:nA
                        Ctmp += A[k, i]'B[j, k]'
                    end
                    _modify!(_add, Ctmp, C, (i,j))
                end
            end
        end
    end
    end # @inbounds
    C
end

# multiply 2x2 matrices
function matmul2x2(tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) where {T,S}
    matmul2x2!(similar(B, promote_op(matprod, T, S), 2, 2), tA, tB, A, B)
end

function matmul2x2!(C::AbstractMatrix, tA, tB, A::AbstractMatrix, B::AbstractMatrix,
                    _add::MulAddMul = MulAddMul())
    require_one_based_indexing(C, A, B)
    if !(size(A) == size(B) == size(C) == (2,2))
        throw(DimensionMismatch(lazy"A has size $(size(A)), B has size $(size(B)), C has size $(size(C))"))
    end
    @inbounds begin
    if tA == 'T'
        # TODO making these lazy could improve perf
        A11 = copy(transpose(A[1,1])); A12 = copy(transpose(A[2,1]))
        A21 = copy(transpose(A[1,2])); A22 = copy(transpose(A[2,2]))
    elseif tA == 'C'
        # TODO making these lazy could improve perf
        A11 = copy(A[1,1]'); A12 = copy(A[2,1]')
        A21 = copy(A[1,2]'); A22 = copy(A[2,2]')
    else
        A11 = A[1,1]; A12 = A[1,2]; A21 = A[2,1]; A22 = A[2,2]
    end
    if tB == 'T'
        # TODO making these lazy could improve perf
        B11 = copy(transpose(B[1,1])); B12 = copy(transpose(B[2,1]))
        B21 = copy(transpose(B[1,2])); B22 = copy(transpose(B[2,2]))
    elseif tB == 'C'
        # TODO making these lazy could improve perf
        B11 = copy(B[1,1]'); B12 = copy(B[2,1]')
        B21 = copy(B[1,2]'); B22 = copy(B[2,2]')
    else
        B11 = B[1,1]; B12 = B[1,2];
        B21 = B[2,1]; B22 = B[2,2]
    end
    _modify!(_add, A11*B11 + A12*B21, C, (1,1))
    _modify!(_add, A11*B12 + A12*B22, C, (1,2))
    _modify!(_add, A21*B11 + A22*B21, C, (2,1))
    _modify!(_add, A21*B12 + A22*B22, C, (2,2))
    end # inbounds
    C
end

# Multiply 3x3 matrices
function matmul3x3(tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) where {T,S}
    matmul3x3!(similar(B, promote_op(matprod, T, S), 3, 3), tA, tB, A, B)
end

function matmul3x3!(C::AbstractMatrix, tA, tB, A::AbstractMatrix, B::AbstractMatrix,
                    _add::MulAddMul = MulAddMul())
    require_one_based_indexing(C, A, B)
    if !(size(A) == size(B) == size(C) == (3,3))
        throw(DimensionMismatch(lazy"A has size $(size(A)), B has size $(size(B)), C has size $(size(C))"))
    end
    @inbounds begin
    if tA == 'T'
        # TODO making these lazy could improve perf
        A11 = copy(transpose(A[1,1])); A12 = copy(transpose(A[2,1])); A13 = copy(transpose(A[3,1]))
        A21 = copy(transpose(A[1,2])); A22 = copy(transpose(A[2,2])); A23 = copy(transpose(A[3,2]))
        A31 = copy(transpose(A[1,3])); A32 = copy(transpose(A[2,3])); A33 = copy(transpose(A[3,3]))
    elseif tA == 'C'
        # TODO making these lazy could improve perf
        A11 = copy(A[1,1]'); A12 = copy(A[2,1]'); A13 = copy(A[3,1]')
        A21 = copy(A[1,2]'); A22 = copy(A[2,2]'); A23 = copy(A[3,2]')
        A31 = copy(A[1,3]'); A32 = copy(A[2,3]'); A33 = copy(A[3,3]')
    else
        A11 = A[1,1]; A12 = A[1,2]; A13 = A[1,3]
        A21 = A[2,1]; A22 = A[2,2]; A23 = A[2,3]
        A31 = A[3,1]; A32 = A[3,2]; A33 = A[3,3]
    end

    if tB == 'T'
        # TODO making these lazy could improve perf
        B11 = copy(transpose(B[1,1])); B12 = copy(transpose(B[2,1])); B13 = copy(transpose(B[3,1]))
        B21 = copy(transpose(B[1,2])); B22 = copy(transpose(B[2,2])); B23 = copy(transpose(B[3,2]))
        B31 = copy(transpose(B[1,3])); B32 = copy(transpose(B[2,3])); B33 = copy(transpose(B[3,3]))
    elseif tB == 'C'
        # TODO making these lazy could improve perf
        B11 = copy(B[1,1]'); B12 = copy(B[2,1]'); B13 = copy(B[3,1]')
        B21 = copy(B[1,2]'); B22 = copy(B[2,2]'); B23 = copy(B[3,2]')
        B31 = copy(B[1,3]'); B32 = copy(B[2,3]'); B33 = copy(B[3,3]')
    else
        B11 = B[1,1]; B12 = B[1,2]; B13 = B[1,3]
        B21 = B[2,1]; B22 = B[2,2]; B23 = B[2,3]
        B31 = B[3,1]; B32 = B[3,2]; B33 = B[3,3]
    end

    _modify!(_add, A11*B11 + A12*B21 + A13*B31, C, (1,1))
    _modify!(_add, A11*B12 + A12*B22 + A13*B32, C, (1,2))
    _modify!(_add, A11*B13 + A12*B23 + A13*B33, C, (1,3))

    _modify!(_add, A21*B11 + A22*B21 + A23*B31, C, (2,1))
    _modify!(_add, A21*B12 + A22*B22 + A23*B32, C, (2,2))
    _modify!(_add, A21*B13 + A22*B23 + A23*B33, C, (2,3))

    _modify!(_add, A31*B11 + A32*B21 + A33*B31, C, (3,1))
    _modify!(_add, A31*B12 + A32*B22 + A33*B32, C, (3,2))
    _modify!(_add, A31*B13 + A32*B23 + A33*B33, C, (3,3))
    end # inbounds
    C
end

const RealOrComplex = Union{Real,Complex}

# Three-argument *
"""
    *(A, B::AbstractMatrix, C)
    A * B * C * D

Chained multiplication of 3 or 4 matrices is done in the most efficient sequence,
based on the sizes of the arrays. That is, the number of scalar multiplications needed
for `(A * B) * C` (with 3 dense matrices) is compared to that for `A * (B * C)`
to choose which of these to execute.

If the last factor is a vector, or the first a transposed vector, then it is efficient
to deal with these first. In particular `x' * B * y` means `(x' * B) * y`
for an ordinary column-major `B::Matrix`. Unlike `dot(x, B, y)`, this
allocates an intermediate array.

If the first or last factor is a number, this will be fused with the matrix
multiplication, using 5-arg [`mul!`](@ref).

See also [`muladd`](@ref), [`dot`](@ref).

!!! compat "Julia 1.7"
    These optimisations require at least Julia 1.7.
"""
*(A::AbstractMatrix, B::AbstractMatrix, x::AbstractVector) = A * (B*x)

*(tu::AdjOrTransAbsVec, B::AbstractMatrix, v::AbstractVector) = (tu*B) * v
*(tu::AdjOrTransAbsVec, B::AdjOrTransAbsMat, v::AbstractVector) = tu * (B*v)

*(A::AbstractMatrix, x::AbstractVector, γ::Number) = mat_vec_scalar(A,x,γ)
*(A::AbstractMatrix, B::AbstractMatrix, γ::Number) = mat_mat_scalar(A,B,γ)
*(α::RealOrComplex, B::AbstractMatrix{<:RealOrComplex}, C::AbstractVector{<:RealOrComplex}) =
    mat_vec_scalar(B,C,α)
*(α::RealOrComplex, B::AbstractMatrix{<:RealOrComplex}, C::AbstractMatrix{<:RealOrComplex}) =
    mat_mat_scalar(B,C,α)

*(α::Number, u::AbstractVector, tv::AdjOrTransAbsVec) = broadcast(*, α, u, tv)
*(u::AbstractVector, tv::AdjOrTransAbsVec, γ::Number) = broadcast(*, u, tv, γ)
*(u::AbstractVector, tv::AdjOrTransAbsVec, C::AbstractMatrix) = u * (tv*C)

*(A::AbstractMatrix, B::AbstractMatrix, C::AbstractMatrix) = _tri_matmul(A,B,C)
*(tv::AdjOrTransAbsVec, B::AbstractMatrix, C::AbstractMatrix) = (tv*B) * C

function _tri_matmul(A,B,C,δ=nothing)
    n,m = size(A)
    # m,k == size(B)
    k,l = size(C)
    costAB_C = n*m*k + n*k*l  # multiplications, allocations n*k + n*l
    costA_BC = m*k*l + n*m*l  #                              m*l + n*l
    if costA_BC < costAB_C
        isnothing(δ) ? A * (B*C) : A * mat_mat_scalar(B,C,δ)
    else
        isnothing(δ) ? (A*B) * C : mat_mat_scalar(A*B, C, δ)
    end
end

# Fast path for two arrays * one scalar is opt-in, via mat_vec_scalar and mat_mat_scalar.

mat_vec_scalar(A, x, γ) = A * (x * γ)  # fallback
mat_vec_scalar(A::StridedMaybeAdjOrTransMat, x::StridedVector, γ) = _mat_vec_scalar(A, x, γ)
mat_vec_scalar(A::AdjOrTransAbsVec, x::StridedVector, γ) = (A * x) * γ

function _mat_vec_scalar(A, x, γ)
    T = promote_type(eltype(A), eltype(x), typeof(γ))
    C = similar(A, T, axes(A,1))
    mul!(C, A, x, γ, false)
end

mat_mat_scalar(A, B, γ) = (A*B) * γ # fallback
mat_mat_scalar(A::StridedMaybeAdjOrTransMat, B::StridedMaybeAdjOrTransMat, γ) =
    _mat_mat_scalar(A, B, γ)

function _mat_mat_scalar(A, B, γ)
    T = promote_type(eltype(A), eltype(B), typeof(γ))
    C = similar(A, T, axes(A,1), axes(B,2))
    mul!(C, A, B, γ, false)
end

mat_mat_scalar(A::AdjointAbsVec, B, γ) = (γ' * (A * B)')' # preserving order, adjoint reverses
mat_mat_scalar(A::AdjointAbsVec{<:RealOrComplex}, B::StridedMaybeAdjOrTransMat{<:RealOrComplex}, γ::RealOrComplex) =
    mat_vec_scalar(B', A', γ')'

mat_mat_scalar(A::TransposeAbsVec, B, γ) = transpose(γ * transpose(A * B))
mat_mat_scalar(A::TransposeAbsVec{<:RealOrComplex}, B::StridedMaybeAdjOrTransMat{<:RealOrComplex}, γ::RealOrComplex) =
    transpose(mat_vec_scalar(transpose(B), transpose(A), γ))

# Four-argument *, by type
*(α::Number, β::Number, C::AbstractMatrix, x::AbstractVector) = (α*β) * C * x
*(α::Number, β::Number, C::AbstractMatrix, D::AbstractMatrix) = (α*β) * C * D
*(α::Number, B::AbstractMatrix, C::AbstractMatrix, x::AbstractVector) = α * B * (C*x)
*(α::Number, vt::AdjOrTransAbsVec, C::AbstractMatrix, x::AbstractVector) = α * (vt*C*x)
*(α::RealOrComplex, vt::AdjOrTransAbsVec{<:RealOrComplex}, C::AbstractMatrix{<:RealOrComplex}, D::AbstractMatrix{<:RealOrComplex}) =
    (α*vt*C) * D # solves an ambiguity

*(A::AbstractMatrix, x::AbstractVector, γ::Number, δ::Number) = A * x * (γ*δ)
*(A::AbstractMatrix, B::AbstractMatrix, γ::Number, δ::Number) = A * B * (γ*δ)
*(A::AbstractMatrix, B::AbstractMatrix, x::AbstractVector, δ::Number, ) = A * (B*x*δ)
*(vt::AdjOrTransAbsVec, B::AbstractMatrix, x::AbstractVector, δ::Number) = (vt*B*x) * δ
*(vt::AdjOrTransAbsVec, B::AbstractMatrix, C::AbstractMatrix, δ::Number) = (vt*B) * C * δ

*(A::AbstractMatrix, B::AbstractMatrix, C::AbstractMatrix, x::AbstractVector) = A * B * (C*x)
*(vt::AdjOrTransAbsVec, B::AbstractMatrix, C::AbstractMatrix, D::AbstractMatrix) = (vt*B) * C * D
*(vt::AdjOrTransAbsVec, B::AbstractMatrix, C::AbstractMatrix, x::AbstractVector) = vt * B * (C*x)

# Four-argument *, by size
*(A::AbstractMatrix, B::AbstractMatrix, C::AbstractMatrix, δ::Number) = _tri_matmul(A,B,C,δ)
*(α::RealOrComplex, B::AbstractMatrix{<:RealOrComplex}, C::AbstractMatrix{<:RealOrComplex}, D::AbstractMatrix{<:RealOrComplex}) =
    _tri_matmul(B,C,D,α)
*(A::AbstractMatrix, B::AbstractMatrix, C::AbstractMatrix, D::AbstractMatrix) =
    _quad_matmul(A,B,C,D)

function _quad_matmul(A,B,C,D)
    c1 = _mul_cost((A,B),(C,D))
    c2 = _mul_cost(((A,B),C),D)
    c3 = _mul_cost(A,(B,(C,D)))
    c4 = _mul_cost((A,(B,C)),D)
    c5 = _mul_cost(A,((B,C),D))
    cmin = min(c1,c2,c3,c4,c5)
    if c1 == cmin
        (A*B) * (C*D)
    elseif c2 == cmin
        ((A*B) * C) * D
    elseif c3 == cmin
        A * (B * (C*D))
    elseif c4 == cmin
        (A * (B*C)) * D
    else
        A * ((B*C) * D)
    end
end
@inline _mul_cost(A::AbstractMatrix) = 0
@inline _mul_cost((A,B)::Tuple) = _mul_cost(A,B)
@inline _mul_cost(A,B) = _mul_cost(A) + _mul_cost(B) + *(_mul_sizes(A)..., last(_mul_sizes(B)))
@inline _mul_sizes(A::AbstractMatrix) = size(A)
@inline _mul_sizes((A,B)::Tuple) = first(_mul_sizes(A)), last(_mul_sizes(B))


#####################################################################
#                                                                   #
# This file provides functions for interfacing with pti .raw files  #
#                                                                   #
#####################################################################
using DataStructures

"""
    get_pti_sections()

Returns `Array` of the names of the sections, in the order that they
appear in a PTI file, v33+
"""
function get_pti_sections()::Array
    return ["CASE IDENTIFICATION", "BUS", "LOAD", "FIXED SHUNT", "GENERATOR", "BRANCH", "TRANSFORMER",
            "AREA INTERCHANGE", "TWO-TERMINAL DC", "VOLTAGE SOURCE CONVERTER", "IMPEDANCE CORRECTION",
            "MULTI-TERMINAL DC", "MULTI-SECTION LINE", "ZONE", "INTER-AREA TRANSFER", "OWNER",
            "FACTS CONTROL DEVICE", "SWITCHED SHUNT", "GNE DEVICE", "INDUCTION MACHINE"]
end

"""
    get_pti_dtypes(field_name)

Returns `OrderedDict` of data types for PTI file section given by `field_name`,
as enumerated by PSS/E Program Operation Manual
"""
function get_pti_dtypes(field_name::AbstractString)::OrderedDict
    transaction_dtypes = OrderedDict{String,Type}("IC" => Int64, "SBASE" => Float64, "REV" => Int64,
                                                  "XFRRAT" => Float64, "NXFRAT" => Float64, "BASFRQ" => Float64)

    bus_dtypes = OrderedDict{String,Type}("I" => Int64, "NAME" => String, "BASKV" => Float64,
                                    "IDE" => Int64, "AREA" => Int64, "ZONE" => Int64, "OWNER" => Int64,
                                    "VM" => Float64, "VA" => Float64,
                                    "NVHI" => Float64, "NVLO" => Float64,
                                    "EVHI" => Float64, "EVLO" => Float64)

    load_dtypes = OrderedDict{String,Type}("I" => Int64, "ID" => String, "STATUS" => Int64, "AREA" => Int64,
                                    "ZONE" => Int64, "PL" => Float64, "QL" => Float64, "IP" => Float64,
                                    "IQ" => Float64, "YP" => Float64, "YQ" => Float64, "OWNER" => Int64,
                                    "SCALE" => Int64, "INTRPT" => Int64)

    fixded_shunt_dtypes = OrderedDict{String,Type}("I" => Int64, "ID" => String, "STATUS" => Int64,
                                            "GL" => Float64, "BL" => Float64)

    generator_dtypes = OrderedDict{String,Type}("I" => Int64, "ID" => String, "PG" => Float64, "QG" => Float64,
                                         "QT" => Float64, "QB" => Float64, "VS" => Float64, "IREG" => Int64,
                                         "MBASE" => Float64, "ZR" => Float64, "ZX" => Float64, "RT" => Float64,
                                         "XT" => Float64, "GTAP" => Float64, "STAT" => Int64, "RMPCT" => Float64,
                                         "PT" => Float64, "PB" => Float64, "O1" => Int64, "F1" => Float64,
                                         "O2" => Int64, "F2" => Float64, "O3" => Int64, "F3" => Float64,
                                         "O4" => Int64, "F4" => Float64, "WMOD" => Int64, "WPF" => Float64)

    branch_dtypes = OrderedDict{String,Type}("I" => Int64, "J" => Int64, "CKT" => String, "R" => Float64, "X" => Float64,
                                      "B" => Float64, "RATEA" => Float64, "RATEB" => Float64, "RATEC" => Float64,
                                      "GI" => Float64, "BI" => Float64, "GJ" => Float64, "BJ" => Float64,
                                      "ST" => Int64, "MET" => Int64, "LEN" => Float64, "O1" => Int64, "F1" => Float64,
                                      "O2" => Int64, "F2" => Float64, "O3" => Int64, "F3" => Float64,
                                      "O4" => Int64, "F4" => Float64)

    transformer_3_dtypes = OrderedDict{String,Type}("I" => Int64, "J" => Int64, "K" => Int64, "CKT" => String,
                                           "CW" => Int64, "CZ" => Int64, "CM" => Int64, "MAG1" => Float64,
                                           "MAG2" => Float64, "NMETR" => Int64, "NAME" => String,
                                           "STAT" => Int64,
                                           "O1" => Int64, "F1" => Float64,
                                           "O2" => Int64, "F2" => Float64,
                                           "O3" => Int64, "F3" => Float64,
                                           "O4" => Int64,  "F4" => Float64,
                                           "VECGRP" => String,

                                           "R1-2" => Float64, "X1-2" => Float64, "SBASE1-2" => Float64,
                                           "R2-3" => Float64, "X2-3" => Float64, "SBASE2-3" => Float64,
                                           "R3-1" => Float64, "X3-1" => Float64, "SBASE3-1" => Float64,
                                           "VMSTAR" => Float64, "ANSTAR" => Float64,

                                           "WINDV1" => Float64, "NOMV1" => Float64, "ANG1" => Float64,
                                           "RATA1" => Float64, "RATB1" => Float64, "RATC1" => Float64,
                                           "COD1" => Int64, "CONT1" => Int64, "RMA1" => Float64, "RMI1" => Float64,
                                           "VMA1" => Float64, "VMI1" => Float64, "NTP1" => Float64, "TAB1" => Int64,
                                           "CR1" => Float64, "CX1" => Float64, "CNXA1" => Float64,

                                           "WINDV2" => Float64, "NOMV2" => Float64, "ANG2" => Float64,
                                           "RATA2" => Float64, "RATB2" => Float64, "RATC2" => Float64,
                                           "COD2" => Int64, "CONT2" => Int64, "RMA2" => Float64, "RMI2" => Float64,
                                           "VMA2" => Float64, "VMI2" => Float64, "NTP2" => Float64, "TAB2" => Int64,
                                           "CR2" => Float64, "CX2" => Float64, "CNXA2" => Float64,

                                           "WINDV3" => Float64, "NOMV3" => Float64, "ANG3" => Float64,
                                           "RATA3" => Float64, "RATB3" => Float64, "RATC3" => Float64,
                                           "COD3" => Int64, "CONT3" => Int64, "RMA3" => Float64, "RMI3" => Float64,
                                           "VMA3" => Float64, "VMI3" => Float64, "NTP3" => Float64, "TAB3" => Int64,
                                           "CR3" => Float64, "CX3" => Float64, "CNXA3" => Float64)

    transformer_2_dtypes = OrderedDict{String,Type}("I" => Int64, "J" => Int64, "K" => Int64, "CKT" => String,
                                           "CW" => Int64, "CZ" => Int64, "CM" => Int64, "MAG1" => Float64,
                                           "MAG2" => Float64, "NMETR" => Int64, "NAME" => String,
                                           "STAT" => Int64,
                                           "O1" => Int64, "F1" => Float64,
                                           "O2" => Int64, "F2" => Float64,
                                           "O3" => Int64, "F3" => Float64,
                                           "O4" => Int64,  "F4" => Float64,
                                           "VECGRP" => String,

                                           "R1-2" => Float64, "X1-2" => Float64, "SBASE1-2" => Float64,

                                           "WINDV1" => Float64, "NOMV1" => Float64, "ANG1" => Float64,
                                           "RATA1" => Float64, "RATB1" => Float64, "RATC1" => Float64,
                                           "COD1" => Int64, "CONT1" => Int64, "RMA1" => Float64, "RMI1" => Float64,
                                           "VMA1" => Float64, "VMI1" => Float64, "NTP1" => Float64, "TAB1" => Int64,
                                           "CR1" => Float64, "CX1" => Float64, "CNXA1" => Float64,

                                           "WINDV2" => Float64, "NOMV2" => Float64)

    area_interchange_dtypes = OrderedDict{String,Type}("I" => Int64, "ISW" => Int64,
                                                       "PDES" => Float64, "PTOL" => Float64,
                                                       "ARNAME" => String)

    two_terminal_line_dtypes = OrderedDict{String,Type}("NAME" => String, "MDC" => Int64, "RDC" => Float64,
                                                 "SETVL" => Float64, "VSCHD" => Float64, "VCMOD" => Float64,
                                                 "RCOMP" => Float64, "DELTI" => Float64, "METER" => String,
                                                 "DCVMIN" => Float64, "CCCITMX" => Int64, "CCCACC" => Float64,
                                                 "IPR" => Int64, "NBR" => Int64, "ANMXR" => Float64,
                                                 "ANMNR" => Float64, "RCR" => Float64, "XCR" => Float64,
                                                 "EBASR" => Float64, "TRR" => Float64, "TAPR" => Float64,
                                                 "TMXR" => Float64, "TMNR" => Float64, "STPR" => Float64,
                                                 "ICR" => Int64, "IFR" => Int64, "ITR" => Int64, "IDR" => String,
                                                 "XCAPR" => Float64, "IPI" => Int64, "NBI" => Int64,
                                                 "ANMXI" => Float64, "ANMNI" => Float64, "RCI" => Float64,
                                                 "XCI" => Float64, "EBASI" => Float64, "TRI" => Float64,
                                                 "TAPI" => Float64, "TMXI" => Float64, "TMNI" => Float64,
                                                 "STPI" => Float64, "ICI" => Int64, "IFI" => Int64,
                                                 "ITI" => Int64, "IDI" => String, "XCAPI" => Float64)

    vsc_line_dtypes = OrderedDict{String,Type}("NAME" => String, "MDC" => Int64, "RDC" => Float64,
                                               "O1" => Int64, "F1" => Float64,
                                               "O2" => Int64, "F2" => Float64,
                                               "O3" => Int64, "F3" => Float64,
                                               "O4" => Int64, "F4" => Float64)

    vsc_subline_dtypes = OrderedDict{String,Type}("IBUS" => Int64, "TYPE" => Int64, "MODE" => Int64,
                                                  "DCSET" => Float64, "ACSET" => Float64,
                                                  "ALOSS" => Float64, "BLOSS" => Float64, "MINLOSS" => Float64,
                                                  "SMAX" => Float64, "IMAX" => Float64, "PWF" => Float64,
                                                  "MAXQ" => Float64, "MINQ" => Float64,
                                                  "REMOT" => Int64, "RMPCT" => Float64)

    impedance_correction_dtypes = OrderedDict{String,Type}("I" => Int64, "T1" => Float64, "F1" => Float64,
                                                    "T2" => Float64, "F2" => Float64, "T3" => Float64, "F3" => Float64,
                                                    "T4" => Float64, "F4" => Float64, "T5" => Float64, "F5" => Float64,
                                                    "T6" => Float64, "F6" => Float64, "T7" => Float64, "F7" => Float64,
                                                    "T8" => Float64, "F8" => Float64, "T9" => Float64, "F9" => Float64,
                                                    "T10" => Float64, "F10" => Float64, "T11" => Float64, "F11" => Float64)

    multi_term_main_dtypes = OrderedDict{String,Type}("NAME" => String, "NCONV" => Int64, "NDCBS" => Int64, "NDCLN" => Int64,
                                              "MDC" => Int64, "VCONV" => Int64, "VCMOD" => Float64, "VCONVN" => Float64)

    multi_term_nconv_dtypes = OrderedDict{String,Type}("IB" => Int64, "N" => Int64, "ANGMX" => Float64, "ANGMN" => Float64,
                                              "RC" => Float64, "XC" => Float64, "EBAS" => Float64, "TR" => Float64,
                                              "TAP" => Float64, "TPMX" => Float64, "TPMN" => Float64, "TSTP" => Float64,
                                              "SETVL" => Float64, "DCPF" => Float64, "MARG" => Float64, "CNVCOD" => Int64)

    multi_term_ndcbs_dtypes = OrderedDict{String,Type}("IDC" => Int64, "IB" => Int64, "AREA" => Int64, "ZONE" => Int64,
                                              "DCNAME" => String, "IDC2" => Int64, "RGRND" => Float64, "OWNER" => Int64)

    multi_term_ndcln_dtypes = OrderedDict{String,Type}("IDC" => Int64, "JDC" => Int64, "DCCKT" => String, "MET" => Int64,
                                              "RDC" => Float64, "LDC" => Float64)

    multi_section_dtypes = OrderedDict{String,Type}("I" => Int64, "J" => Int64, "ID" => String, "MET" => Int64, "DUM1" => Int64,
                                             "DUM2" => Int64, "DUM3" => Int64, "DUM4" => Int64, "DUM5" => Int64,
                                             "DUM6" => Int64, "DUM7" => Int64, "DUM8" => Int64, "DUM9" => Int64)

    zone_dtypes = OrderedDict{String,Type}("I" => Int64, "ZONAME" => String)

    interarea_dtypes = OrderedDict{String,Type}("ARFROM" => Int64, "ARTO" => Int64, "TRID" => String, "PTRAN" => Float64)

    owner_dtypes = OrderedDict{String,Type}("I" => Int64, "OWNAME" => String)

    FACTS_dtypes = OrderedDict{String,Type}("NAME" => String, "I" => Int64, "J" => Int64, "MODE" => Int64, "PDES" => Float64,
                                     "QDES" => Float64, "VSET" => Float64, "SHMX" => Float64, "TRMX" => Float64,
                                     "VTMN" => Float64, "VTMX" => Float64, "VSMX" => Float64, "IMX" => Float64,
                                     "LINX" => Float64, "RMPCT" => Float64, "OWNER" => Int64, "SET1" => Float64,
                                     "SET2" => Float64, "VSREF" => Int64, "REMOT" => Int64, "MNAME" => String)

    switched_shunt_dtypes = OrderedDict{String,Type}("I" => Int64, "MODSW" => Int64, "ADJM" => Int64, "STAT" => Int64,
                                                     "VSWHI" => Float64, "VSWLO" => Float64, "SWREM" => Int64,
                                                     "RMPCT" => Float64, "RMIDNT" => String, "BINIT" => Float64,
                                                     "N1" => Int64, "B1" => Float64, "N2" => Int64, "B2" => Float64,
                                                     "N3" => Int64, "B3" => Float64, "N4" => Int64, "B4" => Float64,
                                                     "N5" => Int64, "B5" => Float64, "N6" => Int64, "B6" => Float64,
                                                     "N7" => Int64, "B7" => Float64, "N8" => Int64, "B8" => Float64)

    # TODO: Account for multiple lines in GNE Device entries
    gne_device_dtypes = OrderedDict{String,Type}("NAME" => String, "MODEL" => String, "NTERM" => Int64, "BUSi" => Int64,
                                                 "NREAL" => Int64, "NINTG" => Int64, "NCHAR" => Int64,

                                                 "STATUS" => Int64, "OWNER" => Int64, "NMETR" => Int64,

                                                 "REALi" => Float64,
                                                 "INTGi" => Int64,
                                                 "CHARi" => String)

    induction_machine_dtypes = OrderedDict{String,Type}("I" => Int64, "ID" => String, "STAT" => Int64, "SCODE" => Int64,
                                                        "DCODE" => Int64, "AREA" => Int64, "ZONE" => Int64, "OWNER" => Int64,
                                                        "TCODE" => Int64, "BCODE" => Int64, "MBASE" => Float64,
                                                        "RATEKV" => Float64, "PCODE" => Int64, "PSET" => Float64,
                                                        "H" => Float64, "A" => Float64, "B" => Float64, "D" => Float64,
                                                        "E" => Float64, "RA" => Float64, "XA" => Float64, "XM" => Float64,
                                                        "R1" => Float64, "X1" => Float64, "R2" => Float64, "X2" => Float64,
                                                        "X3" => Float64, "E1" => Float64, "SE1" => Float64, "E2" => Float64,
                                                        "SE2" => Float64, "IA1" => Float64, "IA2" => Float64, "XAMULT" => Float64)

    dtypes = Dict{String,OrderedDict}("BUS" => bus_dtypes,
                                      "LOAD" => load_dtypes,
                                      "FIXED SHUNT" => fixded_shunt_dtypes,
                                      "GENERATOR" => generator_dtypes,
                                      "BRANCH" => branch_dtypes,
                                      "TRANSFORMER TWO WINDING" => transformer_2_dtypes,
                                      "TRANSFORMER THREE WINDING" => transformer_3_dtypes,
                                      "AREA INTERCHANGE" => area_interchange_dtypes,
                                      "TWO-TERMINAL DC" => two_terminal_line_dtypes,
                                      "VOLTAGE SOURCE CONVERTER" => vsc_line_dtypes,
                                      "VOLTAGE SOURCE CONVERTER SUBLINES" => vsc_subline_dtypes,
                                      "IMPEDANCE CORRECTION" => impedance_correction_dtypes,
                                      "MULTI-TERMINAL DC" => multi_term_main_dtypes,
                                      "MULTI-TERMINAL DC NCONV" => multi_term_nconv_dtypes,
                                      "MULTI-TERMINAL DC NDCBS" => multi_term_ndcbs_dtypes,
                                      "MULTI-TERMINAL DC NDCLN" => multi_term_ndcln_dtypes,
                                      "MULTI-SELECTION" => multi_section_dtypes,
                                      "ZONE" => zone_dtypes,
                                      "INTER-AREA TRANSFER" => interarea_dtypes,
                                      "OWNER" => owner_dtypes,
                                      "FACTS CONTROL DEVICE" => FACTS_dtypes,
                                      "SWITCHED SHUNT" => switched_shunt_dtypes,
                                      "CASE IDENTIFICATION" => transaction_dtypes,
                                      "GNE DEVICE" => gne_device_dtypes,
                                      "INDUCTION MACHINE" => induction_machine_dtypes)

    return dtypes[field_name]
end

"""
    parse_line_element!(data, elements, section)

Parses a single "line" of data elements from a PTI file, as given by `elements`
which is an array of the line, typically split at `,`. Elements are parsed into
data types given by `section` and saved into `data::Dict`

"""
function parse_line_element!(data::Dict, elements::Array, section::AbstractString)
    for (field, dtype) in get_pti_dtypes(section)
        try
            element = shift!(elements)
        catch message
            if isa(message, ArgumentError)
                debug(LOGGER, "Have run out of elements in $section at $field")
                break
            end
        end

        if endswith(element, '\r')
            element = element[1:end-1]
        end

        if startswith(element, "'") && endswith(element, "'")
            dtype = String
            element = element[2:end - 1]
        end

        try
            if dtype != String
                data[field] = parse(dtype, element)
            else
                data[field] = element
            end

        catch message
            if isa(message, ParseError)
                data[field] = element
            else
                debug(LOGGER, "$section $field $dtype $element")
                error(LOGGER, message)
            end
        end
    end
end

"""
    add_section_data!(pti_data, section_data, section)

Adds `section_data::Dict`, which contains all parsed elements of a PTI file
section given by `section`, into the parent `pti_data::Dict`
"""
function add_section_data!(pti_data::Dict, section_data::Dict, section::AbstractString)
    try
        pti_data[section] = append!(pti_data[section], [deepcopy(section_data)])
    catch message
        if isa(message, KeyError)
            pti_data[section] = [deepcopy(section_data)]
        else
            error(LOGGER, message)
        end
    end
end

"""
    get_line_elements(line)

Uses regular expressions to extract all separate data elements from a line of
a PTI file and populate them into an `Array{String}`. Comments, typically
indicated at the end of a line with a `'/'` character, are also extracted
separately, and `Array{Array{String}, String}` is returned.
"""
function get_line_elements(line::AbstractString)::Array
    match_string = r"(-*\d*\.*\d+[eE]*-*\d*)|(\'[^\']*?\')|(\"[^\"]*?\")|(\w+)|\,(\s+)?\,|(\/.*)"
    matches = matchall(match_string, line)

    debug(LOGGER, "$line")
    debug(LOGGER, "$matches")

    elements = []
    comment = ""
    for item in matches
        if startswith(item, ',') && endswith(item, ',') && length(item) == 2
            elements = append!(elements, [""])
        elseif startswith(item, '/')
            comment = item
        else
            elements = append!(elements, [item])
        end
    end

    return [elements, comment]
end

"""
    parse_pti_data(data_string, sections)

Parse a PTI raw file into a `Dict`, given the `data_string` of the file and a
list of the `sections` in the PTI file (typically given by default by
`get_pti_sections()`.
"""
function parse_pti_data(data_string::String, sections::Array)
    data_lines = split(data_string, '\n')
    skip_lines = 0
    skip_sublines = 0
    subsection = ""

    pti_data = Dict{String,Array{Dict}}()

    section = shift!(sections)
    section_data = Dict{String,Any}()

    for (line_number, line) in enumerate(data_lines)
        debug(LOGGER, "$line_number: $line")

        (elements, comment) = get_line_elements(line)

        if length(elements) != 0 && elements[1] == "Q"
            break

        elseif length(elements) != 0 && elements[1] == "0" && line_number != 1
            if line_number == 4
                section = shift!(sections)
            end

            match_string = r"\s*END OF ([\w\s-]+) DATA(?:, BEGIN ([\w\s-]+) DATA)?"
            debug(LOGGER, "$comment")
            matches = match(match_string, comment)

            if !isa(matches, Void)
                guess_section = matches.captures[1]
            else
                guess_section = ""
            end

            if guess_section == section
                section = shift!(sections)
                continue
            else
                info(LOGGER, "At line $line_number, unexpected section: expected: $section, comment specified: $(guess_section)")
                section = shift!(sections)
                continue
            end
        else
            if line_number == 4
                section = shift!(sections)
                section_data = Dict{String,Any}()
            end

            if skip_lines > 0
                skip_lines -= 1
                continue
            end

            debug(LOGGER, join(["Section:", section], " "))
            if section ∉ ["CASE IDENTIFICATION","TRANSFORMER","VOLTAGE SOURCE CONVERTER","MULTI-TERMINAL DC","TWO-TERMINAL DC","GNE DEVICE"]
                section_data = Dict{String,Any}()
                parse_line_element!(section_data, elements, section)

            elseif section == "CASE IDENTIFICATION"
                if line_number == 1
                    parse_line_element!(section_data, elements, section)
                    try
                        if section_data["REV"] < 33
                            warn(LOGGER, "Version $(section_data["REV"]) of PTI format is unsupported, parser may not function correctly.")
                        end
                    catch message
                        if isa(message, KeyError)
                            error(LOGGER, "This file is unrecognized and cannot be parsed")
                        end
                    end
                else
                    section_data["Comment_Line_$(line_number - 1)"] = line
                end

                if line_number < 3
                    continue
                end

            elseif section == "TRANSFORMER"
                section_data = Dict{String,Any}()
                if length(split(data_lines[line_number + 1], ',')) == 3  && parse(Int64, split(line, ',')[3]) == 0 # two winding transformer
                    temp_section = "TRANSFORMER TWO WINDING"
                    elements = split(join(data_lines[line_number:line_number + 3], ','), ',')
                    skip_lines = 3
                elseif length(split(data_lines[line_number + 1], ',')) == 11 && parse(Int64, split(line, ',')[3]) != 0 # three winding transformer
                    temp_section = "TRANSFORMER THREE WINDING"
                    elements = split(join(data_lines[line_number:line_number + 4], ','), ',')
                    skip_lines = 4
                else
                    error(LOGGER, "Cannot detect type of Transformer")
                end

                parse_line_element!(section_data, elements, temp_section)

            elseif section == "VOLTAGE SOURCE CONVERTER"
                if length(split(line, ',')) == 11
                    section_data = Dict{String,Any}()
                    parse_line_element!(section_data, elements, section)
                    skip_sublines = 2
                    continue

                elseif skip_sublines > 0
                    skip_sublines -= 1

                    (elements, comment) = get_line_elements(line)
                    subsection_data = Dict{String,Any}()

                    for (field, dtype) in get_pti_dtypes("$section SUBLINES")
                        element = shift!(elements)
                        subsection_data[field] = parse(dtype, element)
                    end

                    try
                        section_data["CONVERTER BUSES"] = append!(section_data["CONVERTER BUSES"], [deepcopy(subsection_data)])
                    catch message
                        if isa(message, KeyError)
                            section_data["CONVERTER BUSES"] = [deepcopy(subsection_data)]
                            continue
                        else
                            error(LOGGER, message)
                        end
                    end
                end

            elseif section == "TWO-TERMINAL DC"
                section_data = Dict{String,Any}()
                if length(split(line, ',')) == 12
                    elements = split(join(data_lines[line_number:line_number + 2], ','), ',')
                    skip_lines = 2
                end

                parse_line_element!(section_data, elements, section)

            elseif section == "MULTI-TERMINAL DC"
                if skip_sublines == 0
                    section_data = Dict{String,Any}()
                    parse_line_element!(section_data, elements, section)

                    if section_data["NCONV"] > 0
                        skip_sublines = section_data["NCONV"]
                        subsection = "NCONV"
                        continue
                    elseif section_data["NDCBS"] > 0
                        skip_sublines = section_data["NDCBS"]
                        subsection = "NDCBS"
                        continue
                    elseif section_data["NDCLN"] > 0
                        skip_sublines = section_data["NDCLN"]
                        subsection = "NDCLN"
                        continue
                    end
                end

                if skip_sublines > 0
                    skip_sublines -= 1

                    subsection_data = Dict{String,Any}()

                    for (field, dtype) in get_pti_dtypes("$section $subsection")
                        element = shift!(elements)
                        if startswith(element, "'") && endswith(element, "'")
                            subsection_data[field] = element[2:end-1]
                        else
                            subsection_data[field] = parse(dtype, element)
                        end
                    end

                    try
                        section_data["$(subsection[2:end])"] = append!(section_data["$(subsection[2:end])"], [deepcopy(subsection_data)])
                        if skip_sublines > 0 && subsection != "NDCLN"
                            continue
                        end
                    catch message
                        if isa(message, KeyError)
                            section_data["$(subsection[2:end])"] = [deepcopy(subsection_data)]
                            if skip_sublines > 0 && subsection != "NDCLN"
                                continue
                            end
                        else
                            error(LOGGER, message)
                        end
                    end

                    if skip_sublines == 0 && subsection != "NDCLN"
                        if subsection == "NDCBS"
                            skip_sublines = section_data["NDCLN"]
                            subsection = "NDCLN"
                            continue
                        elseif subsection == "NCONV"
                            skip_sublines = section_data["NDCBS"]
                            subsection = "NDCBS"
                            continue
                        end
                    elseif skip_sublines == 0 && subsection == "NDCLN"
                        subsection = ""
                    else
                        continue
                    end
                end

            elseif section == "GNE DEVICE"
                # TODO: handle multiple lines of GNE Device
                warn(LOGGER, "GNE DEVICE parsing is not supported.")
            end
        end
        if subsection != ""
            debug(LOGGER, "appending data")
        end
        add_section_data!(pti_data, section_data, section)
    end

    return pti_data
end

"""
    parse_pti(filename)

Open PTI raw file given by `filename`, passing the file contents as a string
to the main PTI parser, returning a `Dict` of all the data parsed into the
proper types.
"""
function parse_pti(filename::String)::Dict
    data_string = readstring(open(filename))
    pti_data = parse_pti_data(data_string, get_pti_sections())

    return pti_data
end


module TORA

# Workaround for: https://github.com/JuliaRobotics/RigidBodyDynamics.jl/issues/500
using LinearAlgebra; BLAS.set_num_threads(1)

using Colors
using ForwardDiff
using GeometryBasics
using Ipopt
using MeshCat
using MeshCatMechanisms
using NPZ
using Plots
using Requires
using RigidBodyDynamics
using SparseArrays
using SparseDiffTools

using Plots.PlotMeasures: px
using Random: rand!

import Base: length

"""
    solve_with_knitro(problem, robot;
                      initial_guess=Float64[],
                      use_inv_dyn=false,
                      minimise_τ=false,
                      user_options=Dict())

Solve the nonlinear optimization problem with Knitro.

Further options can be set using the keyword arguments. See [Solver Interfaces](@ref).

# Keyword arguments
- `initial_guess::Array{Float64}=Float64[]`: the starting point for the solver.
- `use_inv_dyn::Bool=false`: if true, enables the use of inverse dynamics instead of forward dynamics.
- `minimise_τ::Bool=false`: if true, activates a cost function to minimize the joint torques.
- `user_options::Dict=Dict()`: the user options for Knitro.

See also: [`solve_with_ipopt`](@ref)
"""
function solve_with_knitro end

function __init__()
    @require KNITRO = "67920dd8-b58e-52a8-8622-53c4cffbe346" include("./transcription/knitro.jl")
end

greet() = print("Hello World!")

# `Struct`-defining files
include("./jacobian_data.jl")
include("./robot.jl")
include("./problem.jl")
include("./solverlog.jl")

# Regular source code
include("./constraints/dynamics.jl")
include("./constraints/end_effector.jl")
include("./transcription/ipopt.jl")
include("./plots.jl")

export solve_with_knitro

end # module


    @with_kw struct GPSData
        integrity_status_flag::Union{Nothing,Bool} = nothing
        TOW::Union{Nothing,Int64} = nothing
        alert_flag::Union{Nothing,Bool} = nothing
        anti_spoof_flag::Union{Nothing,Bool} = nothing

        trans_week::Union{Nothing,Int64} = nothing
        codeonl2::Union{Nothing,Int64} = nothing
        ura::Union{Nothing,Float64} = nothing
        svhealth::Union{Nothing,String} = nothing
        IODC::Union{Nothing,String} = nothing
        l2pcode::Union{Nothing,Bool} = nothing
        T_GD::Union{Nothing,Float64} = nothing
        t_oc::Union{Nothing,Int64} = nothing
        a_f2::Union{Nothing,Float64} = nothing
        a_f1::Union{Nothing,Float64} = nothing
        a_f0::Union{Nothing,Float64} = nothing

        IODE_Sub_2::Union{Nothing,String} = nothing
        C_rs::Union{Nothing,Float64} = nothing
        Δn::Union{Nothing,Float64} = nothing
        M_0::Union{Nothing,Float64} = nothing
        C_uc::Union{Nothing,Float64} = nothing
        e::Union{Nothing,Float64} = nothing
        C_us::Union{Nothing,Float64} = nothing
        sqrt_A::Union{Nothing,Float64} = nothing
        t_oe::Union{Nothing,Int64} = nothing
        fit_interval::Union{Nothing,Bool} = nothing
        AODO::Union{Nothing,Int64} = nothing

        C_ic::Union{Nothing,Float64} = nothing
        Ω_0::Union{Nothing,Float64} = nothing
        C_is::Union{Nothing,Float64} = nothing
        i_0::Union{Nothing,Float64} = nothing
        C_rc::Union{Nothing,Float64} = nothing
        ω::Union{Nothing,Float64} = nothing
        Ω_dot::Union{Nothing,Float64} = nothing
        IODE_Sub_3::Union{Nothing,String} = nothing
        IDOT::Union{Nothing,Float64} = nothing
    end

    @with_kw struct GPSL1Constants
        PI = 3.1415926535898
        Ω_dot_e = 7.2921151467e-5
        c = 2.99792458e8
        μ = 3.986005e14
        F =  -4.442807633e-10
    end

    @with_kw mutable struct GNSSDecoderState
        PRN::Int64
        buffer::BitArray{1} = falses(BUFFER_LENGTH)
        data::GPSData = GPSData()
        constants::GPSL1Constants = GPSL1Constants()
        preamble_found::Bool = false
        subframe_count::Int = 0
        prev_30::Bool = false
        prev_29::Bool = false
        data_integrity::Bool = true # pass of parity check
        new_data_needed::Bool = false # IODE must be equal to the 8 LSB of the IODC. If they don't match new data must be collected
        subframes_decoded::MVector{5,Bool} = MVector{5,Bool}(false, false, false, false, false)
        nb_prev::Int = 0
        num_bits_buffered::Int = 0
    end


# This file is a part of UltraNest.jl, licensed under the MIT License (MIT).

__precompile__(true)

"""
    UltraNest

Julia wrapper for Python nested sampling package
[UltraNest](https://github.com/JohannesBuchner/UltraNest).
"""
module UltraNest

using LinearAlgebra
using Random
using Statistics

using PyCall

"""
    const ultranest

The Python `ultranest` module.

Example:

```julia
using UltraNest
smplr = ultranest.ReactiveNestedSampler(paramnames, my_likelihood, kwargs...)
result = smplr.run()
```

See the the
[UltraNest Python documentation](https://johannesbuchner.github.io/UltraNest/)
regarding usage.

    !!! note

    Convention for matrices holding multiple parameter vectors (resp.
    multiple samples): In UltraNest.jl, using Julia's column-major array
    indexing, parameter vectors are stored as rows (not columns) in the
    matrices.
"""
const ultranest = PyNULL()
export ultranest

const scipy_stats = PyNULL()
const numpy = PyNULL()

include("ultra_nest.jl")

function __init__()
    copy!(ultranest, pyimport_conda("ultranest", "ultranest", "conda-forge"))
    copy!(scipy_stats, pyimport_conda("scipy.stats", "scipy"))
    copy!(numpy, pyimport("numpy"))
end

end # module


struct Events{T <: EpidemicModel}
  exposure::Union{Nothing, Vector{Float64}}
  infection::Vector{Float64}
  removal::Union{Nothing, Vector{Float64}}
  individuals::Int64

  function Events{T}(e::V, i::V, r::V) where {T <: SEIR, V <: Vector{Float64}}
    if length(unique((length.([e; i; r])))) != 1
      @error "Length of event time vectors must be equal"
    end
    return new{T}(e, i, r, length(i))
  end

  function Events{T}(n::Integer) where T <: SEIR
    return new{T}(fill(NaN, n), fill(NaN, n), fill(NaN, n), n)
  end

  function Events{T}(e::V, i::V) where {T <: SEI, V <: Vector{Float64}}
    if length(unique((length.([e; i])))) != 1
      @error "Length of event time vectors must be equal"
    end
    return new{T}(e, i, nothing, length(i))
  end

  function Events{T}(n::Integer) where T <: SEI
    return new{T}(fill(NaN, n), fill(NaN, n), nothing, n)
  end

  function Events{T}(i::V, r::V) where {T <: SIR, V <: Vector{Float64}}
    if length(unique((length.([i; r])))) != 1
      @error "Length of event time vectors must be equal"
    end
    return new{T}(nothing, i, r, length(i))
  end

  function Events{T}(n::Integer) where T <: SIR
    return new{T}(nothing, fill(NaN, n), fill(NaN, n), n)
  end

  function Events{T}(i::V) where {T <: SI, V <: Vector{Float64}}
    return new{T}(nothing, i, nothing, length(i))
  end

  function Events{T}(n::Integer) where T <: SI
    return new{T}(nothing, fill(NaN, n), nothing, n)
  end
end

function Events{T}(a::Array{Float64,2}) where T <: EpidemicModel
  if size(a, 2) == 3
    return Events{T}(a[:,1], a[:,2], a[:,3])
  elseif size(a, 2) == 2
    return Events{T}(a[:,1], a[:,2])
  elseif size(a, 2) == 1
    return Events{T}(a[:,1])
  else
    @error "Invalid array size for construction of an $(Events{T}) object"
  end
end

function Events{T}(x::Vector{DiseaseState}) where T <: EpidemicModel
  events = Events{T}(length(x))
  for i = 1:length(x)
    if x[i] in _state_progressions[T]
      for j = _state_progressions[T][2:findfirst(Ref(x[i]) .== _state_progressions[T])]
        events[j][i] = -Inf
      end
    else
      @error "Invalid initial state for individual $i"
    end
  end
  return events
end

function Base.show(io::IO, x::Events{T}) where T <: EpidemicModel
  return print(io, "$T model event times (n=$(x.individuals))")
end

function Base.getindex(x::Events{T}, new_state::DiseaseState) where T <: EpidemicModel
  if new_state == State_E
    return x.exposure
  elseif new_state == State_I
    return x.infection
  elseif new_state == State_R
    return x.removal
  else
    @error "Unrecognized indexing disease state"
  end
end

function Base.getindex(x::Events{T}, states::Vector{DiseaseState}) where T <: EpidemicModel
  y = x[states[1]]
  for i = 2:length(states)
    y = hcat(y, x[states[i]])
  end
  return y
end

function Base.convert(::Type{Array{Float64, 2}}, x::Events{T}) where T <: EpidemicModel
  return x[_state_progressions[T][2:end]]
end

function Base.convert(::Type{Vector{Float64}}, x::Events{T}) where T <: EpidemicModel
  return x[_state_progressions[T][2:end]][:]
end

function Base.convert(::Type{Array{Float64, 2}}, x::Array{Events{T}, 1}) where T <: EpidemicModel
  y = convert(Vector{Float64}, x[1])'
  for i = 2:length(x)
    y = vcat(y, convert(Vector{Float64}, x[i])')
  end
  return y
end

function Base.minimum(x::Events{T}) where T <: EpidemicModel
  y = convert(Array{Float64, 2}, x)
  return minimum(y[.!isnan.(y)])
end

function Base.maximum(x::Events{T}) where T <: EpidemicModel
  y = convert(Array{Float64, 2}, x)
  return maximum(y[.!isnan.(y)])
end

function Statistics.mean(x::Vector{Events{T}}) where T <: EpidemicModel
  return Events{T}(mean([convert(Array{Float64, 2}, i) for i in x]))
end


export generate_mesh

function generate_mesh(;dim::Int64 = 2,
                        order::Int64 = 1,
                        faces::String = "Triangle",
                        opt_iters::Int64 = 0,
                        force_quads::Bool = false)
    @info "Generating a mesh of dimension $dim with $faces faces of order $order"
    gmsh.option.set_number("Mesh.SecondOrderIncomplete", 1)
    if dim != 2
        error("Only 2D meshes are currently supported.")
    end
    if faces == "Triangle"
        # Delaunay (5) handles large element size gradients better
        gmsh.option.set_number("Mesh.Algorithm", 5)
        if order == 1
            gmsh.model.mesh.generate(2)
            for _ in 1:opt_iters
                gmsh.model.mesh.optimize("Laplace2D")
            end
        elseif order == 2
            gmsh.option.set_number("Mesh.HighOrderOptimize", 2) # elastic + optimization
            gmsh.model.mesh.generate(2)
            gmsh.model.mesh.set_order(2)
            for _ in 1:opt_iters
                gmsh.model.mesh.optimize("HighOrderElastic")
                gmsh.model.mesh.optimize("Relocate2D")
                gmsh.model.mesh.optimize("HighOrderElastic")
            end
        else
            error("Mesh order must be 1 or 2.")
        end
    elseif faces == "Quadrilateral"
        gmsh.option.set_number("Mesh.RecombineAll", 1)
        gmsh.option.set_number("Mesh.Algorithm", 8) # Frontal-Delaunay for quads.
        if force_quads
            gmsh.option.set_number("Mesh.RecombinationAlgorithm", 2) # simple full-quad
            gmsh.option.set_number("Mesh.SubdivisionAlgorithm", 1) # All quads
        end
        if order == 1
            gmsh.model.mesh.generate(2)
            for _ in 1:opt_iters
                gmsh.model.mesh.optimize("Laplace2D")
                gmsh.model.mesh.optimize("Relocate2D")
                gmsh.model.mesh.optimize("Laplace2D")
            end
        elseif order == 2
            gmsh.option.set_number("Mesh.HighOrderOptimize", 2)
            gmsh.model.mesh.generate(2)
            gmsh.model.mesh.set_order(2)
            for _ in 1:opt_iters
                gmsh.model.mesh.optimize("HighOrderElastic")
                gmsh.model.mesh.optimize("Relocate2D")
                gmsh.model.mesh.optimize("HighOrderElastic")
            end
        else
            error("Mesh order must be 1 or 2.")
        end
    else
        error("Could not identify face type.")
    end

    return nothing
end


@rule typeof(-)(:out, Marginalisation) (m_in1::Any, m_in2::Any, meta::Any) = begin
    @call_rule typeof(+)(:in1, Marginalisation) (m_out = m_in1, m_in2 = m_in2, meta = meta)
end


using PyCall
@pyimport scipy.linalg.interpolative
println("Julia loaded")

function main(rows, cols, values)
    println("In julia")
    A = sparse(rows, cols, values)
    println(interpolative.interp_decomp(A, 2))
    println("Exiting julia")
end


__precompile__()

module ImagineInterface

using JSON, Unitful, UnitAliases, Random, Mmap
using MappedArrays, IntervalSets, DataStructures, DSP
using ImagineFormat, ImagineHardware
import ImagineHardware:samprate
using AxisArrays
const axes = Base.axes

import Base: convert, show, length, size, isempty, ==, append!, pop!, empty!, replace!#, scale

using Unitful: μm, s, Hz, V

include("metadata_constants.jl")
#Load hardware parameters for all rigs
rig_dir = joinpath(dirname(@__DIR__), "rigs")
dictmap(f, d::T) where T = T((f(p) for p = pairs(d))) #used in rig files
for rig_file in readdir(rig_dir)
    include(joinpath(rig_dir, rig_file))
end
include("samplemapper.jl")
include("imaginesignal.jl")
include("hardware_templates.jl")
include("convenience.jl")
include("parse.jl")
include("sequence_analysis.jl")
include("validate_group.jl")
include("window_validate.jl")
include("validate_single.jl")
include("write.jl")
include("stack.jl")

#metadata_constants.jl
export chip_size,
        max_framerate,
        max_roi

#imaginecommand.jl
export RepeatedValue,
        RLEVector,
        full_length,
        ImagineSignal,
        name,
        rename!,
        duration,
        daq_channel,
        rig_name,
	rawtype,
        worldtype,
        samprate,
        intervals,
        interval_raw,
        interval_volts,
        interval_world,
        sequences,
	sequence_names,
	sequence_lookup,
        mapper,
        get_samples,
        add_sequence!,
        replace!,
        replicate!

#hardware_templates.jl
export rigtemplate

#convenience.jl
export  isfree,
        isdigital,
        isanalog,
        isoutput,
        isinput,
        ispos,
        isposmonitor,
        iscam,
        iscammonitor,
        islas,
        isstim,
        getname,
        getdigital,
        getanalog,
        getinputs,
        getoutputs,
        getfree,
        getfixed,
        getpositioners,
        getpositionermonitors,
        getcameras,
        getcameramonitors,
        getlasers,
        getstimuli,
        getgalvos,
        getgalvomonitors,
        hasmonitor,
        hasactuator,
        monitor_name,
        actuator_name,
        finddigital,
        findanalog,
        findinputs,
        findoutputs,
        findfree,
        findfixed,
        findpositioners,
        findcameras,
        findlasers,
        findstimuli

#parse.jl
export parse_command,
        parse_commands,
        parse_ai,
        parse_di,
        load_signals

#sequence_analysis.jl
export find_pulse_starts,
        find_pulse_stops,
        count_pulses

#validate_singles.jl
export validate_all

#write.jl
export write_commands

#stack.jl
export  apply_lp,
        gen_sweep,
        gen_sawtooth,
        gen_bidi_pos,
        spaced_intervals,
        gen_pulses,
        scale,
        gen_bidirectional_stack,
        gen_unidirectional_stack,
        gen_stepped_stack,
        gen_2d_timeseries
#deprecations
@deprecate ImagineCommand ImagineSignal
@deprecate decompress(args...;kwargs...) get_samples(args...;kwargs...)

end #module


include("lattices.jl")

mutable struct ElectronTinder
    """ Database to match lonely up spins with lonely down spins by site"""
    lonely_up::Array{Int}
    lonely_down::Array{Int}
end

function addUser!(t::ElectronTinder,site::Int,spin::Int)
    spin==1 ? append!(t.lonely_up,site) : append!(t.lonely_down,site)
end

function SwipeRight(database::ElectronTinder)
    """ Creates list of matches (moves) generated by
    electron tinder's database """
    @assert length(database.lonely_up) == length(database.lonely_down)
    lu = database.lonely_up
    ld = database.lonely_down
    nmatches = length(lu)
    matches = Array{ExchangeMove}(undef,nmatches)
    for i in 1:nmatches
        matches[i] = ExchangeMove((lu[i],ld[i]))
    end
    return matches
end

using Statistics: mean
using LightGraphs: neighborhood
function Swap(chain::DoubleGutzwillerChain,regionsize::Int)
    """ regionsize ≈ r, the radius of the region selected """

    states = get_State(chain) # (state1, state2)
    lg = get_Model(chain)["lattice"].graph
    ns = prod(get_Model(chain)["dims"])

    type = typeof(states[1].det_A_up)
    swap_per_site = zeros(type,ns)

    # redundancy for mixed BC lattice
    lg0 = SimpleGraph(adjacency_matrix(lg))

    for v = 1:ns
        sites = unique(neighborhood(lg0,v,regionsize))
        swap_per_site[v] = compute_swapregion(chain,sites)
    end

    return mean(swap_per_site)
end

function compute_swapregion(chain::DoubleGutzwillerChain,
                            sites::Array)
    """ This function is somewhat nontrivial, the following is the
    layout of how it works. First, a list of exchange moves is created
    for each replica that maps the current configuration to the new
    one after swapping. This happens in a two stages: first the spin
    spin configurations inside the regions are compared to see where they
    disagree. Every site where they disagree, an entry is added the
    appropriate spin list in Electron Tinder. Finally, the completed
    ElectronTinder database is fed into SwipeRight to generate a list of moves.

    After this, the list of moves are submitted to preexisting functions
    to compute the determinant ratio produced by that sequence of moves.

    Finally what is returned is the expectation value of the swap operator
    acting on this region which is,
    det(swapped copy 1)*det(swapped copy 2) / (det(og copy 1) * det(og copy 2))

    Later, I may modify this to be more general/support more than 2 copies,
    but for now it seems like thats more trouble than its worth.
    """
    s0 = get_State(chain)
    (spinc1, spinc2) = (s0[1].spin_config.sc,
                        s0[2].spin_config.sc)

    if sum(spinc1[sites])!=sum(spinc2[sites]) # regions must have same quantum #
        return 0
    end
    if spinc1[sites] == spinc2[sites]
        return 1
    end

    Tinder = (ElectronTinder([],[]),ElectronTinder([],[]))
    for site in sites
        if spinc1[site] != spinc2[site]
            addUser!(Tinder[1],site,spinc1[site])
            addUser!(Tinder[2],site,spinc2[site])
        end
    end

    (MoveList1, MoveList2) = SwipeRight.(Tinder)
    @assert size(MoveList1) == size(MoveList2)

    # now because update_chain! modifies the state, make copies of
    # the two states to update
    free_chains = deepcopy(get_Replicas(chain))
    free_states = get_State.(free_chains)

    @assert length(MoveList1) == length(MoveList2) "compute swapregion: move lists
    for two copies are of different length"

    # try to do "fast" determinant update with all moves simultaneously
    n_moves = length(MoveList1)
    num_particles = length(free_states[1].r_up) # not very pretty

    type = typeof(s0[1].det_A_up) # complexf64 or f64

    U_up_copy1 = zeros(type,(num_particles,n_moves))
    U_down_copy1 = zeros(type,(num_particles,n_moves))
    V_up_copy1 = zeros(type,(num_particles,n_moves))
    V_down_copy1 = zeros(type,(num_particles,n_moves))

    U_up_copy2 = zeros(type,(num_particles,n_moves))
    U_down_copy2 = zeros(type,(num_particles,n_moves))
    V_up_copy2 = zeros(type,(num_particles,n_moves))
    V_down_copy2 = zeros(type,(num_particles,n_moves))

    for i in 1:n_moves
        # update_Rs!(free_states[1],MoveList1[i])
        # update_Rs!(free_states[2],MoveList2[i])

        U_up_copy1[:,i], V_up_copy1[:,i], U_down_copy1[:,i], V_down_copy1[:,i] =
        get_update_vectors(free_states[1],MoveList1[i])

        U_up_copy2[:,i], V_up_copy2[:,i], U_down_copy2[:,i], V_down_copy2[:,i] =
        get_update_vectors(free_states[2],MoveList2[i])

    end

    # its important to do the determinant ratio updates all at once
    # brute force reevaluation of determinants code left in for debugging

    #
    # filled_states = get_Wavefunctions(free_chains[1])
    #
    # det1up = get_Determinant(free_states[1].r_up,filled_states)
    # det1down = get_Determinant(free_states[1].r_down,filled_states)
    # det2up = get_Determinant(free_states[2].r_up,filled_states)
    # det2down = get_Determinant(free_states[2].r_down,filled_states)

    det_ratio_1u = det_ratio_factor(free_states[1].A_inv_up,U_up_copy1,V_up_copy1)
    det_ratio_1d = det_ratio_factor(free_states[1].A_inv_down,U_down_copy1,V_down_copy1)
    det_ratio_2u = det_ratio_factor(free_states[2].A_inv_up,U_up_copy2,V_up_copy2)
    det_ratio_2d = det_ratio_factor(free_states[2].A_inv_down,U_down_copy2,V_down_copy2)

    fast_ratio = det_ratio_1u * det_ratio_1d * det_ratio_2u * det_ratio_2d

    # ss1 = get_test_state(free_chains[1],free_states[1].r_up, free_states[1].r_down)
    # ss2 = get_test_state(free_chains[2],free_states[2].r_up, free_states[2].r_down)

    # sdp = det1up * det1down * det2up * det2down
    # odp = s0[1].det_A_up * s0[1].det_A_down * s0[2].det_A_up * s0[2].det_A_down

    # print("sdp: ",sdp," odp: ",odp,"\n")

    return fast_ratio
end