"""
AU - Unrolled axis
F - Factor, step size per unroll
N - How many times is it unrolled
AV - Vectorized axis
W - vector width
M - bitmask indicating whether each factor is masked
i::I - index
"""
struct Unroll{AU,F,N,AV,W,M,I}
    i::I
end
@inline Unroll{AU,F,N,AV,W,M}(i::I) where {AU,F,N,AV,W,M,I} = Unroll{AU,F,N,AV,W,M,I}(i)

const VectorIndexCore{W} = Union{Vec{W},MM{W},Unroll{<:Any,<:Any,<:Any,<:Any,W}}
const VectorIndex{W} = Union{VectorIndexCore{W},LazyMulAdd{<:Any,<:Any,<:VectorIndexCore{W}}}

# const BoolVec = Union{Mask,VecUnroll{<:Any, <:Any, Bool, <: Mask}}

const SCOPE_METADATA = """
!1 = !{!\"noaliasdomain\"}
!2 = !{!\"noaliasscope\", !1}
!3 = !{!2}
"""
const LOAD_SCOPE_FLAGS = ", !alias.scope !3";
const STORE_SCOPE_FLAGS = ", !noalias !3";

const USE_TBAA = false
# use TBAA?
# define: LOAD_SCOPE_TBAA, LOAD_SCOPE_TBAA_FLAGS, SCOPE_METADATA, STORE_TBAA, SCOPE_FLAGS, STORE_TBAA_FLAGS
let
    LOAD_TBAA = """
    !4 = !{!"jtbaa"}
    !5 = !{!6, !6, i64 0, i64 0}
    !6 = !{!"jtbaa_arraybuf", !4, i64 0}
    """;
    LOAD_TBAA_FLAGS = ", !tbaa !5";
    global const LOAD_SCOPE_TBAA = USE_TBAA ? SCOPE_METADATA * LOAD_TBAA : SCOPE_METADATA;
    global const LOAD_SCOPE_TBAA_FLAGS = USE_TBAA ? LOAD_SCOPE_FLAGS * LOAD_TBAA_FLAGS : LOAD_SCOPE_FLAGS

    global const STORE_TBAA = USE_TBAA ? """
    !4 = !{!"jtbaa", !5, i64 0}
    !5 = !{!"jtbaa"}
    !6 = !{!"jtbaa_data", !4, i64 0}
    !7 = !{!8, !8, i64 0}
    !8 = !{!"jtbaa_arraybuf", !6, i64 0}
    """ : ""
    global const STORE_TBAA_FLAGS = USE_TBAA ? ", !tbaa !7" : ""
end

"""
An omnibus offset constructor.

The general motivation for generating the memory addresses as LLVM IR rather than combining multiple lllvmcall Julia functions is
that we want to minimize the `inttoptr` and `ptrtoint` calculations as we go back and fourth. These can get in the way of some
optimizations, such as memory address calculations.
It is particulary import for `gather` and `scatter`s, as these functions take a `Vec{W,Ptr{T}}` argument to load/store a
`Vec{W,T}` to/from. If `sizeof(T) < sizeof(Int)`, converting the `<W x \$(typ)*` vectors of pointers in LLVM to integer
vectors as they're represented in Julia will likely make them too large to fit in a single register, splitting the operation
into multiple operations, forcing a corresponding split of the `Vec{W,T}` vector as well.
This would all be avoided by not promoting/widenting the `<W x \$(typ)>` into a vector of `Int`s.

For this last issue, an alternate workaround would be to wrap a `Vec` of 32-bit integers with a type that defines it as a pointer for use with
internal llvmcall functions, but I haven't really explored this optimization.
"""
function offset_ptr(
    ::Type{T}, ind_type::Symbol, indargname, ibits::Int, W::Int, X::Int, M::Int, O::Int, forgep::Bool
) where {T}
    T_sym = JULIA_TYPES[T]
    offset_ptr(T_sym, sizeof_T, ind_type, indargname, ibits, W, X, M, O, forgep)
end
function offset_ptr(
    T_sym::Symbol, ind_type::Symbol, indargname, ibits::Int, W::Int, X::Int, M::Int, O::Int, forgep::Bool
)
    sizeof_T = JULIA_TYPE_SIZE[T_sym]
    i = 0
    Morig = M
    isbit = T_sym === :Bit
    if isbit
        typ = "i1"
        vtyp = isone(W) ? typ : "<$W x i1>"
        M = max(1, M >> 3)
        O >>= 3
        @assert ((isone(X) | iszero(X)) && (ind_type !== :Vec)) "indexing BitArrays with a vector not currently supported."
    else
        typ = LLVM_TYPES_SYM[T_sym]
        vtyp = vtype(W, typ) # vtyp is dest type
    end
    instrs = String[]
    if M == 0
        ind_type = :StaticInt
    elseif ind_type === :StaticInt
        M = 0
    end
    if isone(W)
        X = 1
    end
    if iszero(M)
        tz = intlog2(sizeof_T)
        tzf = sizeof_T
        index_gep_typ = typ
    else
        tz = min(trailing_zeros(M), 3)
        tzf = 1 << tz
        index_gep_typ = ((tzf == sizeof_T) | iszero(M)) ? typ : "i$(tzf << 3)"
        M >>= tz
    end
    # after this block, we will have a index_gep_typ pointer
    if iszero(O)
        push!(instrs, "%ptr.$(i) = inttoptr $(JULIAPOINTERTYPE) %0 to $(index_gep_typ)*"); i += 1
    else # !iszero(O)
        if !iszero(O & (tzf - 1)) # then index_gep_typ works for the constant offset
            offset_gep_typ = "i8"
            offset = O
        else # then we need another intermediary
            offset_gep_typ = index_gep_typ
            offset = O >>> tz
        end
        push!(instrs, "%ptr.$(i) = inttoptr $(JULIAPOINTERTYPE) %0 to $(offset_gep_typ)*"); i += 1
        push!(instrs, "%ptr.$(i) = getelementptr inbounds $(offset_gep_typ), $(offset_gep_typ)* %ptr.$(i-1), i32 $(offset)"); i += 1
        if forgep && iszero(M) && (iszero(X) || isone(X))
            push!(instrs, "%ptr.$(i) = ptrtoint $(offset_gep_typ)* %ptr.$(i-1) to $(JULIAPOINTERTYPE)"); i += 1
            return instrs, i
        elseif offset_gep_typ != index_gep_typ
            push!(instrs, "%ptr.$(i) = bitcast $(offset_gep_typ)* %ptr.$(i-1) to $(index_gep_typ)*"); i += 1
        end
    end
    # will do final type conversion
    if ind_type === :Vec
        if isone(M)
            indname = indargname
        else
            indname = "indname"
            constmul = llvmconst(W, "i$(ibits) $M")
            push!(instrs, "%$(indname) = mul nsw <$W x i$(ibits)> %$(indargname), $(constmul)")
        end
        push!(instrs, "%ptr.$(i) = getelementptr inbounds $(index_gep_typ), $(index_gep_typ)* %ptr.$(i-1), <$W x i$(ibits)> %$(indname)"); i += 1
        if forgep
            push!(instrs, "%ptr.$(i) = ptrtoint <$W x $index_gep_typ*> %ptr.$(i-1) to <$W x $JULIAPOINTERTYPE>"); i += 1
        elseif index_gep_typ != vtyp
            push!(instrs, "%ptr.$(i) = bitcast <$W x $index_gep_typ*> %ptr.$(i-1) to <$W x $typ*>"); i += 1
        end
        return instrs, i
    end
    if ind_type === :Integer
        if isbit
            scale = 8
            if ispow2(Morig)
                if abs(Morig) ≥ 8
                    M = Morig >> 3
                    if M != 1
                        indname = "indname"
                        push!(instrs, "%$(indname) = mul nsw i$(ibits) %$(indargname), $M")
                    else
                        indname = indargname
                    end
                else
                    shifter = 3 - intlog2(Morig)
                    push!(instrs, "%shiftedind = ashr i$(ibits) %$(indargname), $shifter")
                    if Morig > 0
                        indname = "shiftedind"
                    else
                        indname = "indname"
                        push!(instrs, "%$(indname) = mul i$(ibits) %shiftedind, -1")
                    end
                end
            else
                @assert iszero(Morig) "Scale factors on bit accesses must be 0 or a power of 2."
                indname = "0"
            end
        else
            if isone(M)
                indname = indargname
            elseif iszero(M)
                indname = "0"
            else
                indname = "indname"
                push!(instrs, "%$(indname) = mul nsw i$(ibits) %$(indargname), $M")
            end
            # TODO: if X != 1 and X != 0, check if it is better to gep -> gep, or broadcast -> add -> gep
        end
        push!(instrs, "%ptr.$(i) = getelementptr inbounds $(index_gep_typ), $(index_gep_typ)* %ptr.$(i-1), i$(ibits) %$(indname)"); i += 1
    end
    # ind_type === :Integer || ind_type === :StaticInt
    if !(isone(X) | iszero(X)) # vec
        vibytes = min(4, REGISTER_SIZE ÷ W)
        vityp = "i$(8vibytes)"
        vi = join((X*w for w ∈ 0:W-1), ", $vityp ")
        if typ !== index_gep_typ
            push!(instrs, "%ptr.$(i) = bitcast $(index_gep_typ)* %ptr.$(i-1) to $(typ)*"); i += 1
        end
        push!(instrs, "%ptr.$(i) = getelementptr inbounds $(typ), $(typ)* %ptr.$(i-1), <$W x $(vityp)> <$vityp $vi>"); i += 1
        if forgep
            push!(instrs, "%ptr.$(i) = ptrtoint <$W x $typ*> %ptr.$(i-1) to <$W x $JULIAPOINTERTYPE>"); i += 1
        end
        return instrs, i
    end
    if forgep # if forgep, just return now
        push!(instrs, "%ptr.$(i) = ptrtoint $(index_gep_typ)* %ptr.$(i-1) to $JULIAPOINTERTYPE"); i += 1
    elseif index_gep_typ != vtyp
        push!(instrs, "%ptr.$(i) = bitcast $(index_gep_typ)* %ptr.$(i-1) to $(vtyp)*"); i += 1
    end
    instrs, i
end

gep_returns_vector(W::Int, X::Int, M::Int, ind_type::Symbol) = (!isone(W) && ((ind_type === :Vec) || !(isone(X) | iszero(X))))
#::Type{T}, ::Type{I}, W::Int = 1, ivec::Bool = false, constmul::Int = 1) where {T <: NativeTypes, I <: Integer}
function gep_quote(
    ::Type{T}, ind_type::Symbol, ::Type{I}, W::Int, X::Int, M::Int, O::Int, forgep::Bool
) where {T, I}
    T_sym = JULIA_TYPES[T]
    I_sym = JULIA_TYPES[I]
    gep_quote(T_sym, ind_type, I_sym, W, X, M, O, forgep)
end
function gep_quote(
    T_sym::Symbol, ind_type::Symbol, I_sym::Symbol, W::Int, X::Int, M::Int, O::Int, forgep::Bool
)
    sizeof_T = JULIA_TYPE_SIZE[T_sym]
    sizeof_I = JULIA_TYPE_SIZE[I_sym]
    if W > 1 && ind_type !== :Vec
        X, Xr = divrem(X, sizeof_T)
        @assert iszero(Xr)
    end
    if iszero(O) && (iszero(X) | isone(X)) && (iszero(M) || ind_type === :StaticInt)
        return Expr(:block, Expr(:meta, :inline), :ptr)
    end
    ibits = 8sizeof_I
    # ::Type{T}, ind_type::Symbol, indargname = '1', ibytes::Int, W::Int = 1, X::Int = 1, M::Int = 1, O::Int = 0, forgep::Bool = false
    instrs, i = offset_ptr(T_sym, ind_type, '1', ibits, W, X, M, O, true)
    ret = Expr(:curly, :Ptr, T_sym)
    lret = JULIAPOINTERTYPE
    if gep_returns_vector(W, X, M, ind_type)
        ret = Expr(:curly, :_Vec, W, ret)
        lret = "<$W x $lret>"
    end

    args = Expr(:curly, :Tuple, Expr(:curly, :Ptr, T_sym))
    largs = String[JULIAPOINTERTYPE]
    arg_syms = Union{Symbol,Expr}[:ptr]

    if !(iszero(M) || ind_type === :StaticInt)
        push!(arg_syms, Expr(:call, :data, :i))
        if ind_type === :Integer
            push!(args.args, I_sym)
            push!(largs, "i$(ibits)")
        else
            push!(args.args, Expr(:curly, :_Vec, W, I_sym))
            push!(largs, "<$W x i$(ibits)>")
        end
    end
    push!(instrs, "ret $lret %ptr.$(i-1)")
    llvmcall_expr("", join(instrs, "\n"), ret, args, lret, largs, arg_syms)
end

@generated function gep(ptr::Ptr{T}, i::I) where {I <: Integer, T <: NativeTypes}
    gep_quote(T, :Integer, I, 1, 1, 1, 0, true)
end
@generated function gep(ptr::Ptr{T}, ::StaticInt{N}) where {N, T <: NativeTypes}
    gep_quote(T, :StaticInt, Int, 1, 1, 0, N, true)
end
@generated function gep(ptr::Ptr{T}, i::LazyMulAdd{M,O,I}) where {T <: NativeTypes, I <: Integer, O, M}
    gep_quote(T, :Integer, I, 1, 1, M, O, true)
end
@generated function gep(ptr::Ptr{T}, i::Vec{W,I}) where {W, T <: NativeTypes, I <: Integer}
    gep_quote(T, :Vec, I, W, 1, 1, 0, true)
end
@generated function gep(ptr::Ptr{T}, i::LazyMulAdd{M,O,Vec{W,I}}) where {W, T <: NativeTypes, I <: Integer, M, O}
    gep_quote(T, :Vec, I, W, 1, M, O, true)
end
@inline gesp(ptr::AbstractStridedPointer, i) = similar_no_offset(ptr, gep(ptr, i))

function vload_quote(
    ::Type{T}, ::Type{I}, ind_type::Symbol, W::Int, X::Int, M::Int, O::Int, mask::Bool, align::Bool = false
) where {T <: NativeTypes, I <: Integer}
    T_sym = JULIA_TYPES[T]
    I_sym = JULIA_TYPES[I]
    vload_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align)
end
function vload_quote(
    T_sym::Symbol, I_sym::Symbol, ind_type::Symbol, W::Int, X::Int, M::Int, O::Int, mask::Bool, align::Bool = false
)
    isbit = T_sym === :Bit
    if !isbit && W > 1
        return vload_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, :(_Vec{$W,$T_sym}))
    end
    jtyp = isbit ? (isone(W) ? :Bool : mask_type_symbol(W)) : T_sym
    vload_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, jtyp)
end
function vload_split_quote(W::Int, sizeof_T::Int, mask::Bool, align::Bool)
    D, r1 = divrem(W * sizeof_T, REGISTER_SIZE)
    Wnew, r2 = divrem(W, D)
    @assert (iszero(r1) & iszero(r2)) "If loading more than a vector, Must load a multiple of the vector width."
    q = Expr(:block,Expr(:meta,:inline))
    # ind_type = :StaticInt, :Integer, :Vec
    push!(q.args, :(isplit = splitvectortotuple(StaticInt{$D}(), StaticInt{$Wnew}(), i)))
    mask && push!(q.args, :(msplit = splitvectortotuple(StaticInt{$D}(), StaticInt{$Wnew}(), m)))
    t = Expr(:tuple)
    alignval = Expr(:call, Expr(:curly, :Val, align))
    for d ∈ 1:D
        call = Expr(:call, :vload, :ptr)
        push!(call.args, Expr(:ref, :isplit, d))
        mask && push!(call.args, Expr(:ref, :msplit, d))
        push!(call.args, alignval)
        push!(t.args, call)
    end
    push!(q.args, :(VecUnroll($t)))
    q
end
function vload_quote(
    T_sym::Symbol, I_sym::Symbol, ind_type::Symbol, W::Int, X::Int, M::Int, O::Int, mask::Bool, align::Bool, ret
)
    sizeof_T = JULIA_TYPE_SIZE[T_sym]
    if W * sizeof_T > REGISTER_SIZE
        return vload_split_quote(W, sizeof_T, mask, align)
    end
    sizeof_I = JULIA_TYPE_SIZE[I_sym]
    ibits = 8sizeof_I
    if W > 1 && ind_type !== :Vec
        X, Xr = divrem(X, sizeof_T)
        @assert iszero(Xr)
    end
    instrs, i = offset_ptr(T_sym, ind_type, '1', ibits, W, X, M, O, false)
    grv = gep_returns_vector(W, X, M, ind_type)
    # considers booleans to only occupy 1 bit in memory, so they must be handled specially
    isbit = T_sym === :Bit
    if isbit
        # @assert !grv "gather's not are supported with `BitArray`s."
        mask = false # TODO: not this?
        # typ = "i$W"
        alignment = (align & (!grv)) ? cld(W,8) : 1
        typ = "i1"
    else
        alignment = (align & (!grv)) ? _get_alignment(W, T_sym) : _get_alignment(0, T_sym)
        typ = LLVM_TYPES_SYM[T_sym]
    end

    decl = LOAD_SCOPE_TBAA
    dynamic_index = !(iszero(M) || ind_type === :StaticInt)

    vtyp = vtype(W, typ)
    mask && truncate_mask!(instrs, '1' + dynamic_index, W, 0)
    if grv
        loadinstr = "$vtyp @llvm.masked.gather." * suffix(W, T_sym) * '.' * ptr_suffix(W, T_sym)
        decl *= "declare $loadinstr(<$W x $typ*>, i32, <$W x i1>, $vtyp)"
        m = mask ? m = "%mask.0" : llvmconst(W, "i1 1")
        passthrough = mask ? "zeroinitializer" : "undef"
        push!(instrs, "%res = call $loadinstr(<$W x $typ*> %ptr.$(i-1), i32 $alignment, <$W x i1> $m, $vtyp $passthrough)" * LOAD_SCOPE_TBAA_FLAGS)
    elseif mask
        suff = suffix(W, T_sym)
        loadinstr = "$vtyp @llvm.masked.load." * suff * ".p0" * suff
        decl *= "declare $loadinstr($vtyp*, i32, <$W x i1>, $vtyp)"
        push!(instrs, "%res = call $loadinstr($vtyp* %ptr.$(i-1), i32 $alignment, <$W x i1> %mask.0, $vtyp zeroinitializer)" * LOAD_SCOPE_TBAA_FLAGS)
    else
        push!(instrs, "%res = load $vtyp, $vtyp* %ptr.$(i-1), align $alignment" * LOAD_SCOPE_TBAA_FLAGS)
    end
    if isbit
        lret = string('i', max(8,W))
        if W > 1
            if W < 8
                push!(instrs, "%resint = bitcast <$W x i1> %res to i$(W)")
                push!(instrs, "%resfinal = zext i$(W) %resint to i8")
            else
                push!(instrs, "%resfinal = bitcast <$W x i1> %res to i$(W)")
            end
        else
            push!(instrs, "%resfinal = zext i1 %res to i8")
        end
        push!(instrs, "ret $lret %resfinal")
    else
        lret = vtyp
        push!(instrs, "ret $vtyp %res")
    end
    args = Expr(:curly, :Tuple, Expr(:curly, :Ptr, T_sym))
    largs = String[JULIAPOINTERTYPE]
    arg_syms = Union{Symbol,Expr}[:ptr]
    if dynamic_index
        push!(arg_syms, :(data(i)))
        if ind_type === :Integer
            push!(args.args, I_sym)
            push!(largs, "i$(ibits)")
        else
            push!(args.args, :(_Vec{$W,$I_sym}))
            push!(largs, "<$W x i$(ibits)>")
        end
    end
    if mask
        push!(arg_syms, :(data(m)))
        push!(args.args, mask_type(W))
        push!(largs, "i$(max(8,nextpow2(W)))")
    end
    if isbit && W > 1
        quote
            $(Expr(:meta,:inline))
            Mask{$W}($(llvmcall_expr(decl, join(instrs, "\n"), ret, args, lret, largs, arg_syms, true)))
        end
    else
        llvmcall_expr(decl, join(instrs, "\n"), ret, args, lret, largs, arg_syms)
    end
end
# vload_quote(T, ::Type{I}, ind_type::Symbol, W::Int, X, M, O, mask, align = false)
@generated function vload(ptr::Ptr{T}, ::Val{A}) where {T <: NativeTypes, A}
    vload_quote(T, Int, :StaticInt, 1, 1, 0, 0, false, A)
end
@generated function vload(ptr::Ptr{T}, i::I, ::Val{A}) where {A, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Integer, 1, 1, 1, 0, false, A)
end
@generated function vload(ptr::Ptr{T}, ::StaticInt{N}, ::Val{A}) where {A, N, T <: NativeTypes}
    vload_quote(T, Int, :StaticInt, 1, 1, 0, N, false, A)
end
@generated function vload(ptr::Ptr{T}, i::Vec{W,I}, ::Val{A}) where {A, W, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Vec, W, 1, 1, 0, false, A)
end
@generated function vload(ptr::Ptr{T}, i::MM{W,X,I}, ::Val{A}) where {A, W, X, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Integer, W, X, 1, 0, false, A)
end
@generated function vload(ptr::Ptr{T}, ::MM{W,X,StaticInt{N}}, ::Val{A}) where {A, W, X, T <: NativeTypes, N}
    vload_quote(T, Int, :StaticInt, W, X, 1, N, false, A)
end
@generated function vload(ptr::Ptr{T}, i::LazyMulAdd{M,O,I}, ::Val{A}) where {A, M, O, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Integer, 1, 1, M, O, false, A)
end
@generated function vload(ptr::Ptr{T}, i::LazyMulAdd{M,O,Vec{W,I}}, ::Val{A}) where {A, W, M, O, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Vec, W, 1, M, O, false, A)
end
@generated function vload(ptr::Ptr{T}, i::LazyMulAdd{M,O,MM{W,X,I}}, ::Val{A}) where {A, W, M, O, X, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Integer, W, X*M, M, O, false, A)
end
@generated function vload(ptr::Ptr{T}, i::LazyMulAdd{M,O,MM{W,X,StaticInt{I}}}, ::Val{A}) where {A, W, M, O, X, T <: NativeTypes, I}
    vload_quote(T, Int, :StaticInt, W, X*M, M, O + I*M, false, A)
end

@generated function vload(ptr::Ptr{T}, i::Vec{W,I}, m::Mask{W}, ::Val{A}) where {A, W, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Vec, W, 1, 1, 0, true, A)
end
@generated function vload(ptr::Ptr{T}, i::MM{W,X,I}, m::Mask{W}, ::Val{A}) where {A, W, X, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Integer, W, X, 1, 0, true, A)
end
@generated function vload(ptr::Ptr{T}, ::MM{W,X,StaticInt{N}}, m::Mask{W}, ::Val{A}) where {A, W, X, T <: NativeTypes, N}
    vload_quote(T, Int, :StaticInt, W, X, 1, N, true, A)
end
@generated function vload(ptr::Ptr{T}, i::LazyMulAdd{M,O,Vec{W,I}}, m::Mask{W}, ::Val{A}) where {A, W, M, O, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Vec, W, 1, M, O, true, A)
end
@generated function vload(ptr::Ptr{T}, i::LazyMulAdd{M,O,MM{W,X,I}}, m::Mask{W}, ::Val{A}) where {A, W, M, O, X, T <: NativeTypes, I <: IntegerTypes}
    vload_quote(T, I, :Integer, W, X*M, M, O, true, A)
end
@generated function vload(ptr::Ptr{T}, i::LazyMulAdd{M,O,MM{W,X,StaticInt{I}}}, m::Mask{W}, ::Val{A}) where {A, W, M, O, X, T <: NativeTypes, I}
    vload_quote(T, Int, :StaticInt, W, X*M, M, O + I*M, true, A)
end

@inline function _vload_scalar(ptr::Ptr{Bit}, i::Integer, ::Val{A}) where {A}
    d = i >> 3; r = i & 7;
    u = vload(Base.unsafe_convert(Ptr{UInt8}, ptr), d, Val{A}())
    (u >> r) % Bool
end
@inline vload(ptr::Ptr{Bit}, i::IntegerTypesHW, ::Val{A}) where {A} = _vload_scalar(ptr, i, Val{A}())
# avoid ambiguities
@inline vload(ptr::Ptr{Bit}, ::StaticInt{N}, ::Val{A}) where {A,N} = _vload_scalar(ptr, StaticInt{N}(), Val{A}())

@inline vload(ptr::Union{Ptr,AbstractStridedPointer}) = vload(ptr, Val{false}())
@inline vloada(ptr::Union{Ptr,AbstractStridedPointer}) = vload(ptr, Val{true}())
@inline vload(ptr::Union{Ptr,AbstractStridedPointer}, i::Union{Number,Tuple,Unroll}) = vload(ptr, i, Val{false}())
@inline vloada(ptr::Union{Ptr,AbstractStridedPointer}, i::Union{Number,Tuple,Unroll}) = vload(ptr, i, Val{true}())
@inline vload(ptr::Union{Ptr,AbstractStridedPointer}, i::Union{Number,Tuple,Unroll}, m::Mask) = vload(ptr, i, m, Val{false}())
@inline vloada(ptr::Union{Ptr,AbstractStridedPointer}, i::Union{Number,Tuple,Unroll}, m::Mask) = vload(ptr, i, m, Val{true}())
@inline vload(ptr::Union{Ptr,AbstractStridedPointer}, i::Union{Number,Tuple,Unroll}, b::Bool) = vload(ptr, i, b, Val{false}())
@inline vloada(ptr::Union{Ptr,AbstractStridedPointer}, i::Union{Number,Tuple,Unroll}, b::Bool) = vload(ptr, i, b, Val{true}())

@inline function vload(ptr::Ptr{T}, i::Number, b::Bool, ::Val{A}) where {T,A}
    if b
        vload(ptr, i, Val{A}())
    else
        zero(T)
    end
end
@inline vwidth_from_ind(i::Tuple) = vwidth_from_ind(i, StaticInt(1), StaticInt(0))
@inline vwidth_from_ind(i::Tuple{}, ::StaticInt{W}, ::StaticInt{U}) where {W,U} = (StaticInt{W}(), StaticInt{U}())
@inline function vwidth_from_ind(i::Tuple{<:AbstractSIMDVector{W},Vararg}, ::Union{StaticInt{1},StaticInt{W}}, ::StaticInt{U}) where {W,U}
    vwidth_from_ind(Base.tail(i), StaticInt{W}(), StaticInt{W}(U))
end
@inline function vwidth_from_ind(
    i::Tuple{<:VecUnroll{U,W},Vararg}, ::Union{StaticInt{1},StaticInt{W}}, ::Union{StaticInt{0},StaticInt{U}}
) where {U,W}
    vwidth_from_ind(Base.tail(i), StaticInt{W}(), StaticInt{W}(U))
end
@inline zero_init(::Type{T}, ::StaticInt{1}, ::StaticInt{0}) where {T} = zero(T)
@inline zero_init(::Type{T}, ::StaticInt{W}, ::StaticInt{0}) where {W,T} = vzero(Val(W), T)
@inline zero_init(::Type{T}, ::StaticInt{W}, ::StaticInt{U}) where {W,U,T} = zero(VecUnroll{U,W,T,Vec{W,T}})

@inline zero_init(::Type{T}, ::Tuple{StaticInt{W},StaticInt{U}}) where {W,U,T} = zero_init(T, StaticInt{W}(), StaticInt{U}())

@inline function vload(ptr::Ptr{T}, i::Tuple, b::Bool, ::Val{A}) where {T,A}
    if b
        vload(ptr, i, Val{A}())
    else
        zero_init(T, vwidth_from_ind(i))
    end
end
@generated function zero_vecunroll(::Val{N}, ::Val{W}, ::Type{T}) where {N,W,T}
    Expr(:block, Expr(:meta, :inline), :(zero(VecUnroll{$(N-1),$W,$T,Vec{$W,$T}})))
end
@inline function vload(ptr::Ptr{T}, i::Unroll{AU,F,N,AV,W,M,I}, b::Bool, ::Val{A}) where {T,AU,F,N,AV,W,M,I,A}
    if b
        vload(ptr, i, Val{A}())
    else
        zero_vecunroll(Val{N}(), Val{W}(), T)
        # VecUnroll(ntuple(@inline(_ -> vzero(Val{W}(), T)), Val{N}()))
    end
end

function vstore_quote(
    ::Type{T}, ::Type{I}, ind_type::Symbol, W::Int, X::Int, M::Int, O::Int, mask::Bool, align::Bool, noalias::Bool, nontemporal::Bool
) where {T <: NativeTypes, I <: Integer}
    T_sym = JULIA_TYPES[T]
    I_sym = JULIA_TYPES[I]
    vstore_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, noalias, nontemporal)
end
function vstore_quote(
    T_sym::Symbol, I_sym::Symbol, ind_type::Symbol, W::Int, X::Int, M::Int, O::Int, mask::Bool, align::Bool, noalias::Bool, nontemporal::Bool
)
    if W > 1
        vstore_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, noalias, nontemporal, :(_Vec{$W,$T_sym}))
    else
        vstore_quote(T_sym, I_sym, ind_type, W, X, M, O, mask, align, noalias, nontemporal, T_sym)
    end
end
function vstore_quote(
    T_sym::Symbol, I_sym::Symbol, ind_type::Symbol, W::Int, X::Int, M::Int, O::Int, mask::Bool, align::Bool, noalias::Bool, nontemporal::Bool, jtyp
)
    sizeof_T = JULIA_TYPE_SIZE[T_sym]
    sizeof_I = JULIA_TYPE_SIZE[I_sym]
    ibits = 8sizeof_I
    if W > 1 && ind_type !== :Vec
        X, Xr = divrem(X, sizeof_T)
        @assert iszero(Xr)
    end
    instrs, i = offset_ptr(T_sym, ind_type, '2', ibits, W, X, M, O, false)

    grv = gep_returns_vector(W, X, M, ind_type)

    align != nontemporal # should I do this?
    alignment = (align & (!grv)) ? _get_alignment(W, T_sym) : _get_alignment(0, T_sym)

    decl = noalias ? SCOPE_METADATA * STORE_TBAA : STORE_TBAA
    metadata = noalias ? STORE_SCOPE_FLAGS * STORE_TBAA_FLAGS : STORE_TBAA_FLAGS
    dynamic_index = !(iszero(M) || ind_type === :StaticInt)

    typ = LLVM_TYPES_SYM[T_sym]
    lret = vtyp = vtype(W, typ)
    mask && truncate_mask!(instrs, '2' + dynamic_index, W, 0)
    if grv
        storeinstr = "void @llvm.masked.scatter." * suffix(W, T_sym) * '.' * ptr_suffix(W, T_sym)
        decl *= "declare $storeinstr($vtyp, <$W x $typ*>, i32, <$W x i1>)"
        m = mask ? m = "%mask.0" : llvmconst(W, "i1 1")
        push!(instrs, "call $storeinstr($vtyp %1, <$W x $typ*> %ptr.$(i-1), i32 $alignment, <$W x i1> $m)" * metadata)
        # push!(instrs, "call $storeinstr($vtyp %1, <$W x $typ*> %ptr.$(i-1), i32 $alignment, <$W x i1> $m)")
    elseif mask
        suff = suffix(W, T_sym)
        storeinstr = "void @llvm.masked.store." * suff * ".p0" * suff
        decl *= "declare $storeinstr($vtyp, $vtyp*, i32, <$W x i1>)"
        push!(instrs, "call $storeinstr($vtyp %1, $vtyp* %ptr.$(i-1), i32 $alignment, <$W x i1> %mask.0)" * metadata)
    elseif nontemporal
        push!(instrs, "store $vtyp %1, $vtyp* %ptr.$(i-1), align $alignment, !nontemporal !{i32 1}" * metadata)
    else
        push!(instrs, "store $vtyp %1, $vtyp* %ptr.$(i-1), align $alignment" * metadata)
    end
    push!(instrs, "ret void")
    ret = :Cvoid; lret = "void"
    ptrtyp = Expr(:curly, :Ptr, T_sym)
    args = if W > 1
        Expr(:curly, :Tuple, ptrtyp, Expr(:curly, :NTuple, W, Expr(:curly, :VecElement, T_sym)))
    else
        Expr(:curly, :Tuple, ptrtyp, T_sym)
    end
    largs = String[JULIAPOINTERTYPE, vtyp]
    arg_syms = Union{Symbol,Expr}[:ptr, Expr(:call, :data, :v)]
    if dynamic_index
        push!(arg_syms, :(data(i)))
        if ind_type === :Integer
            push!(args.args, I_sym)
            push!(largs, "i$(ibits)")
        else
            push!(args.args, :(_Vec{$W,$I_sym}))
            push!(largs, "<$W x i$(ibits)>")
        end
    end
    if mask
        push!(arg_syms, :(data(m)))
        push!(args.args, mask_type(W))
        push!(largs, "i$(max(8,nextpow2(W)))")
    end
    llvmcall_expr(decl, join(instrs, "\n"), ret, args, lret, largs, arg_syms)
end

@generated function vstore!(
    ptr::Ptr{T}, v::T, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T <: NativeTypes, A, S, NT}
    vstore_quote(T, Int, :StaticInt, 1, 1, 0, 0, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::T, i::I, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T <: NativeTypes, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, 1, 1, 1, 0, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::T, ::StaticInt{N}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T <: NativeTypes, N, A, S, NT}
    vstore_quote(T, Int, :StaticInt, 1, 1, 0, N, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::Vec{W,I}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Vec, W, 1, 1, 0, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::MM{W,X,I}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, X, 1, 0, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::I, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, sizeof(T), 1, 0, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, A, S, NT}
    vstore_quote(T, Int, :StaticInt, W, sizeof(T), 0, 0, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, ::StaticInt{N}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, N, A, S, NT}
    vstore_quote(T, Int, :StaticInt, W, sizeof(T), 0, N, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, ::MM{W,X,StaticInt{N}}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, N, A, S, NT}
    vstore_quote(T, Int, :StaticInt, W, X, 0, N, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::T, i::LazyMulAdd{M,O,I}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T <: NativeTypes, M, O, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, 1, 1, M, O, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,Vec{W,I}}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, M, O, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Vec, W, 1, M, O, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,MM{W,X,I}}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, M, O, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, X*M, M, O, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,MM{W,X,StaticInt{I}}}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, M, O, I, A, S, NT}
    vstore_quote(T, Int, :StaticInt, W, X*M, M, O + M*I, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,I}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, M, O, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, sizeof(T), M, O, false, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::Vec{W,I}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Vec, W, 1, 1, 0, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::MM{W,X,I}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, X, 1, 0, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::I, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, sizeof(T), 1, 0, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, ::StaticInt{N}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, N, A, S, NT}
    vstore_quote(T, Int, :StaticInt, W, sizeof(T), 0, N, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, ::MM{W,X,StaticInt{N}}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, N, A, S, NT}
    vstore_quote(T, Int, :StaticInt, W, X, 0, N, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,Vec{W,I}}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, M, O, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Vec, W, 1, M, O, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,MM{W,X,I}}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, M, O, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, X*M, M, O, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,MM{W,X,StaticInt{I}}}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, X, T <: NativeTypes, M, O, I, A, S, NT}
    vstore_quote(T, Int, :StaticInt, W, X*M, M, O + M*I, true, A, S, NT)
end
@generated function vstore!(
    ptr::Ptr{T}, v::Vec{W,T}, i::LazyMulAdd{M,O,I}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T <: NativeTypes, M, O, I <: IntegerTypes, A, S, NT}
    vstore_quote(T, I, :Integer, W, sizeof(T), M, O, true, A, S, NT)
end

# broadcasting scalar stores
@inline function vstore!(
    ptr::Ptr{T}, v::Base.HWReal, i::VectorIndex{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T, A, S, NT}
    vstore!(ptr, convert(Vec{W,T}, v), i, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{T}, v::Base.HWReal, i::VectorIndex{W}, m::Mask, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T, A, S, NT}
    vstore!(ptr, convert(Vec{W,T}, v), i, m, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{T}, v::Base.HWReal, i::MM{1}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T, A, S, NT}
    vstore!(ptr, convert(T, v), i.i, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{T}, v::Base.HWReal, i::MM{1}, m::Mask{1}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T, A, S, NT}
    Bool(m) && vstore!(ptr, convert(T, v), i.i, Val{A}(), Val{S}(), Val{NT}())
    nothing
end
@inline function vstore!(
    ptr::Ptr{T}, v::Base.HWReal, i::LazyMulAdd{M,O,MM{1,X,I}}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T, A, S, NT, M,O,X,I}
    vstore!(ptr, convert(T, v), _materialize(i).i, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{T}, v::Base.HWReal, i::LazyMulAdd{M,O,MM{1,X,I}}, m::Mask{1}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T, A, S, NT, M,O,X,I}
    Bool(m) && vstore!(ptr, convert(T, v), _materialize(i).i, Val{A}(), Val{S}(), Val{NT}())
    nothing
end

# @inline function vstore!(ptr::Ptr{Bit}, m::Mask{W,U}, i, ::Val{A}, ::Val{S}, ::Val{NT}) where {W,U,A,S,NT}
#     @assert W == 8sizeof(U)
#     vstore!(Base.unsafe_convert(Ptr{U}, ptr), data(m), i, Val{A}(), Val{S}(), Val{NT}())
# end

# BitArray stores
@inline function vstore!(ptr::Ptr{Bit}, v::Mask{W,U}, ::Val{A}, ::Val{S}, ::Val{NT}) where {W, U, A, S, NT}
    vstore!(Base.unsafe_convert(Ptr{U}, ptr), data(v), Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{Bit}, v::Mask{W,U}, i::VectorIndex{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, U, A, S, NT}
    vstore!(Base.unsafe_convert(Ptr{U}, ptr), data(v), data(i) >> 3, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{Bit}, v::Mask{W,U}, i::VectorIndex{W}, m::Mask, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, U, A, S, NT}
    ishift = data(i) >> 3
    u = bitselect(data(m), vload(Base.unsafe_convert(Ptr{U}, ptr), ishift, Val{A}()), data(v))
    vstore!(Base.unsafe_convert(Ptr{U}, ptr), u, ishift, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(f::F, ptr::Ptr{Bit}, v::Mask{W,U}, ::Val{A}, ::Val{S}, ::Val{NT}) where {W, U, A, S, NT, F<:Function}
    vstore!(f, Base.unsafe_convert(Ptr{U}, ptr), data(v), Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    f::F, ptr::Ptr{Bit}, v::Mask{W,U}, i::VectorIndex{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, U, A, S, NT, F<:Function}
    vstore!(f, Base.unsafe_convert(Ptr{U}, ptr), data(v), data(i) >> 3, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    f::F, ptr::Ptr{Bit}, v::Mask{W,U}, i::VectorIndex{W}, m::Mask, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, U, A, S, NT, F<:Function}
    ishift = data(i) >> 3
    u = bitselect(data(m), vload(Base.unsafe_convert(Ptr{U}, ptr), ishift, Val{A}()), data(v))
    vstore!(f, Base.unsafe_convert(Ptr{U}, ptr), u, ishift, Val{A}(), Val{S}(), Val{NT}())
end

@inline function vstore!(ptr::Ptr{T}, v, ::Val{A}, ::Val{S}, ::Val{NT}) where {T, A, S, NT}
    vstore!(ptr, convert(T, v), Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{T}, v, i, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T, A, S, NT}
    vstore!(ptr, convert(T, v), i, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{T}, v, i::VectorIndex{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W,T, A, S, NT}
    vstore!(ptr, convert(Vec{W,T}, v), i, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    ptr::Ptr{T}, v, i, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T, A, S, NT}
    vstore!(ptr, convert(Vec{W,T}, v), i, m, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(f::F, ptr::Ptr{T}, v, ::Val{A}, ::Val{S}, ::Val{NT}) where {T, A, S, NT, F<:Function}
    vstore!(f, ptr, convert(T, v), Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    f::F, ptr::Ptr{T}, v, i, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T, A, S, NT, F<:Function}
    vstore!(f, ptr, convert(T, v), i, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    f::F, ptr::Ptr{T}, v, i::VectorIndex{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T, A, S, NT, F<:Function}
    vstore!(f, ptr, convert(Vec{W,T}, v), i, Val{A}(), Val{S}(), Val{NT}())
end
@inline function vstore!(
    f::F, ptr::Ptr{T}, v, i, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {W, T, A, S, NT, F<:Function}
    vstore!(f, ptr, convert(Vec{W,T}, v), i, m, Val{A}(), Val{S}(), Val{NT}())
end

for (store,align,alias,nontemporal) ∈ [
    (:vstore!,false,false,false),
    (:vstorea!,true,false,false),
    (:vstorent!,true,false,true),
    (:vnoaliasstore!,false,true,false),
    (:vnoaliasstorea!,true,true,false),
    (:vnoaliasstorent!,true,true,true)
]
    @eval begin
        @inline function $store(ptr::Union{Ptr,AbstractStridedPointer}, v::Number)
            vstore!(ptr, v, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end
        @inline function $store(ptr::Union{Ptr,AbstractStridedPointer}, v::Number, i::Union{Number,Tuple,Unroll})
            vstore!(ptr, v, i, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end
        @inline function $store(ptr::Union{Ptr,AbstractStridedPointer}, v::Number, i::Union{Number,Tuple,Unroll}, m::Mask)
            vstore!(ptr, v, i, m, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end
        @inline function $store(ptr::Union{Ptr,AbstractStridedPointer}, v::Number, i::Union{Number,Tuple,Unroll}, b::Bool)
            b && vstore!(ptr, v, i, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end

        @inline function $store(f::F, ptr::Union{Ptr,AbstractStridedPointer}, v::Number) where {F<:Function}
            vstore!(f, ptr, v, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end
        @inline function $store(f::F, ptr::Union{Ptr,AbstractStridedPointer}, v::Number, i::Union{Number,Tuple,Unroll}) where {F<:Function}
            vstore!(f, ptr, v, i, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end
        @inline function $store(f::F, ptr::Union{Ptr,AbstractStridedPointer}, v::Number, i::Union{Number,Tuple,Unroll}, m::Mask) where {F<:Function}
            vstore!(f, ptr, v, i, m, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end
        @inline function $store(f::F, ptr::Union{Ptr,AbstractStridedPointer}, v::Number, i::Union{Number,Tuple,Unroll}, b::Bool) where {F<:Function}
            b && vstore!(f, ptr, v, i, Val{$align}(), Val{$alias}(), Val{$nontemporal}())
        end
    end
end

# unroll
@inline Base.Broadcast.broadcastable(u::Unroll) = (u,)

"""
Returns a vector of expressions for a set of unrolled indices.

"""
function unrolled_indicies(D::Int, AU::Int, F::Int, N::Int, AV::Int, W::Int)
    baseind = Expr(:tuple)
    for d in 1:D
        i = Expr(:call, :Zero)
        if d == AV && W > 1
            i = Expr(:call, Expr(:curly, :MM, W), i)
        end
        push!(baseind.args, i)
    end
    WF = AU == AV ? W : 1
    inds = Vector{Expr}(undef, N)
    inds[1] = baseind
    for n in 1:N-1
        ind = copy(baseind)
        i = Expr(:call, Expr(:curly, :StaticInt, n*F*WF))
        if AU == AV && W > 1
            i = Expr(:call, Expr(:curly, :MM, W), i)
        end
        ind.args[AU] = i
        inds[n+1] = ind
    end
    inds
end

function vload_unroll_quote(D::Int, AU::Int, F::Int, N::Int, AV::Int, W::Int, M::UInt, mask::Bool, align::Bool)
    t = Expr(:tuple)
    inds = unrolled_indicies(D, AU, F, N, AV, W)
    # TODO: Consider doing some alignment checks before accepting user's `align`?
    alignval = Expr(:call, Expr(:curly, :Val, align))
    for n in 1:N
        l = Expr(:call, :vload, :gptr, inds[n])
        (mask && (M % Bool)) && push!(l.args, :m)
        push!(l.args, alignval)
        M >>= 1
        push!(t.args, l)
    end
    quote
        $(Expr(:meta, :inline))
        gptr = gesp(ptr, u.i)
        VecUnroll($t)
    end
end

@generated function vload(ptr::AbstractStridedPointer{T,D}, u::Unroll{AU,F,N,AV,W,M,I}, ::Val{A}) where {A,AU,F,N,AV,W,M,I,T,D}
    vload_unroll_quote(D, AU, F, N, AV, W, M, false, A)
end
@generated function vload(ptr::AbstractStridedPointer{T,D}, u::Unroll{AU,F,N,AV,W,M,I}, m::Mask{W}, ::Val{A}) where {A,AU,F,N,AV,W,M,I,T,D}
    vload_unroll_quote(D, AU, F, N, AV, W, M, true, A)
end

function vstore_unroll_quote(
    D::Int, AU::Int, F::Int, N::Int, AV::Int, W::Int, M::UInt, mask::Bool, align::Bool, noalias::Bool, nontemporal::Bool
)
    t = Expr(:tuple)
    inds = unrolled_indicies(D, AU, F, N, AV, W)
    q = quote
        $(Expr(:meta, :inline))
        gptr = gesp(ptr, u.i)
        t = data(v)
    end
    alignval = Expr(:call, Expr(:curly, :Val, align))
    noaliasval = Expr(:call, Expr(:curly, :Val, noalias))
    nontemporalval = Expr(:call, Expr(:curly, :Val, nontemporal))
    for n in 1:N
        l = Expr(:call, :vstore!, :gptr, Expr(:ref, :t, n), inds[n])
        (mask && (M % Bool)) && push!(l.args, :m)
        push!(l.args, alignval, noaliasval, nontemporalval)
        M >>= 1
        push!(q.args, l)
    end
    q
end
@generated function vstore!(
    ptr::AbstractStridedPointer{T,D}, v::VecUnroll{Nm1,W}, u::Unroll{AU,F,N,AV,W,M,I}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {AU,F,N,AV,W,M,I,T,D,Nm1,S,A,NT}
    @assert Nm1+1 == N
    vstore_unroll_quote(D, AU, F, N, AV, W, M, false, A, S, NT)
end
@generated function vstore!(
    ptr::AbstractStridedPointer{T,D}, v::VecUnroll{Nm1,W}, u::Unroll{AU,F,N,AV,W,M,I}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {AU,F,N,AV,W,M,I,T,D,Nm1,S,A,NT}
    @assert Nm1+1 == N
    vstore_unroll_quote(D, AU, F, N, AV, W, M, true, A, S, NT)
end
function vstore_unroll_i_quote(Nm1, Wsplit, W, A, S, NT, mask::Bool)
    N = Nm1 + 1
    @assert N*Wsplit == W
    q = Expr(:block, Expr(:meta, :inline), :(vt = data(v)), :(im = _materialize(i)))
    if mask
        let U = mask_type_symbol(Wsplit)
            push!(q.args, :(mt = data(vconvert(VecUnroll{$Nm1,$Wsplit,Bit,Mask{$Wsplit,$U}}, m))))
        end
    end
    j = 0
    alignval = Expr(:call, Expr(:curly, :Val, A))
    aliasval = Expr(:call, Expr(:curly, :Val, S))
    notmpval = Expr(:call, Expr(:curly, :Val, NT))
    for n ∈ 1:N
        shufflemask = Expr(:tuple)
        for w ∈ 1:Wsplit
            push!(shufflemask.args, j)
            j += 1
        end
        ex = :(vstore!(ptr, vt[$n], shufflevector(im, Val{$shufflemask}())))
        mask && push!(ex.args, Expr(:ref, :mt, n))
        push!(ex.args, alignval)
        push!(ex.args, aliasval)
        push!(ex.args, notmpval)
        push!(q.args, ex)
    end
    q
end
@generated function vstore!(
    ptr::Ptr{T}, v::VecUnroll{Nm1,Wsplit}, i::VectorIndex{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T,Nm1,Wsplit,W,A,S,NT}
    vstore_unroll_i_quote(Nm1, Wsplit, W, A, S, NT, false)
end
@generated function vstore!(
    ptr::Ptr{T}, v::VecUnroll{Nm1,Wsplit}, i::VectorIndex{W}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T,Nm1,Wsplit,W,A,S,NT}
    vstore_unroll_i_quote(Nm1, Wsplit, W, A, S, NT, true)
end
function vstorebit_unroll_i_quote(Nm1, Wsplit, W, A, S, NT, mask::Bool)
    N = Nm1 + 1
    @assert N*Wsplit == W
    @assert W == 8
    # q = Expr(:block, Expr(:meta, :inline), :(vt = data(v)), :(im = _materialize(i)), :(u = 0x00))
    q = Expr(:block, Expr(:meta, :inline), :(vt = data(v)), :(u = 0x00))
    j = 0
    while true
        push!(q.args, :(u |= data($(Expr(:ref, :vt, (N-j))))))
        j += 1
        j == N && break
        push!(q.args, :(u <<= $Wsplit))
    end
    alignval = Expr(:call, Expr(:curly, :Val, A))
    aliasval = Expr(:call, Expr(:curly, :Val, S))
    notmpval = Expr(:call, Expr(:curly, :Val, NT))
    mask && push!(q.args, :(u = bitselect(data(m), vload(Base.unsafe_convert(Ptr{$(mask_type_symbol(W))}, ptr), (i.i >> 3), Val{$A}()), u)))
    call = Expr(:call, :vstore!, :(reinterpret(Ptr{UInt8}, ptr)), :u, :(i.i >> 3))
    push!(call.args, alignval, aliasval, notmpval)
    push!(q.args, call)
    q
end
@generated function vstore!(
    ptr::Ptr{Bit}, v::VecUnroll{Nm1,Wsplit,Bit,Mask{Wsplit,UInt8}}, i::MM{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {Nm1,Wsplit,W,A,S,NT}
    # 1 + 1
    vstorebit_unroll_i_quote(Nm1, Wsplit, W, A, S, NT, false)
end
@generated function vstore!(
    ptr::Ptr{Bit}, v::VecUnroll{Nm1,Wsplit,Bit,Mask{Wsplit,UInt8}}, i::MM{W}, m::Mask{W}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {Nm1,Wsplit,W,A,S,NT}
    vstorebit_unroll_i_quote(Nm1, Wsplit, W, A, S, NT, true)
end

# @inline vstore!(::typeof(identity), ptr, v, u) = vstore!(ptr, v, u)
# @inline vstore!(::typeof(identity), ptr, v, u, m) = vstore!(ptr, v, u, m)
# @inline vnoaliasstore!(::typeof(identity), ptr, v, u) = vnoaliasstore!(ptr, v, u)
# @inline vnoaliasstore!(::typeof(identity), ptr, v, u, m) = vnoaliasstore!(ptr, v, u, m)

# If `::Function` vectorization is masked, then it must not be reduced by `::Function`.
@generated function vstore!(
    ::Function, ptr::AbstractStridedPointer{T,D,C}, vu::VecUnroll{U,W}, u::Unroll{AU,F,N,AV,W,M,I}, m, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T,D,C,U,AU,F,N,W,M,I,AV,A,S,NT}
    @assert N == U + 1
    # mask means it isn't vectorized
    @assert AV > 0 "AV ≤ 0, but masking what, exactly?"
    Expr(:block, Expr(:meta, :inline), :(vstore!(ptr, vu, u, m, Val{$A}(), Val{$S}(), Val{$NT}())))
end

function transposeshuffle(split, W, offset::Bool)
    tup = Expr(:tuple)
    w = 0
    S = 1 << split
    i = offset ? S : 0
    while w < W
        for s ∈ 0:S-1
            push!(tup.args, w + s + i)
        end
        for s ∈ 0:S-1
            # push!(tup.args, w + W + s)
            push!(tup.args, w + W + s + i)
        end
        w += 2S
    end
    Expr(:call, Expr(:curly, :Val, tup))
end

function horizonal_reduce_store_expr(W, Ntotal, (C,D,AU,F), op = :+, reduct = :vsum, noalias::Bool = false)
    N = ((C == AU) && isone(F)) ? prevpow2(Ntotal) : 0
    q = Expr(:block, Expr(:meta, :inline), :(v = data(vu)))
    store = noalias ? :vnoaliasstore! : :vstore!
    @assert ispow2(W)
    if N > 1
        if N < Ntotal
            push!(q.args, :(gptr = gesp(ptr, u.i)))
            push!(q.args, :(bptr = pointer(gptr)))
        else
            push!(q.args, :(bptr = gep(ptr, u.i)))
        end
        extractblock = Expr(:block)
        vectors = [Symbol(:v_, n) for n ∈ 0:N-1]
        for n ∈ 1:N
            push!(extractblock.args, Expr(:(=), vectors[n], Expr(:ref, :v, n)))
        end
        push!(q.args, Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), extractblock))
        ncomp = 0
        minWN = min(W,N)
        while ncomp < N
            Nt = minWN;
            Wt = W
            splits = 0
            while Nt > 1
                Nt >>>= 1
                shuffle0 = transposeshuffle(splits, Wt, false)
                shuffle1 = transposeshuffle(splits, Wt, true)
                splits += 1
                for nh ∈ 1:Nt
                    n1 = 2nh
                    n0 = n1 - 1
                    v0 = vectors[n0 + ncomp]; v1 = vectors[n1 + ncomp]; vh = vectors[nh + ncomp];
                    # combine n0 and n1
                    push!(q.args, Expr(
                        :(=), vh, Expr(
                            :call, op,
                            Expr(:call, :shufflevector, v0, v1, shuffle0),
                            Expr(:call, :shufflevector, v0, v1, shuffle1))
                    ))
                end
            end
            # v0 is now the only vector
            v0 = vectors[ncomp + 1]
            while Wt > minWN
                Wh = Wt >>> 1
                v0new = Symbol(v0, Wt)
                push!(q.args, Expr(
                    :(=), v0new, Expr(
                        :call, op,
                        Expr(:call, :shufflevector, v0, Expr(:call, Expr(:curly, :Val, Expr(:tuple, [w for w ∈ 0:Wh-1]...)))),
                        Expr(:call, :shufflevector, v0, Expr(:call, Expr(:curly, :Val, Expr(:tuple, [w for w ∈ Wh:Wt-1]...)))))
                )
                      )
                v0 = v0new
                Wt = Wh
            end
            push!(q.args, Expr(:call, store, :bptr, v0))
            ncomp += minWN
        end
    else
        push!(q.args, :(gptr = gesp(ptr, u.i)))
    end
    if N < Ntotal
        zeroexpr = Expr(:call, Expr(:curly, :StaticInt, 0))
        ind = Expr(:tuple); foreach(_ -> push!(ind.args, zeroexpr), 1:D)
        for n ∈ N+1:Ntotal
            (n > N+1) && (ind = copy(ind)) # copy to avoid overwriting old
            ind.args[AU] = Expr(:call, Expr(:curly, :StaticInt, F*(n-1)))
            scalar = Expr(:call, reduct, Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), Expr(:ref, :v, n)))
            push!(q.args, Expr(:call, store, :gptr, scalar, ind))
        end
    end
    q
end
@generated function vstore!(
    ::G, ptr::AbstractStridedPointer{T,D,C}, vu::VecUnroll{U,W}, u::Unroll{AU,F,N,AV,W,M,I}, ::Val{A}, ::Val{S}, ::Val{NT}
) where {T,D,C,U,AU,F,N,W,M,I,G<:Function,AV,A,S,NT}
    @assert N == U + 1
    if G === typeof(identity) || AV > 0
        return Expr(:block, Expr(:meta, :inline), :(vstore!(ptr, vu, u, Val{$A}(), Val{$S}(), Val{$NT}())))
    elseif G === typeof(vsum)
        op = :+; reduct = :vsum
    elseif G === typeof(vprod)
        op = :*; reduct = :vprod
    else
        throw("Function $f not recognized.")
    end
    horizonal_reduce_store_expr(W, N, (C,D,AU,F), op, reduct, S)
end

function lazymulunroll_load_quote(M,O,N,mask,align)
    t = Expr(:tuple)
    alignval = Expr(:call, Expr(:curly, :Val, align))
    for n in 1:N+1
        ind = if (M != 1) | (O != 0)
            :(LazyMulAdd{$M,$O}(u[$n]))
        else
            Expr(:ref, :u, n)
        end
        call = if mask
            Expr(:call, :vload, :ptr, ind, Expr(:ref, :mt, n), alignval)
        else
            Expr(:call, :vload, :ptr, ind, alignval)
        end
        push!(t.args, call)
    end
    q = Expr(:block, Expr(:meta, :inline), :(u = data(um)))
    mask && push!(q.args, :(mt = data(m)))
    push!(q.args, Expr(:call, :VecUnroll, t))
    q
end
@generated function vload(ptr::Ptr{T}, um::VecUnroll{N,W,I,V}, ::Val{A}) where {T,N,W,I,V,A}
    lazymulunroll_load_quote(1,0,N,false,A)
end
@generated function vload(ptr::Ptr{T}, um::VecUnroll{N,W,I,V}, m::VecUnroll{N,W,Bit,Mask{W,U}}, ::Val{A}) where {T,N,W,I,V,A,U}
    lazymulunroll_load_quote(1,0,N,true,A)
end
@inline function vload(ptr::Ptr{T}, um::VecUnroll{N,W,I,V}, m::Mask, ::Val{A}) where {T,N,W,I,V,A}
    vload(ptr, um, VecUnroll(splitvectortotuple(StaticInt{N}() + One(), StaticInt{W}(), m)), Val{A}())
end
@generated function vload(ptr::Ptr{T}, um::LazyMulAdd{M,O,VecUnroll{N,W,I,V}}, ::Val{A}) where {T,M,O,N,W,I,V,A}
    lazymulunroll_load_quote(M,O,N,false,A)
end
@generated function vload(ptr::Ptr{T}, um::LazyMulAdd{M,O,VecUnroll{N,W,I,V}}, m::VecUnroll{N,W,Bit,Mask{W,U}}, ::Val{A}) where {T,M,O,N,W,I,V,A,U}
    lazymulunroll_load_quote(M,O,N,true,A)
end
@inline function vload(ptr::Ptr{T}, um::LazyMulAdd{M,O,VecUnroll{N,W,I,V}}, m::Mask, ::Val{A}) where {T,M,O,N,W,I,V,A}
    vload(ptr, um, VecUnroll(splitvectortotuple(StaticInt{N}() + One(), StaticInt{W}(), m)), Val{A}())
end
function lazymulunroll_store_quote(M,O,N,mask,align,noalias,nontemporal)
    q = Expr(:block, Expr(:meta, :inline), :(u = um.data.data), :(v = vm.data.data))
    alignval = Expr(:call, Expr(:curly, :Val, align))
    noaliasval = Expr(:call, Expr(:curly, :Val, noalias))
    nontemporalval = Expr(:call, Expr(:curly, :Val, nontemporal))
    for n in 1:N+1
        push!(q.args, Expr(:call, :vstore!, :ptr, Expr(:ref, :v, n), :(LazyMulAdd{$M,$O}(u[$n])), alignval, noaliasval, nontemporalval))
    end
    q
end

@generated function prefetch(ptr::Ptr{Cvoid}, ::Val{L}, ::Val{R}) where {L, R}
    @assert L ∈ (0,1,2,3)
    @assert R ∈ (0, 1)
    decl = "declare void @llvm.prefetch(i8*, i32, i32, i32)"
    instrs = """
        %addr = inttoptr $JULIAPOINTERTYPE %0 to i8*
        call void @llvm.prefetch(i8* %addr, i32 $R, i32 $L, i32 1)
        ret void
    """
    llvmcall_expr(decl, instrs, Cvoid, :(Tuple{Ptr{Cvoid}}), "void", [JULIAPOINTERTYPE], [:ptr])
end
@inline prefetch(ptr::Ptr{T}, ::Val{L}, ::Val{R}) where {T,L,R} = prefetch(Base.unsafe_convert(Ptr{Cvoid}, ptr), Val{L}(), Val{R}())

@inline function prefetch(ptr::Union{AbstractStridedPointer,Ptr}, i, ::Val{Locality}, ::Val{ReadOrWrite}) where {Locality, ReadOrWrite}
    prefetch(gep(ptr, i), Val{Locality}(), Val{ReadOrWrite}())
end
@inline prefetch(ptr::Ptr) = prefetch(ptr, Val{3}(), Val{0}())
@inline prefetch(ptr::Ptr, ::Val{L}) where {L} = prefetch(ptr, Val{L}(), Val{0}())
@inline prefetch(ptr::Ptr, i) = prefetch(ptr, i, Val{3}(), Val{0}())
@inline prefetch(ptr::Ptr, i, ::Val{L}) where {L} = prefetch(ptr, i, Val{L}(), Val{0}())

@inline prefetch0(x, i) = prefetch(gep(stridedpointer(x), (data(i),)), Val{3}(), Val{0}())
@inline prefetch0(x, I::Tuple) = prefetch(gep(stridedpointer(x), data.(I)), Val{3}(), Val{0}())
@inline prefetch0(x, i, j) = prefetch(gep(stridedpointer(x), (data(i), data(j))), Val{3}(), Val{0}())
# @inline prefetch0(x, i, j, oi, oj) = prefetch(gep(stridedpointer(x), (data(i) + data(oi) - 1, data(j) + data(oj) - 1)), Val{3}(), Val{0}())
@inline prefetch1(x, i) = prefetch(gep(stridedpointer(x), (data(i),)), Val{2}(), Val{0}())
@inline prefetch1(x, i, j) = prefetch(gep(stridedpointer(x), (data(i), data(j))), Val{2}(), Val{0}())
# @inline prefetch1(x, i, j, oi, oj) = prefetch(gep(stridedpointer(x), (data(i) + data(oi) - 1, data(j) + data(oj) - 1)), Val{2}(), Val{0}())
@inline prefetch2(x, i) = prefetch(gep(stridedpointer(x), (data(i),)), Val{1}(), Val{0}())
@inline prefetch2(x, i, j) = prefetch(gep(stridedpointer(x), (data(i), data(j))), Val{1}(), Val{0}())
# @inline prefetch2(x, i, j, oi, oj) = prefetch(gep(stridedpointer(x), (data(i) + data(oi) - 1, data(j) + data(oj) - 1)), Val{1}(), Val{0}())

@generated function lifetime_start!(ptr::Ptr{T}, ::Val{L}) where {L,T}
    decl = "declare void @llvm.lifetime.start(i64, i8* nocapture)"
    instrs = "%ptr = inttoptr $JULIAPOINTERTYPE %0 to i8*\ncall void @llvm.lifetime.start(i64 $(L*sizeof(T)), i8* %ptr)\nret void"
    llvmcall_expr(decl, instrs, :Cvoid, :(Tuple{Ptr{$T}}), "void", [JULIAPOINTERTYPE], [:ptr])
end
@generated function lifetime_end!(ptr::Ptr{T}, ::Val{L}) where {L,T}
    decl = "declare void @llvm.lifetime.end(i64, i8* nocapture)"
    instrs = "%ptr = inttoptr $JULIAPOINTERTYPE %0 to i8*\ncall void @llvm.lifetime.end(i64 $(L*sizeof(T)), i8* %ptr)\nret void"
    llvmcall_expr(decl, instrs, :Cvoid, :(Tuple{Ptr{$T}}), "void", [JULIAPOINTERTYPE], [:ptr])
end

@inline lifetime_start!(ptr::Ptr) = lifetime_start!(ptr, Val{-1}())
@inline lifetime_end!(ptr::Ptr) = lifetime_end!(ptr, Val{-1}())
# Fallback is to do nothing. Intention is (e.g.) for PaddedMatrices/StackPointers.
@inline lifetime_start!(::Any) = nothing
@inline lifetime_end!(::Any) = nothing

@generated function compressstore!(ptr::Ptr{T}, v::Vec{W,T}, mask::Mask{W,U}) where {W,T <: NativeTypes, U<:Unsigned}
    @assert 8sizeof(U) >= W
    typ = LLVM_TYPES[T]
    vtyp = "<$W x $typ>"
    mtyp_input = LLVM_TYPES[U]
    mtyp_trunc = "i$W"
    instrs = String["%ptr = inttoptr $JULIAPOINTERTYPE %1 to $typ*"]
    truncate_mask!(instrs, '2', W, 0)
    decl = "declare void @llvm.masked.compressstore.$(suffix(W,T))($vtyp, $typ*, <$W x i1>)"
    push!(instrs, "call void @llvm.masked.compressstore.$(suffix(W,T))($vtyp %0, $typ* %ptr, <$W x i1> %mask.0)\nret void")
    llvmcall_expr(decl, join(instrs,"\n"), :Cvoid, :(Tuple{NTuple{$W,VecElement{$T}}, Ptr{$T}, $U}), "void", [vtyp, JULIAPOINTERTYPE, "i$(8sizeof(U))"], [:(data(v)), :ptr, :(data(mask))])
end

@generated function expandload(ptr::Ptr{T}, mask::Mask{W,U}) where {W, T <: NativeTypes, U<:Unsigned}
    @assert 8sizeof(U) >= W
    typ = LLVM_TYPES[T]
    vtyp = "<$W x $typ>"
    vptrtyp = "<$W x $typ*>"
    mtyp_input = LLVM_TYPES[U]
    mtyp_trunc = "i$W"
    instrs = String[]
    push!(instrs, "%ptr = inttoptr $JULIAPOINTERTYPE %0 to $typ*")
    if mtyp_input == mtyp_trunc
        push!(instrs, "%mask = bitcast $mtyp_input %1 to <$W x i1>")
    else
        push!(instrs, "%masktrunc = trunc $mtyp_input %1 to $mtyp_trunc")
        push!(instrs, "%mask = bitcast $mtyp_trunc %masktrunc to <$W x i1>")
    end
    decl = "declare $vtyp @llvm.masked.expandload.$(suffix(W,T))($typ*, <$W x i1>, $vtyp)"
    push!(instrs, "%res = call $vtyp @llvm.masked.expandload.$(suffix(W,T))($typ* %ptr, <$W x i1> %mask, $vtyp zeroinitializer)\nret $vtyp %res")
    llvmcall_expr(decl, join(instrs,"\n"), :(NTuple{$W,VecElement{$T}}), :(Tuple{Ptr{$T}, $U}), vtyp, [JULIAPOINTERTYPE, "i$(8sizeof(U))"], [:ptr, :(data(mask))])
end

@inline vload(::StaticInt{N}, args...) where {N} = StaticInt{N}()
@inline stridedpointer(::StaticInt{N}) where {N} = StaticInt{N}()
@inline zero_offsets(::StaticInt{N}) where {N} = StaticInt{N}()


"""
BatchedFeaturedGraph

A batched representation of the FeaturedGraph, to enable parallel computation.

It is deliberately more restrictive to prevent incorrect usage.
"""
struct BatchedFeaturedGraph{T <: Real} <: AbstractFeaturedGraph
    graph::AbstractArray{T, 3}
    nf::AbstractArray{T, 3}
    ef::AbstractArray{T, 4}
    gf::AbstractMatrix{T}

    function BatchedFeaturedGraph{T}(graph, nf, ef, gf) where T <: Real
        check_dimensions(graph, nf, ef, gf)
        return new{T}(graph, nf, ef, gf)
    end

    function BatchedFeaturedGraph{T}(
        graph;
        nf=zeros(0, size(graph, 1), size(graph, 3)),
        ef=zeros(0, size(graph, 1), size(graph, 1), size(graph, 3)),
        gf=zeros(0, size(graph, 3))
    ) where T <: Real
        check_dimensions(graph, nf, ef, gf)
        return new{T}(graph, nf, ef, gf)
    end
end

BatchedFeaturedGraph(graph, nf, ef, gf) = BatchedFeaturedGraph{Float32}(graph, nf, ef, gf)

function BatchedFeaturedGraph{T}(fgs::Vector{FG}) where {T <: Real, FG <: FeaturedGraph}
    ngraphs = length(fgs)
    maxNodes = Base.maximum(nv, fgs)
    nfLength = size(fgs[1].nf, 1)
    efLength = size(fgs[1].ef, 1)
    gfLength = size(fgs[1].gf, 1)

    graph = zeros(T, maxNodes, maxNodes, ngraphs)
    nf = zeros(T, nfLength, maxNodes, ngraphs)
    ef = zeros(T, efLength, maxNodes, maxNodes, ngraphs)
    gf = zeros(T, gfLength, ngraphs)

    for (i, fg) in enumerate(fgs)
        graph[1:nv(fg),1:nv(fg),i] = fg.graph
        nf[:, 1:nv(fg), i] = fg.nf
        ef[:, 1:nv(fg), 1:nv(fg), i] = fg.ef
        gf[:, i] = fg.gf
    end

    return BatchedFeaturedGraph{T}(graph, nf, ef, gf)
end

BatchedFeaturedGraph(fgs::Vector{FG}) where {FG <: FeaturedGraph} = BatchedFeaturedGraph{Float32}(fgs)

# ========== Accessing ==========
# code from GraphSignals.jl

"""
    graph(::BatchedFeaturedGraph)
Get referenced graph.
"""
graph(fgs::BatchedFeaturedGraph) = fgs.graph

"""
    node_feature(::BatchedFeaturedGraph)
Get node feature attached to graph.
"""
node_feature(fgs::BatchedFeaturedGraph) = fgs.nf

"""
    edge_feature(::BatchedFeaturedGraph)
Get edge feature attached to graph.
"""
edge_feature(fgs::BatchedFeaturedGraph) = fgs.ef

"""
    global_feature(::BatchedFeaturedGraph)
Get global feature attached to graph.
"""
global_feature(fgs::BatchedFeaturedGraph) = fgs.gf

"""
    has_graph(::BatchedFeaturedGraph)
Check if graph is available or not.
"""
has_graph(fgs::BatchedFeaturedGraph) = fgs.graph != zeros(0,0,0)

"""
    has_node_feature(::BatchedFeaturedGraph)
Check if node feature is available or not.
"""
has_node_feature(fgs::BatchedFeaturedGraph) = !isempty(fgs.nf)

"""
    has_edge_feature(::BatchedFeaturedGraph)
Check if edge feature is available or not.
"""
has_edge_feature(fgs::BatchedFeaturedGraph) = !isempty(fgs.ef)

"""
    has_global_feature(::BatchedFeaturedGraph)
Check if global feature is available or not.
"""
has_global_feature(fgs::BatchedFeaturedGraph) = !isempty(fgs.gf)

# ========== LightGraphs compatibility ==========

LightGraphs.nv(fgs::BatchedFeaturedGraph) = nv(graph(fgs))
LightGraphs.nv(g::AbstractArray{T, 3}) where T<:Real = size(g,1)

Flux.@functor BatchedFeaturedGraph


"""
    cm2inch(x)

Convert length in cm to inch units: 1 inch = 2.54 cm.
"""
cm2inch(x) = x/2.54

"""
    embed_system(m;name)

Embeds system `m` as the single component of a larger system.
This helps to match the naming of the states, parameters, observables to
the namespace of the system.

```jldocstest; output=false
using ModelingToolkit, DifferentialEquations
using MTKHelpers
function samplesystem(;name)
    @variables t
    D = Differential(t)
    sts = @variables x(t) RHS(t)  # RHS is observed
    ps = @parameters τ       # parameters
    ODESystem([ RHS  ~ (1 - x)/τ, D(x) ~ RHS ], t, sts, ps; name)
end
@named m = samplesystem()
@named sys = embed_system(m)

# note that keys are `m.x`,`m.τ` or `m.RHS`.
# Hence only m needs to be defined rather than all the states, parameters,
# and observables of the system.
prob = ODEProblem(sys, [m.x => 0.0], (0.0,10.0), [m.τ => 3.0])
sol = solve(prob);
dx1 = sol[m.RHS][1:3]
dx2 = sol[getproperty(m,:RHS)][1:3]  # access by symbols

# using Plots
# plot(sol, vars=[m.x,m.RHS])
# plot(sol, vars=getproperty.(Ref(m),[:x, :RHS])) # access by symbols
length(dx2) == 3
# output
true
```
"""
function embed_system(m;name)
    @named _sys_embed = ODESystem(Equation[], ModelingToolkit.get_iv(m))
    sys = compose(_sys_embed, [m]; name)
    structural_simplify(sys)
end

"""
    symbol(t)

Extract the inner symbol from a Term or Num object.
"""
function symbol(t::Term); symbol(t.f); end
symbol(num::Num) = symbol(num.val)
symbol(s) = Symbol(s)

"""
    strip_namespace(s)

Omit the part before the first dot.
"""
function strip_namespace(s::String); match(r"[^.₊]+$",s).match; end
function strip_namespace(s::Symbol); Symbol(strip_namespace(string(s))); end

"""
    statesyms(sys::ODYSystem)
    parsyms(sys::ODYSystem)

Extract the basic symbols without namespace of system states and system parameters.
"""
function statesyms(sys::ODESystem); symbol.(states(sys)); end
parsyms(sys::ODESystem) = symbol.(parameters(sys))


# This file is part of Kpax3. License is MIT.

abstract type KData end

"""
# Genetic data

## Description

DNA data and its metadata.

## Fields

* `data` Multiple sequence alignment (MSA) encoded as a binary (UInt8) matrix
* `id` units' ids
* `ref` reference sequence, i.e. a vector of the same length of the original
sequences storing the values of homogeneous sites. SNPs are instead represented
by a value of 29
* `val` vector with unique values per MSA site
* `key` vector with indices of each value

## Details

Let `n` be the total number of units and `ml` be the total number of unique
values observed at SNP `l`. Define m = m1 + ... + mL, where L is the total
number of SNPs.

`data` is a `m`-by-`n` indicator matrix, i.e. `data[j, i]` is `1` if unit `i`
possesses value `j`, `0` otherwise.

The value associated with column `j` can be obtained by `val[j]` while the SNP
position by `findall(ref == 29)[key[j]]`.

## References

Pessia A., Grad Y., Cobey S., Puranen J. S. and Corander J. (2015). K-Pax2:
Bayesian identification of cluster-defining amino acid positions in large
sequence datasets. _Microbial Genomics_ **1**(1).
<http://dx.doi.org/10.1099/mgen.0.000025>.
"""
struct NucleotideData <: KData
  data::Matrix{UInt8}
  id::Vector{String}
  ref::Vector{UInt8}
  val::Vector{UInt8}
  key::Vector{Int}
end

function NucleotideData(settings::KSettings)
  missval = if (length(settings.miss) == 1) && (settings.miss[1] == 0x00)
              0x00
            else
              UInt8('?')
            end

  (data, id, ref) = readfasta(settings.ifile, false, settings.miss, settings.l,
                              settings.verbose, settings.verbosestep)
  (bindata, val, key) = categorical2binary(data, UInt8(127), missval)

  NucleotideData(bindata, id, ref, val, key)
end

"""
# Genetic data

## Description

Amino acid data and its metadata.

## Fields

* `data` Multiple sequence alignment (MSA) encoded as a binary (UInt8) matrix
* `id` units' ids
* `ref` reference sequence, i.e. a vector of the same length of the original
sequences storing the values of homogeneous sites. SNPs are instead represented
by a value of 29
* `val` vector with unique values per MSA site
* `key` vector with indices of each value

## Details

Let `n` be the total number of units and `ml` be the total number of unique
values observed at SNP `l`. Define m = m1 + ... + mL, where L is the total
number of SNPs.

`data` is a `m`-by-`n` indicator matrix, i.e. `data[j, i]` is `1` if unit `i`
possesses value `j`, `0` otherwise.

The value associated with column `j` can be obtained by `val[j]` while the SNP
position by `findall(ref == 29)[key[j]]`.

## References

Pessia A., Grad Y., Cobey S., Puranen J. S. and Corander J. (2015). K-Pax2:
Bayesian identification of cluster-defining amino acid positions in large
sequence datasets. _Microbial Genomics_ **1**(1).
<http://dx.doi.org/10.1099/mgen.0.000025>.
"""
struct AminoAcidData{T} <: KData
  # generalize AminoAcidData to represent generic categorical data. Code must
  # be refactored
  data::Matrix{UInt8}
  id::Vector{String}
  ref::Vector{T}
  val::Vector{T}
  key::Vector{Int}
end

function AminoAcidData(settings::KSettings)
  missval = if (length(settings.miss) == 1) && (settings.miss[1] == 0x00)
              0x00
            else
              UInt8('?')
            end

  (data, id, ref) = readfasta(settings.ifile, true, settings.miss, settings.l,
                              settings.verbose, settings.verbosestep)
  (bindata, val, key) = categorical2binary(data, UInt8(127), missval)

  AminoAcidData(bindata, id, ref, val, key)
end

"""
# CSV data

## Description

Generic data and its metadata.

## Fields

* `data` original data matrix encoded as a binary (UInt8) matrix
* `id` units' ids
* `ref` reference observation, i.e. a vector of the same length of the original
observations storing the values of homogeneous sites. Polymorphisms are instead
represented by the string "."
* `val` vector with unique values per dataset column
* `key` vector with indices of each value

## Details

Let `n` be the total number of units and `ml` be the total number of unique
values observed at polymorphic column `l`. Define m = m1 + ... + mL, where L is
the total number of polymorphic columns.

`data` is a `m`-by-`n` indicator matrix, i.e. `data[j, i]` is `1` if unit `i`
possesses value `j`, `0` otherwise.

The value associated with column `j` can be obtained by `val[j]` while the
polymorphic position by `findall(ref == ".")[key[j]]`.

## References

Pessia A., Grad Y., Cobey S., Puranen J. S. and Corander J. (2015). K-Pax2:
Bayesian identification of cluster-defining amino acid positions in large
sequence datasets. _Microbial Genomics_ **1**(1).
<http://dx.doi.org/10.1099/mgen.0.000025>.
"""
function CategoricalData(settings::KSettings)
  (data, id, ref) = readdata(settings.ifile, ',', settings.misscsv,
                             settings.verbose, settings.verbosestep)
  (bindata, val, key) = categorical2binary(data, "")

  #CategoricalData(bindata, id, ref, val, key)
  # return an AminoAcidData object until the code has been refactored
  AminoAcidData(bindata, id, ref, val, key)
end

#=
struct CategoricalData <: KData
  data::Matrix{UInt8}
  id::Vector{String}
  ref::Vector{String}
  val::Vector{String}
  key::Vector{Int}
end
=#


#=
Copyright (c) 2018-2022 Chris Coey, Lea Kapelevich, and contributors

This Julia package Hypatia.jl is released under the MIT license; see LICENSE
file in the root directory or at https://github.com/chriscoey/Hypatia.jl
=#

insts = OrderedDict()
insts["minimal"] = [
    ((2, false, true, true, true),),
    ((2, false, false, true, true),),
    ((2, true, true, true, true),),
    ((2, true, false, true, true),),
    ((2, false, true, true, false),),
    ((2, false, false, true, false),),
    ((2, true, true, true, false),),
    ((2, true, false, true, false),),
    ((2, false, false, false, false),),
    ((2, true, false, false, false),),
    ]
insts["fast"] = [
    ((5, false, true, true, true),),
    ((5, false, false, true, false),),
    ((5, true, true, true, false),),
    ((5, true, false, true, true),),
    ((5, false, true, false, false),),
    ((5, false, false, false, false),),
    ((5, true, true, false, false),),
    ((5, true, false, false, false),),
    ((20, false, true, true, false),),
    ((20, false, false, true, true),),
    ((20, true, true, true, true),),
    ((20, true, false, true, false),),
    ((20, false, true, false, false),),
    ((20, false, false, false, false),),
    ((20, true, true, false, false),),
    ((20, true, false, false, false),),
    ((100, false, true, false, false),),
    ((100, false, false, false, false),),
    ]
insts["various"] = [
    ((50, false, true, true, true),),
    ((50, false, false, true, true),),
    ((50, true, true, true, false),),
    ((50, true, false, true, false),),
    ((50, false, true, false, false),),
    ((50, false, false, false, false),),
    ((50, true, true, false, false),),
    ((50, true, false, false, false),),
    ((100, false, true, true, true),),
    ((100, false, false, true, false),),
    ((100, true, true, true, false),),
    ((100, true, false, true, true),),
    ((100, false, true, false, false),),
    ((100, false, false, false, false),),
    ((100, true, true, false, false),),
    ((100, true, false, false, false),),
    ((200, false, true, false, false),),
    ((200, false, false, false, false),),
    ((200, false, true, true, false),),
    ((200, false, false, true, false),),
    ((200, true, true, true, true),),
    ((200, true, false, true, true),),
    ((400, false, true, false, false),),
    ((400, false, false, false, false),),
    ((400, false, true, true, false),),
    ((400, false, false, true, true),),
    ((400, true, true, true, true),),
    ((400, true, false, true, false),),
    ]
return (NearestPSDJuMP, insts)


module PlasmaBenchmarks

using Plasma
using BSON: @save

include("two-stream.jl")
include("weibel.jl")
# include("1d1v.jl")
# include("3d3v.jl")

end


module Js2x

# Write your package code here.

export Profile, ProfileDataPoint, point_counts, avg_brightness, laser_on_times, read_edger_board

const MAX_LENGTH = 21
const MAX_PROFILES = MAX_LENGTH * 12
const MAX_VERTICAL = 243
mutable struct ProfileDataPoint
    x::Int32
    y::Int32
    brightness::Int32
end
mutable struct Profile
	sequenceNumber::Int32
	location::Int32
    sendLocation::Int32
	laserOnTime::Int32
	timeInHead::Int32
	inputs::Int32
	flags::Int32
	laserIndex::Int32
	reserved2::Int32
	numberPoints::Int32
	data::Vector{ProfileDataPoint}
end

point_counts(profiles) = map(prof->prof.numberPoints,profiles)
laser_on_times(profiles) = map(prof->prof.laserOnTime,profiles)
avg_brightness(profiles) =  map(prof->mean(map(pd->pd.brightness,prof.data)),profiles)

function read_profile(file)
    sequenceNumber = read(file,Int32)
    location = read(file,Int32)
    sendLocation = read(file,Int32)
    laserOnTime = read(file,Int32)
    timeInHead = read(file,Int32)
    inputs = read(file,Int32)
    flags = read(file,Int32)
    laserIndex = read(file,Int32)
    reserved2 = read(file,Int32)
    numberPoints = read(file,Int32)
    data = Vector{ProfileDataPoint}(undef,numberPoints)
    for i = 1:MAX_VERTICAL
        x = read(file,Int32)
        y = read(file,Int32)
        brightness = read(file,Int32)
        if (i <= numberPoints)
            data[i] = ProfileDataPoint(x,y,brightness)
        end
    end
    return Profile(sequenceNumber,location,sendLocation,laserOnTime,timeInHead,inputs,flags,laserIndex,reserved2,numberPoints,data)
end

function read_edger_board(filename)
    file = open(filename)
    seek(file,796184)
    profiles = Array{Profile}(undef,MAX_PROFILES,2)
    for i=1:MAX_PROFILES
        for j = 1:2
            profiles[i,j] = read_profile(file)
        end
    end
    return profiles
end

end


"""
TODO
"""
mutable struct HyperParameters{Fl<:AbstractFloat}
    num::Int
    names::Vector{String}
    # Poistion of the hyperparameter on the constrained and uncontrained values
    position::Dict{String,Int}
    # Minimizer vector corresponds to which hyperparameter
    minimizer_hyperparameter_position::Dict{Int,String}
    constrained_values::Vector{Fl}
    unconstrained_values::Vector{Fl}
    fixed_constrained_values::Dict{String,Fl}

    function HyperParameters{Fl}(names::Vector{String}) where Fl
        num = length(names)
        positions = Dict(names .=> collect(1:num))
        minimizer_hyperparameter_position = Dict{Int,String}()
        constrained_values = fill(Fl(NaN), num)
        unconstrained_values = fill(Fl(NaN), num)
        fixed_constrained_values = Dict{String,Fl}()
        return new{Fl}(
            num,
            names,
            positions,
            minimizer_hyperparameter_position,
            constrained_values,
            unconstrained_values,
            fixed_constrained_values,
        )
    end
end

function has_hyperparameter(model::StateSpaceModel)
    return :hyperparameters in fieldnames(typeof(model))
end

function register_unconstrained_values!(
    model::StateSpaceModel, unconstrained_values::Vector{Fl}
) where Fl
    return register_unconstrained_values!(model.hyperparameters, unconstrained_values)
end
function register_unconstrained_values!(
    hyperparameters::HyperParameters, unconstrained_values::Vector{Fl}
) where Fl
    for (i, val) in enumerate(unconstrained_values)
        update_unconstrained_value!(
            hyperparameters, hyperparameters.minimizer_hyperparameter_position[i], val
        )
    end
    return hyperparameters
end

function fill_minimizer_hyperparameter_position!(model::StateSpaceModel)
    fill_minimizer_hyperparameter_position!(model.hyperparameters)
    return model
end

function fill_minimizer_hyperparameter_position!(hyperparameters::HyperParameters)
    # Reset minimizer hyperparameter_position
    hyperparameters.minimizer_hyperparameter_position = Dict{Int,String}()
    i = 1
    for name in get_names(hyperparameters)
        if is_free(name, hyperparameters)
            hyperparameters.minimizer_hyperparameter_position[i] = name
            i += 1
        end
    end
    return hyperparameters
end

function handle_optim_initial_hyperparameters(model::StateSpaceModel)
    # Register default initial hyperparameters
    initial_hyperparameters!(model)
    # Possibly fix some user specified initial hyperparameters
    fix_hyperparameters!(model)
    # Associate the position of each index of the optimizer vector with a hyperparameter
    fill_minimizer_hyperparameter_position!(model)
    # Register the unconstrained initial_parameters
    unconstrain_hyperparameters!(model)
    # Return the unconstrained values of the initial_parameters
    return get_free_unconstrained_values(model)
end

get_hyperparameters(model::StateSpaceModel) = model.hyperparameters

"""
    get_names(model::StateSpaceModel)

Get the names of the hyperparameters registered on a `StateSpaceModel`.
"""
get_names(model::StateSpaceModel) = get_names(model.hyperparameters)

get_names(hyperparameters::HyperParameters) = hyperparameters.names

"""
    number_hyperparameters(model::StateSpaceModel)

Get the number of hyperparameters registered on a `StateSpaceModel`.
"""
function number_hyperparameters(model::StateSpaceModel)
    return number_hyperparameters(model.hyperparameters)
end

number_hyperparameters(hyperparameters::HyperParameters) = hyperparameters.num

position(str::String, hyperparameters::HyperParameters) = hyperparameters.position[str]

get_position(hyperparameters::HyperParameters) = hyperparameters.position

function get_constrained_values(model::StateSpaceModel)
    return get_constrained_values(model.hyperparameters)
end

function get_constrained_values(hyperparameters::HyperParameters)
    return hyperparameters.constrained_values
end
function get_unconstrained_values(model::StateSpaceModel)
    return get_unconstrained_values(model.hyperparameters)
end

function get_unconstrained_values(hyperparameters::HyperParameters)
    return hyperparameters.unconstrained_values
end

function get_fixed_constrained_values(hyperparameters::HyperParameters)
    return hyperparameters.fixed_constrained_values
end

function get_constrained_value(model::StateSpaceModel, str::String)
    return get_constrained_value(model.hyperparameters, str)
end

function get_constrained_value(hyperparameters::HyperParameters, str::String)
    return hyperparameters.constrained_values[position(str, hyperparameters)]
end

function update_constrained_value!(model::StateSpaceModel, str::String, value)
    return update_constrained_value!(model.hyperparameters, str, value)
end

function update_constrained_value!(hyperparameters::HyperParameters, str::String, value)
    return hyperparameters.constrained_values[position(str, hyperparameters)] = value
end

function get_unconstrained_value(model::StateSpaceModel, str::String)
    return get_unconstrained_value(model.hyperparameters, str)
end

function get_unconstrained_value(hyperparameters::HyperParameters, str::String)
    return hyperparameters.unconstrained_values[position(str, hyperparameters)]
end

function update_unconstrained_value!(model::StateSpaceModel, str::String, value)
    return update_unconstrained_value!(model.hyperparameters, str, value)
end

function update_unconstrained_value!(hyperparameters::HyperParameters, str::String, value)
    return hyperparameters.unconstrained_values[position(str, hyperparameters)] = value
end

function get_free_unconstrained_values(model::StateSpaceModel)
    return get_free_unconstrained_values(model.hyperparameters)
end

function get_free_unconstrained_values(hyperparameters::HyperParameters{Fl}) where Fl
    free_unconstrained_values = Vector{Fl}(
        undef, number_free_hyperparameters(hyperparameters)
    )
    for (k, v) in hyperparameters.minimizer_hyperparameter_position
        free_unconstrained_values[k] = get_unconstrained_value(hyperparameters, v)
    end
    return free_unconstrained_values
end

function has_hyperparameter(str::String, hyperparameters::HyperParameters)
    return haskey(hyperparameters.position, str)
end

function has_hyperparameter(str::Vector{String}, hyperparameters::HyperParameters)
    for s in str
        if !has_hyperparameter(s, hyperparameters)
            return false
        end
    end
    return true
end

"""
    set_initial_hyperparameters!(model::StateSpaceModel,
                                 initial_hyperparameters::Dict{String, <:Real})

Fill a model with user inputed initial points for hyperparameter optimzation.

# Example
```jldoctest
julia> model = LocalLevel(rand(100))
LocalLevel model

julia> get_names(model)
2-element Array{String,1}:
 "sigma2_ε"
 "sigma2_η"

julia> set_initial_hyperparameters!(model, Dict("sigma2_η" => 100.0))
LocalLevel model

julia> model.hyperparameters.constrained_values
2-element Array{Float64,1}:
 NaN
 100.0
```
"""
function set_initial_hyperparameters!(
    model::StateSpaceModel, initial_hyperparameters::Dict{String,<:Real}
)
    for (k, v) in initial_hyperparameters
        # Only update if it has not been updated before.
        if isnan(get_constrained_value(model, k))
            update_constrained_value!(model, k, v)
        end
    end
    return model
end

function is_fixed(str::String, hyperparameters::HyperParameters)
    return haskey(hyperparameters.fixed_constrained_values, str)
end

function number_free_hyperparameters(model::StateSpaceModel)
    return number_free_hyperparameters(model.hyperparameters)
end

function number_free_hyperparameters(hyperparameters::HyperParameters)
    return number_hyperparameters(hyperparameters) -
           number_fixed_hyperparameters(hyperparameters)
end

function is_free(str::String, hyperparameters::HyperParameters)
    return !is_fixed(str, hyperparameters)
end

function number_fixed_hyperparameters(model::StateSpaceModel)
    return number_fixed_hyperparameters(model.hyperparameters)
end

function number_fixed_hyperparameters(hyperparameters::HyperParameters)
    return length(hyperparameters.fixed_constrained_values)
end

"""
    fix_hyperparameters!(model::StateSpaceModel, fixed_hyperparameters::Dict)

Fixes the desired hyperparameters so that they are not considered as decision variables in
the model estimation.

# Example
```jldoctest
julia> model = LocalLevel(rand(100))
LocalLevel model

julia> get_names(model)
2-element Array{String,1}:
 "sigma2_ε"
 "sigma2_η"

julia> fix_hyperparameters!(model, Dict("sigma2_ε" => 100.0))
LocalLevel model

julia> model.hyperparameters.fixed_constrained_values
Dict{String,Float64} with 1 entry:
  "sigma2_ε" => 100.0
```
"""
function fix_hyperparameters! end

function fix_hyperparameters!(model::StateSpaceModel, fixed_hyperparameters::Dict)
    fix_hyperparameters!(model.hyperparameters, fixed_hyperparameters)
    return model
end

function fix_hyperparameters!(
    hyperparameters::HyperParameters{Fl}, fixed_hyperparameters::Dict{String,Fl}
) where Fl
    # assert we have these hyperparameters
    @assert has_hyperparameter(collect(keys(fixed_hyperparameters)), hyperparameters)
    # build dict
    hyperparameters.fixed_constrained_values = fixed_hyperparameters
    return hyperparameters
end

fix_hyperparameters!(model::StateSpaceModel) = fix_hyperparameters!(model.hyperparameters)
function fix_hyperparameters!(hyperparameters::HyperParameters{Fl}) where Fl
    for (name, value) in hyperparameters.fixed_constrained_values
        update_constrained_value!(hyperparameters, name, value)
    end
    return hyperparameters
end

"""
    isfitted(model::StateSpaceModel) -> Bool

Verify if `model` is fitted, i.e., returns `false` if there is at least one `NaN` entry in
the hyperparameters.
"""
function isfitted(model::StateSpaceModel)
    c1 = any(isnan, model.hyperparameters.unconstrained_values)
    c2 = any(isnan, model.hyperparameters.constrained_values)
    if c1 || c2
        return false
    else
        return true
    end
end

# Some special constraint functions
"""
    constrain_box!(model::StateSpaceModel, str::String, lb::Fl, ub::Fl) where Fl

Map a constrained hyperparameter ``\\psi \\in [lb, ub]`` to an unconstrained hyperparameter
``\\psi_* \\in \\mathbb{R}``.

The mapping is
``\\psi = lb + \\frac{ub - lb}{1 + \\exp(-\\psi_{*})}``
"""
function constrain_box!(model::StateSpaceModel, str::String, lb::Fl, ub::Fl) where Fl
    update_constrained_value!(
        model, str, lb + ((ub - lb) / (1 + exp(-get_unconstrained_value(model, str))))
    )
    return model
end

"""
    unconstrain_box!(model::StateSpaceModel, str::String, lb::Fl, ub::Fl) where Fl

Map an unconstrained hyperparameter ``\\psi_* \\in \\mathbb{R}`` to a constrained hyperparameter
``\\psi \\in [lb, ub]``.

The mapping is
``\\psi_* = -\\ln \\frac{ub - lb}{\\psi - lb} - 1``
"""
function unconstrain_box!(model::StateSpaceModel, str::String, lb::Fl, ub::Fl) where Fl
    update_unconstrained_value!(
        model, str, -log((ub - lb) / (get_constrained_value(model, str) - lb) - 1)
    )
    return model
end

"""
    constrain_variance!(model::StateSpaceModel, str::String)

Map a constrained hyperparameter ``\\psi \\in \\mathbb{R}^+`` to an unconstrained hyperparameter
``\\psi_* \\in \\mathbb{R}``.

The mapping is
``\\psi = \\psi_*^2``
"""
function constrain_variance!(model::StateSpaceModel, str::String)
    update_constrained_value!(model, str, (get_unconstrained_value(model, str))^2)
    return model
end

"""
    unconstrain_variance!(model::StateSpaceModel, str::String)

Map an unconstrained hyperparameter ``\\psi_{*} \\in \\mathbb{R}`` to a constrained hyperparameter
``\\psi \\in \\mathbb{R}^+``.

The mapping is ``\\psi_{*} = \\sqrt{\\psi}``.
"""
function unconstrain_variance!(model::StateSpaceModel, str::String)
    update_unconstrained_value!(model, str, sqrt(get_constrained_value(model, str)))
    return model
end

"""
    constrain_identity!(model::StateSpaceModel, str::String)

Map an constrained hyperparameter ``\\psi \\in \\mathbb{R}`` to an unconstrained hyperparameter
``\\psi_* \\in \\mathbb{R}``. This function is necessary to copy values from a location to another
inside `HyperParameters`

The mapping is
``\\psi = \\psi_*``
"""
function constrain_identity!(model::StateSpaceModel, str::String)
    update_constrained_value!(model, str, get_unconstrained_value(model, str))
    return model
end

"""
    unconstrain_identity!(model::StateSpaceModel, str::String)

Map an unconstrained hyperparameter ``\\psi_* \\in \\mathbb{R}`` to a constrained hyperparameter
``\\psi \\in \\mathbb{R}``. This function is necessary to copy values from a location to another
inside `HyperParameters`

The mapping is
``\\psi_* = \\psi``
"""
function unconstrain_identity!(model::StateSpaceModel, str::String)
    update_unconstrained_value!(model, str, get_constrained_value(model, str))
    return model
end


function mean(x::Vector{Real})
    n = size(x, 1)
    if n > 0
        return sum(x) / n
    else
        return 0
    end
end

function _exp_c(x::Float64)
    ccall(:exp, Float64, (Float64,), x)
end

function _log_c(x::Float64)
    ccall(:log, Float64, (Float64,), x)
end

function _log1p_c(x::Float64)
    ccall(:log1p, Float64, (Float64,), x)
end

function _sig_c(x::Float64)
    1 / (1 + _exp_c(-x))
end

function _sig_cplus(x::Float64)
    1 / (1 + _exp_c(x))
end

function _gemmblas(A::Matrix{Float64}, B::Matrix{Float64}, ol_max::Matrix{Float64})
    sa = size(A)
    ol = copy(ol_max)
    ccall(
        ("dgemm_64_", "libopenblas64_"),
        Cvoid,
        (
            Ref{UInt8},
            Ref{UInt8},
            Ref{Int64},
            Ref{Int64},
            Ref{Int64},
            Ref{Float64},
            Ptr{Float64},
            Ref{Int64},
            Ptr{Float64},
            Ref{Int64},
            Ref{Float64},
            Ptr{Float64},
            Ref{Int64},
        ),
        'T',
        'N',
        sa[2],
        sa[2],
        sa[1],
        1.0,
        A,
        max(1, stride(A, 2)),
        B,
        max(1, stride(B, 2)),
        -one(Float64),
        ol,
        max(1, stride(ol, 2)),
    )
    return ol::Matrix{Float64}
end

function _gemvblas(
    A::Matrix{Float64},
    x::Vector{Float64},
    cons::Vector{Float64},
    sizex::Int64,
)
    ccall(
        ("dgemv_64_", "libopenblas64_"),
        Cvoid,
        (
            Ref{UInt8},
            Ref{Int64},
            Ref{Int64},
            Ref{Float64},
            Ptr{Float64},
            Ref{Int64},
            Ptr{Float64},
            Ref{Int64},
            Ref{Float64},
            Ptr{Float64},
            Ref{Int64},
        ),
        'N',
        size(A, 1),
        sizex,
        1.0,
        A,
        max(1, stride(A, 2)),
        x,
        1,
        -1.0,
        cons,
        1,
    )
    return cons
end

function _gemvblasT(A::Matrix{Float64}, x::Vector{Float64}, sizeA2::Int64)
    cons = zeros(Float64, sizeA2)
    ccall(
        ("dgemv_64_", "libopenblas64_"),
        Cvoid,
        (
            Ref{UInt8},
            Ref{Int64},
            Ref{Int64},
            Ref{Float64},
            Ptr{Float64},
            Ref{Int64},
            Ptr{Float64},
            Ref{Int64},
            Ref{Float64},
            Ptr{Float64},
            Ref{Int64},
        ),
        'T',
        size(A, 1),
        size(A, 2),
        1.0,
        A,
        max(1, stride(A, 2)),
        x,
        1,
        -1.0,
        cons,
        1,
    )
    return cons
end

function _combinations(T::Int64)
    comb = vcat([[[t, i] for i = (t+1):T] for t = 1:T]...)
end

function _myqle(x::Vector{Float64}, lx::Int64, k::Int64)
    y = ones(k) * Inf
    n = 1
    y[1] = copy(x[1])
    for i = 2:lx
        current = copy(x[i])
        k2 = 1
        while k2 <= k
            if y[k2] > current
                y[(k2+1):n] .= y[k2:(n-1)]
                y[k2] = copy(current)
                k2 = k + 1
                if n < k
                    n += 1
                end
            end
            k2 += 1
        end
    end
    return y[k]
end

function _mycopy(NH::Neighbourhood, NH_new::Neighbourhood)
    NH_new.x = copy(NH.x)
    NH_new.f = copy(NH.f)
    NH_new.obj = copy(NH.obj)
    NH_new.infeas = copy(NH.infeas)
    NH_new.ol = copy(NH.ol)
    NH_new.iu = copy(NH.iu)
    return NH_new::Neighbourhood
end

function _comp_f(NH::Neighbourhood, OptFeas::Float64)
    return (1 - OptFeas) * (sum(NH.infeas + NH.ol) + NH.iu) - OptFeas * (minimum(NH.obj))
end

function _myqle_simp(x::Vector{Float64}, ind::Vector{Float64})
    R = size(x, 1)
    alphaR = Int.(ceil.(R .* (ind)))
    alphaR[findall(alphaR .< 1)] .= 1
    alphaR[findall(alphaR .> R)] .= R
    αQle = Inf
    xsorted = sort(x)
    return sort(x)[alphaR]
end

function _cutR(
    x;
    start = "minimum",
    stop = "maximum",
    nBins = 2,
    returnBreaks = true,
    returnMidPts = false,
)
    if (start == "minimum")
        start = minimum(x)
    end
    if (stop == "maximum")
        stop = maximum(x)
    end
    bw = (stop - start) / (nBins - 1)
    midPts = zeros(nBins)
    for i = 1:nBins
        midPts[i] = start + (i - 1) * bw
    end
    breaks = collect(range(start - (bw / 2); length = nBins + 1, stop = stop + (bw / 2)))
    y = zeros(size(x, 1))
    for j = 1:size(x, 1)
        for i = 1:nBins
            if (x[j] >= breaks[i]) && (x[j] < breaks[i+1])
                y[j] = i
            end
            if i == nBins && x[j] == breaks[i+1]
                y[j] = i
            end
        end
    end
    if (returnBreaks == true || returnMidPts == true)
        if returnMidPts == false
            return (Int.(y), breaks)
        elseif returnBreaks == false
            return (Int.(y), midPts)
        else
            return (Int.(y), breaks, midPts)
        end
    else
        return Int.(y)
    end
end

function _desume_pars(
    pars::DataFrames.DataFrame;
    model = "2PL", #1PL, 2PL, 3PL
    parametrization = "at-b",
) #"at-b, at-ab, at+b, at+ab")
    n_items = size(pars, 1)
    if (model == "1PL")
        ("b" in names(pars)) ||
            ("d" in names(pars)) ||
            error("difficulty parameter b or d not defined")
        c = zeros(Float64, n_items)
        a = ones(Float64, n_items)
    elseif (model == "2PL")
        ("a" in names(pars) || "a1" in names(pars)) ||
            error("discrimination parameter a1 or a not defined")
        ("b" in names(pars)) ||
            ("d" in names(pars)) ||
            error("difficulty parameter b or d not defined")
        if "a" in names(pars)
            a = pars.a
        else
            a = pars.a1
        end
        c = zeros(Float64, n_items)
    elseif (model == "3PL")
        ("a1" in names(pars) || "a" in names(pars)) ||
            error("discrimination parameter a1 not defined")
        ("b" in names(pars)) ||
            ("d" in names(pars)) ||
            error("difficulty parameter b or d not defined")
        ("c" in names(pars)) ||
            ("g" in names(pars)) ||
            error("guessing parameter c or g not defined")
        if "a" in names(pars)
            a = pars.a
        else
            a = pars.a1
        end
        if ("c" in names(pars))
            c = pars.c
        else
            c = pars.g
        end
    elseif (model == "grm")
        ("a1" in names(pars) || "a" in names(pars)) ||
            error("discrimination parameter a1 not defined")
    end
    b = zeros(Float64, n_items, 0)
    nb = 0
    # TODO grm
    for n in names(pars)
        if startswith(string(n), "b")
            nb += 1
            b = hcat(b, pars[!, n])
        end
    end
    # nd = 0
    # d = zeros(Float64, size(pars, 1), 1)
    # for n in names(pars)
    #     if startswith(string(n), "d")
    #         nd += 1
    #         d = hcat(d, pars[!, n])
    #     end
    # end
    # if nd > 0
    #     d = mapslices(x -> b .- x, d; dims = 2)
    # end
    a2 = ones(n_items) #default: "at+b" a*theta + b
    if parametrization == "at-b" #a*theta - b
        b = -b
    elseif parametrization == "at-ab" #a*(theta - b)
        a2 = copy(a)
        b = -b
    elseif parametrization == "at+ab" #a*(theta + b)
        a2 = copy(a)
    end
    return a, a2, b, c
end

"""
	fs_to_items(xₜ, n_items, fs_items)

# Description

Ungroup items grouped by friend sets.

# Arguments

- **`xₜ::Vector{Float64}`** : Required. grouped items 0-1 vector.
- **`n_items::Int64`** : Required. Number if items.
- **`fs_items::Vector{Vector{Int64}}`** : Required. vector of items included in the friend sets.

# Output

- **`x_Iₜ::Vector{Float64}`** Returns a 0-1 vector at item level.

"""
function fs_to_items(xₜ::Vector{Float64}, n_items::Int64, fs_items::Vector{Vector{Int64}})
    xₜ_taken = findall(xₜ .== one(Float64))
    x_Iₜ = zeros(Float64, n_items)
    for i in xₜ_taken
        x_Iₜ[fs_items[i]] .= one(Float64)
    end
    return x_Iₜ::Vector{Float64}
end

"""
	item_char(pars::DataFrames.DataFrame, theta::Vector{Float64}; model = "2PL", parametrization = "at-b", D = 1, derivatives = false)

# Description

Compute the item characteristic function (probability of correct answer).

# Arguments

- **`pars::DataFrames.DataFrame`** : Required. DataFrame with item parameters (difficulty: b or d, discrimination: a or a1, guessing: c or g).
- **`theta::Vector{Float64}`** : Required. Vector of ability points.
- **`model`** : Optional. Default: `"2PL`". Values:  `"1PL`", `"2PL`", `"3PL`". IRT model.
- **`parametrization`** : Optional. Default: \"at-b\". Values:  \"at-b\", \"at-ab\", \"at+b\", \"at+ab\". IRT model parametrization. Ex: at-b is ``Da(\\theta-b)``.
- **`D`** : Optional. Default: 1.
- **`derivatives`** : Optional. Default: false. If it's true compute the derivatives. Ow a `zeros(0,0)` matrix is returned.

# Output

- **`p::Matrix{Float64}`**: Matrix of probabilites.
- **`pder::Matrix{Float64}`**: Matrix of derivatives.
"""
function item_char(
    pars::DataFrames.DataFrame,
    theta::Vector{Float64};
    model = "2PL", #1PL, 2PL, 3PL
    parametrization = "at-b", #"at-b, at-ab, at+b, at+ab"
    D = 1,#any number (1, 1.7)
    derivatives = false,
)

    n_items = size(pars, 1)
    pder = zeros(0, 0)
    a, a2, b, c = _desume_pars(pars; model = model, parametrization = parametrization)
    nb = size(b, 2)
    N = size(theta, 1)
    p =
        _sig_c.([
            D * (a[i] * theta[n] + a2[i] * b[i, j]) for i = 1:n_items, n = 1:N, j = 1:nb
        ])
    #pder =  eachslice(((1 .- p) .* p), dims = 1) .* a

    if model != "grm"
        p = c .+ ((1 .- c) .* p)
    else
        # TODO grm
        #generalized PCM, does not work
        # p[:, :, 1] = 1 .- p[:, :, 1]
        # pder[:, :, 1] = .- pder[:, :, 1]
        # for k = (nb - 1) :  -1 : 2
        # 	p[:, :, k] = p[:, :, k] - p[:, :, k-1]
        # 	pder[:, :, k] = pder[:, :, k] - pder[:, :, k-1]
        # end
    end
    if derivatives
        pder = mapslices(x -> (1 .- x) .* ((x .- c) ./ (1 .- c)) .* a, p; dims = 1)
    end
    return p, pder
end

"""
	item_char(pars::DataFrames.DataFrame, theta::Float64; model = "2PL", parametrization = "at-b", D = 1, derivatives = false)

# Description

Compute the item characteristic function (probability of correct answer).

# Arguments

- **`pars::DataFrames.DataFrame`** : Required. DataFrame with item parameters (difficulty: b or d, discrimination: a or a1, guessing: c or g).
- **`theta::Float64`** : Required. Ability point.
- **`model`** : Optional. Default: `"2PL`". Values:  `"1PL`", `"2PL`", `"3PL`". IRT model.
- **`parametrization`** : Optional. Default: \"at-b\". Values:  \"at-b\", \"at-ab\", \"at+b\", \"at+ab\". IRT model parametrization. Ex: at-b is ``Da(\\theta-b)``.
- **`D`** : Optional. Default: 1.
- **`derivatives`** : Optional. Default: false. If it's true compute the derivatives. Ow a `zeros(0,0)` matrix is returned.

# Output

- **`p::Matrix{Float64}`**: Matrix of probabilites.
- **`pder::Matrix{Float64}`**: Matrix of derivatives.
"""
function item_char(
    pars::DataFrames.DataFrame,
    theta::Float64;
    model = "2PL", #1PL, 2PL, 3PL
    parametrization = "at-b", #"at-b, at-ab, at+b, at+ab"
    D = 1,#any number (1, 1.6)
    derivatives = false,
)
    return item_char(
        pars,
        [theta],
        model = model, #1PL, 2PL, 3PL
        parametrization = parametrization, #"at-b, at-ab, at+b, at+ab"
        D = D,#any number (1, 1.6)
        derivatives = derivatives,
    )
end

"""
    item_info(
        pars::DataFrames.DataFrame,
        theta::Vector{Float64};
        model = "2PL", #1PL, 2PL, 3PL, grm
        parametrization = "at-b", #"at-b, at-ab, at+b, at+ab"
        D = 1,
    )

# Description

Compute the item Fisher information function.

# Arguments

- **`pars::DataFrames.DataFrame`** : Required. DataFrame with item parameters (difficulty: b or d, discrimination: a or a1, guessing: c or g).
- **`theta::Vector{Float64}`** : Required. Vector of ability points.
- **`model`** : Optional. Default: `"2PL`". Values:  `"1PL`", `"2PL`", `"3PL`". IRT model.
- **`parametrization`** : Optional. Default: \"at-b\". Values:  \"at-b\", \"at-ab\", \"at+b\", \"at+ab\". IRT model parametrization. Ex: at-b is ``Da(\\theta-b)``.
- **`D`** : Optional. Default: 1.

# Output

- **`i::Matrix{Float64}`**: Matrix of the item information.
"""
function item_info(
    pars::DataFrames.DataFrame,
    theta::Vector{Float64};
    model = "2PL", #1PL, 2PL, 3PL, grm
    parametrization = "at-b", #"at-b, at-ab, at+b, at+ab"
    D = 1,
) #true, false

    n_items = size(pars, 1)
    a, a2, b, c = _desume_pars(pars; model = model, parametrization = parametrization)
    p, pder = item_char(
        pars,
        theta;
        model = model,
        parametrization = parametrization,
        derivatives = true,
    )
    nb = size(p, 3)
    if model != "grm"
        #i = (a.^2 ) .* ((1 .- p) ./ p) .* ((p .- c) ./ (1 .- c)).^2
        i = pder .^ 2 ./ (p .* (1 .- p))
    else
        error("GRM still not supported.")
        #i = pder .^ 2 ./ p
    end
    i = sum(i, dims = 3)[:, :, 1]
    return i::Matrix{Float64}
end

"""
    item_info(
        pars::DataFrames.DataFrame,
        theta::Float64;
        model = "2PL", #1PL, 2PL, 3PL, grm
        parametrization = "at-b", #"at-b, at-ab, at+b, at+ab"
        D = 1,
    )

# Description

Compute the item Fisher information function.

# Arguments

- **`pars::DataFrames.DataFrame`** : Required. DataFrame with item parameters (difficulty: b or d, discrimination: a or a1, guessing: c or g).
- **`theta::Float64`** : Required. Vector of ability points.
- **`model`** : Optional. Default: `"2PL`". Values:  `"1PL`", `"2PL`", `"3PL`". IRT model.
- **`parametrization`** : Optional. Default: \"at-b\". Values:  \"at-b\", \"at-ab\", \"at+b\", \"at+ab\". IRT model parametrization. Ex: at-b is ``Da(\\theta-b)``.
- **`D`** : Optional. Default: 1.

# Output

- **`i::Matrix{Float64}`**: Matrix of the item information.
"""
function item_info(
    pars::DataFrames.DataFrame,
    theta::Float64;
    model = "2PL", #1PL, 2PL, 3PL, grm
    parametrization = "at-b", #"at-b, at-ab, at+b, at+ab"
    D = 1,
) #true, false
    return item_info(pars, [theta], model = model, parametrization = parametrization, D = D)[
        :,
        1,
        :,
    ]
end

function student_likelihood(
    f::Float64,
    r::Vector{Float64},
    i_index::Vector{Int64},
    a::Vector{Float64},
    b::Vector{Float64},
    theta::Float64;
    logL = false,
)
    n_items = size(a, 1)
    likel = 0
    for i in i_index
        p = b[i] + a[i] * theta
        likel += r[i] * p - _log1p_c(_exp_c(p))
    end
    return likel::Float64
end

include("resp_gen.jl")