################################################################################
#
#  NfOrd/ResidueRing.jl : Quotients of maximal orders of number fields
#
# This file is part of Hecke.
#
# Copyright (c) 2015, 2016: Claus Fieker, Tommy Hofmann
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#
# (C) 2016 Tommy Hofmann
#
################################################################################

export NfOrdQuoRing, NfOrdQuoRingElem, quo, *, -, ==, deepcopy, divrem,
       gcd, inv, parent, show, divexact, isone, iszero, howell_form,
       strong_echelon_form, triangularize, det, euclid, xxgcd, annihilator

################################################################################
#
#  Assert
#
################################################################################

add_assert_scope(:NfOrdQuoRing)

set_assert_level(:NfOrdQuoRing, 0)

################################################################################
#
#  Field access
#
################################################################################

elem_type(::Type{NfOrdQuoRing}) = NfOrdQuoRingElem

elem_type(::NfOrdQuoRing) = NfOrdQuoRingElem

base_ring(Q::NfOrdQuoRing) = Q.base_ring

ideal(Q::NfOrdQuoRing) = Q.ideal

basis_mat(Q::NfOrdQuoRing) = Q.basis_mat

parent(x::NfOrdQuoRingElem) = x.parent

parent_type(::Type{NfOrdQuoRingElem}) = NfOrdQuoRing

################################################################################
#
#  Hashing
#
################################################################################

hash(x::NfOrdQuoRingElem, h::UInt) = hash(x.elem, h)

################################################################################
#
#  Functions to allow polynomial and polynomial ring constructions
#
################################################################################

needs_parentheses(::NfOrdQuoRingElem) = true

isnegative(::NfOrdQuoRingElem) = false

Nemo.promote_rule{S <: Integer}(::Type{NfOrdQuoRingElem},
                                ::Type{S}) = NfOrdQuoRingElem

Nemo.promote_rule(::Type{NfOrdQuoRingElem}, ::Type{fmpz}) = NfOrdQuoRingElem

################################################################################
#
#  Copying
#
################################################################################

Base.deepcopy_internal(x::NfOrdQuoRingElem, dict::ObjectIdDict) =
        NfOrdQuoRingElem(parent(x), Base.deepcopy_internal(x.elem, dict))

#copy(x::NfOrdQuoRingElem) = deepcopy(x)

################################################################################
#
#  I/O
#
################################################################################

function show(io::IO, Q::NfOrdQuoRing)
  print(io, "Quotient of $(Q.base_ring)")
end

function show(io::IO, x::NfOrdQuoRingElem)
  print(io, "$(x.elem)")
end

################################################################################
#
#  Parent object overloading
#
################################################################################

function (O::NfOrdQuoRing)(x::NfOrdElem)
  parent(x) != base_ring(O) && error("Cannot coerce element into the quotient ring")
  return NfOrdQuoRingElem(O, x)
end

function (Q::NfOrdQuoRing)(x::Integer)
  return Q(base_ring(Q)(x))
end

function (Q::NfOrdQuoRing)(x::fmpz)
  return Q(base_ring(Q)(x))
end

################################################################################
#
#  Quotient function
#
# (and standart helpers)
#
################################################################################
doc"""
    quo(O::NfOrd, I::NfOrdIdl) -> NfOrdQuoRing, Map
> The quotient ring $O/I$ as a ring together with the section $M: O/I \to O$.
> The pointwise inverse of $M$ is the canonical projection $O\to O/I$.
"""
function quo(O::NfOrd, I::NfOrdIdl)
  # We should check that I is not zero
  Q = NfOrdQuoRing(O, I)
  f = NfOrdQuoMap(O, Q)
  return Q, f
end

doc"""
    ResidueRing(O::NfOrd, I::NfOrdIdl) -> NfOrdQuoRing
> The quotient ring $O$ modulo $I$ as a new ring.
"""
Nemo.ResidueRing(O::NfOrd, I::NfOrdIdl) = NfOrdQuoRing(O, I)

doc"""
    lift(O::NfOrd, a::NfOrdQuoRingElem) -> NfOrdElem
> Returns a lift of $a$ back to $O$.
"""
function lift(O::NfOrd, a::NfOrdQuoRingElem)
  f = NfOrdQuoMap(O, parent(a))
  return preimage(f, a)
end

################################################################################
#
#  Arithmetic
#
################################################################################

function +(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  parent(x) != parent(y) && error("Elements must have same parents")
  return parent(x)(x.elem + y.elem)
end

function -(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  parent(x) != parent(y) && error("Elements must have same parents")
  return parent(x)(x.elem - y.elem)
end

function -(x::NfOrdQuoRingElem)
  return parent(x)(-x.elem)
end

function *(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  parent(x) != parent(y) && error("Elements must have same parents")
  return parent(x)(x.elem * y.elem)
end

function mul!(z::NfOrdQuoRingElem, x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  mul!(z.elem, x.elem, y.elem)
  mod!(z.elem, parent(z))
  return z
end

function *(x::Integer, y::NfOrdQuoRingElem)
  return parent(y)(x * y.elem)
end

*(x::NfOrdQuoRingElem, y::Integer) = y*x

function *(x::fmpz, y::NfOrdQuoRingElem)
  return parent(x)(x * y.elem)
end

*(x::NfOrdQuoRingElem, y::fmpz) = y*x

function ^(a::NfOrdQuoRingElem, f::fmpz)
  if nbits(f) < 64
    return a^Int(f)
  end
  f==0 && return one(parent(a))
  f==1 && return deepcopy(a)
  if f<0
    f=-f
    a = inv(a)
  end
  b = a^(div(f, 2))
  b *= b
  if isodd(f)
    b *= a
  end
  return b
end

#^(a::NfOrdQuoRingElem, f::Integer) = a^fmpz(f)

function ^(a::NfOrdQuoRingElem, b::Int)
  if b == 0
    return one(parent(a))
  elseif b == 1
    return deepcopy(a)
  else
    if b < 0
      a = inv(a)
      b = -b
    end
    bit = ~((~UInt(0)) >> 1)
    while (UInt(bit) & b) == 0
      bit >>= 1
    end
    z = deepcopy(a)
    bit >>= 1
    while bit != 0
      z = mul!(z, z, z)
      if (UInt(bit) & b) != 0
        z = mul!(z, z, a)
      end
      bit >>= 1
    end
    return z
  end
end

################################################################################
#
#  Special elements
#
################################################################################

iszero(x::NfOrdQuoRingElem) = iszero(x.elem)

isone(x::NfOrdQuoRingElem) = isone(x.elem)

function one(Q::NfOrdQuoRing)
  return Q(one(Q.base_ring))
end

function zero(Q::NfOrdQuoRing)
  return Q(zero(Q.base_ring))
end

################################################################################
#
#  Equality
#
################################################################################

==(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem) = x.elem == y.elem

################################################################################
#
#  Exact division
#
################################################################################

function divexact(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  b, z = isdivisible(x, y)
  @assert b
  return z
end

function isdivisible(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  parent(x) != parent(y) && error("Elements must have same parents")

  iszero(y) && error("Dividing by zero")

  if iszero(x)
    return true, zero(parent(x))
  end

  R = parent(x)
  d = degree(base_ring(R))
  # We cannot solve with non-square matrices.
  # Thus build the matrix
  # ( 1   x    0  )
  # ( 0  M_y   I  )
  # ( 0  M_I   0  ).
  # Compute the UPPER RIGHT HNF ->
  # ( 1   0   u )
  # ( *   *   * )
  # u will be the coefficient vector of the quotient

  V = R.tmp_div
  A = representation_matrix(y.elem)
  B = parent(x).basis_mat

  V[1, 1] = 1

  a = elem_in_basis(x.elem, Val{false})

  for i in 1:d
    V[1, 1 + i] = a[i]
  end

  _copy_matrix_into_matrix(V, 2, 2, A)   # this really is a copy
  _copy_matrix_into_matrix(V, 2 + d, 2, B) # this really is a copy

  for i in 1:d
    V[1 + i, d + 1 + i] = 1
  end

  hnf_modular_eldiv!(V, minimum(parent(x).ideal))

  for i in 2:(d + 1)
    if !iszero(V[1, i])
  #if !iszero(sub(V, 1:1, 2:(d + 1)))
      ccall((:fmpz_mat_zero, :libflint), Void, (Ptr{fmpz_mat}, ), &V)
      return false, zero(parent(x))
    end
  end

  z = R(-base_ring(R)(fmpz[ V[1, i] for i in (d + 2):(2*d + 1)])) # V[1, i] is always a copy

  ccall((:fmpz_mat_zero, :libflint), Void, (Ptr{fmpz_mat}, ), &V)

  @hassert :NfOrdQuoRing 1 z*y == x
  return true, z
end

################################################################################
#
#  Strong exact division
#
################################################################################

function _divexact_strong(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  n = euclid(x)
  m = euclid(y)
  @hassert :NfOrdQuoRing 1 mod(n, m) == 0
  target = divexact(n, m)

  #println("target valuation: $target")
  #println("doing first an ordinary divexact with\n $x \n and \n $y")
  q0 = divexact(x, y)
  #println("valuation of first quotient: $(euclid(q0))")

  if euclid(q0) == target
    return q0
  else
    i = annihilator(y)
    #println("generator of annihilator: $i")

    q = q0 + rand(parent(x))*i

    k = 0
    while euclid(q) != target
      k += 1
      q = q0 + rand(parent(x))*i
      #println("current valuation $(euclid(q))")
      if k > 500
        error("Could not find proper quotion for strong division")
      end
    end
  end

  @hassert :NfOrdQuoRing 1 q*y == x
  @hassert :NfOrdQuoRing 1 euclid(q) *euclid(y) == euclid(x)

  return q
end

################################################################################
#
#  Inverse element
#
################################################################################

function inv(x::NfOrdQuoRingElem)
  return divexact(one(parent(x)), x)
end

################################################################################
#
#  Euclidean function
#
################################################################################

function euclid(x::NfOrdQuoRingElem)
  if iszero(x)
    return fmpz(-1)
  end

  U = parent(x).tmp_euc

  d = degree(base_ring(parent(x)))

  _copy_matrix_into_matrix(U, 1, 1, representation_matrix(x.elem))
  _copy_matrix_into_matrix(U, d + 1, 1, parent(x).basis_mat)

  hnf_modular_eldiv!(U, parent(x).ideal.minimum)

  z = fmpz(1)

  for i in 1:degree(base_ring(parent(x)))
    mul!(z, z, U[i, i])
  end

  @hassert :NfOrdQuoRing 1 z == norm(ideal(parent(x.elem), x.elem) + parent(x).ideal)

  return z
end

################################################################################
#
#  Division with remainder
#
################################################################################

function divrem(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  #q = rand(parent(x))
  #r = x - q*y
  #e = euclid(y)

  # This should be only one case and don't do the try/catch crap
  # Write a proper _is_divisible function

  #if e == 1
  #  q = x*inv(y)
  #  r = x - q*y
  #  @hassert :NfOrdQuoRing 1 iszero(x - q*y)
  #  @hassert :NfOrdQuoRing 1 euclid(r) < e
  #  return q, r
  #end

  #try q = divexact(x, y)
  #  r = x - q*y
  #  @hassert :NfOrdQuoRing 1 iszero(x - q*y)
  #  @hassert :NfOrdQuoRing 1 euclid(r) < e
  #  return q, r
  #catch
  #end

  b, q = isdivisible(x, y)
  if b
    return q, zero(parent(x))
  end

  e = euclid(y)

  cnt = 0
  while true
    cnt += 1
    q = rand(parent(x))
    r = x - q*y
    if euclid(r) < e
      return q, r
    end
    if cnt > 1000
      error("Something odd in divrem for $x $y $(parent(x))")
    end
  end

  @hassert :NfOrdQuoRing 1 euclid(r) < e
end

################################################################################
#
#  Random elements
#
################################################################################

function rand(Q::NfOrdQuoRing)
  A = basis_mat(Q)
  B = basis(base_ring(Q))
  z = rand(fmpz(1):A[1,1]) * B[1]

  for i in 2:rows(A)
    z = z + rand(fmpz(1):A[i, i]) * B[i]
  end

  return Q(z)
end

################################################################################
#
#  Annihilator
#
################################################################################

function annihilator(x::NfOrdQuoRingElem)
  I = parent(x).ideal
  O = base_ring(parent(x))
  d = degree(O)
  f = NfOrdQuoMap(O, parent(x))
  U = parent(x).tmp_ann
  # We first compute a Z-basis of the annihilator ideal
  # The basis is the kernel of the following matrix
  # ( M_I )
  # ( M_x )
   _copy_matrix_into_matrix(U, 1, 1, representation_matrix(x.elem))
   _copy_matrix_into_matrix(U, d + 1, 1, I.basis_mat)

  m = _kernel(U)
  I = ideal(O, _hnf_modular_eldiv(sub(m, 1:degree(O), 1:degree(O)),
                                  minimum(I), :lowerleft))
  z = f(I)
  #@assert iszero(z*x)
  return z
end

################################################################################
#
#  GCD
#
################################################################################

function gcd(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  Q = parent(x)
  O = base_ring(Q)

  if iszero(x)
    if iszero(y)
      return zero(Q)
    else
      return y
    end
  else
    if iszero(y)
      return x
    end
  end

  I = ideal(O, x.elem) + ideal(O, y.elem)

  f = NfOrdQuoMap(O, parent(x))

  return f(I)
end

################################################################################
#
#  Extended extended GCD
#
################################################################################

function xxgcd(x::NfOrdQuoRingElem, y::NfOrdQuoRingElem)
  Q = parent(x)
  O = base_ring(Q)

  d = degree(O)

  if iszero(x)
    return deepcopy(y), Q(O(0)), Q(O(1)), Q(O(-1)), Q(O(0))
  elseif iszero(y)
    return deepcopy(x), Q(O(1)), Q(O(0)), Q(O(0)), Q(O(1))
  end

  g = gcd(x, y)

  e = _divexact_strong(x, g)
  f = _divexact_strong(y, g)

  M_e = representation_matrix(e.elem)
  M_f = representation_matrix(f.elem)

  M_I = Q.basis_mat

  # let us build
  # ( 1  Q(1) 0  0 )
  # ( 0  M_e  I  0 )
  # ( 0  M_f  0  I )
  # ( 0  M_I  0  0 )

  a = elem_in_basis(Q(O(1)).elem, Val{false})

  V = parent(x).tmp_xxgcd

  V[1, 1] = 1

  for i in 1:d
    V[1, 1 + i] = a[i]
  end

  Hecke._copy_matrix_into_matrix(V, 2, 2, M_e)
  Hecke._copy_matrix_into_matrix(V, d + 2, 2, M_f)
  Hecke._copy_matrix_into_matrix(V, 2*d + 2, 2, M_I)

  for i in 1:2*d
    V[1+i, 1 + d + i] = 1
  end

  #U = V

  hnf_modular_eldiv!(V, minimum(Q.ideal))::fmpz_mat

  u = Q(-O([ V[1,i] for i in (d + 2):(2*d + 1)]))
  v = Q(-O([ V[1,i] for i in (2*d + 2):(3*d + 1)]))

  @hassert :NfOrdQuoRing 1 Q(O(1)) == u*e - (v*(-f))

  ccall((:fmpz_mat_zero, :libflint), Void, (Ptr{fmpz_mat}, ), &V)

  return g, u, v, -f, e
end

function (M::Generic.MatSpace{NfOrdQuoRingElem})(x::Generic.Mat{NfOrdElem})
  z = map(base_ring(M), x.entries)::Array{NfOrdQuoRingElem, 2}
  return M(z)::Generic.Mat{NfOrdQuoRingElem}
end
################################################################################
#
#  Hensel Lifting
#
################################################################################
function lift(R::NmodPolyRing, a::fq_nmod)
  f = R()
  for i=0:degree(parent(a))-1
    setcoeff!(f, i, _get_coeff_raw(a, i))
  end
  return f
end

function (Zx::FmpzPolyRing)(a::nf_elem)
  b = Zx()
  @assert denominator(a) == 1
  for i=0:a.elem_length
    setcoeff!(b, i, numerator(coeff(a, i)))
  end
  return b
end

#missing:
#  assertion on the lifting
#  filtering of the roots (maybe we don't want all?)
#  find all/ one/ few roots
#  break down in various modules:
#  - find powers of ideal
#  - find LLL/HNF basis
#  - lifting?
#  use ResRing(Poly) instead of doing % pgg?
#  an open variant where k is increased until we have a root?

function _hensel(f::Generic.Poly{nf_elem}, p::Int, k::Int; max_roots::Int = degree(f))
  k = max(k, 2)
  #assumes f squarefree
  #assumes constant_coefficient(f) != 0
  ZX, X = FlintZZ["X"]
  Rp = ResidueRing(FlintZZ, p, cached=false)
  Rpt, t = PolynomialRing(Rp, "t", cached=false)
  K = base_ring(f)

  gp = Rpt(K.pol)
  #to avoid embarrasment...
  @assert !iszero(discriminant(gp))

  #find the prime ideal - as I don't want to use orders, this is
  #fun (computing a max order just for this is wasteful)
  #fun fact: if g = prod g_i mod p^k, then P_i^k = <p^k, g_i>
  #so instead of powering, and simplify and such, lets write it down
  lp = factor(gp).fac
  if length(lp) > 1
    g1 = lift(ZX, first(keys(lp)))
    gg = hensel_lift(ZX(K.pol), g1, fmpz(p), k)
  else
    gg = ZX(K.pol)
  end
  # now for all i<= k, <p^i, K(gg)+p^i> is a normal presentation for
  #                                     the prime ideal power.
  #later we'll get the HNF matrix for selected powers as well

  #set up the mod p data:
  #need FiniteField as I need to factor (roots)
  S = FqNmodFiniteField(first(keys(lp)), :z, false)
  ST, T = PolynomialRing(S,"T", cached=false)
  fp = ST([S(Rpt(coeff(f, i))) for i=0:degree(f)])
  rt = roots(fp)
  #we're going to lift the roots - and for efficiency 1/f'(rt) as well:
  fps = derivative(fp)
  irt = [inv(fps(x)) for x= rt]

  # this is in the finite field, but now I need it in the res. ring.
  # in ZX for ease of transport.
  RT = [lift(ZX, lift(Rpt, x)) for x = rt]
  IRT = [lift(ZX, lift(Rpt, x)) for x = irt]

  #the den: ala Kronnecker:
  den = ZX(derivative(K.pol)(gen(K)))
  iden = inv(derivative(K.pol)(gen(K)))

  #for the result, to check for stabilising

  res = nf_elem[K(0) for x = RT]
  rs = nf_elem[]

  #we'll be working in (Z/p^k)[t]/gg

  #an "optimal" lifting chain:
  pr = [k]
  while k>1
    k = div(k+1, 2)
    push!(pr, k)
  end
  pr = reverse(pr)

  #lets start:

  for i=2:length(pr)
    pp = fmpz(p)^pr[i]
    Q = ResidueRing(FlintZZ, pp, cached=false)
    Qt, t = PolynomialRing(Q,"t", cached=false)

    #possibly this should be done with max precision and then adjusted down
    #the poly mod P^??
    fp = [Qt(ZX(coeff(f, k))) for k=0:degree(f)]

    #we need to evaluate fp and fp' at the roots (later)
    #given that we evaluate "by hand" we don't need polynomials

    pgg = Qt(gg) #we'll do the reductions by hand - possibly not optimal

    #the lattice for reco:
    n = degree(K)
    M = zero_matrix(FlintZZ, n, n)
    for j=1:degree(pgg)
      M[j,j] = pp
    end
    pt = t^(degree(pgg)-1)
    for j=degree(pgg)+1:n
      pt = (pt*t) % pgg
      M[j,j] = 1
      for k=0:degree(pt)
        M[j, k+1] = -lift(coeff(pt, k))
      end
    end
    #this is (or should be) the HNF basis for P^??
    M = lll(M)
    Mi, d = pseudo_inv(M)

    for j=1:length(RT)
      if RT[j] == 0
        continue
      end
      #to eval fp and the derivative, we pre-compute the powers of the
      #evaluation point - to save on large products...

      pow = [Qt(1), Qt(RT[j])]
      while length(pow) <= degree(f)+1
        push!(pow, pow[2]*pow[end] % pgg)
      end

      eval_f = sum(pow[k] * fp[k] for k=1:length(fp)) % pgg
      eval_fs = sum(pow[k-1] *(k-1)*fp[k] for k=2:length(fp)) % pgg

      #double lift:
      #IRT = invmod(fp'(rt), p^k)
      # using x -> x(2-xy) to compute the inverse of y
      IRT[j] = lift(ZX, Qt(IRT[j])*(Qt(2-IRT[j]*eval_fs) % pgg) %pgg)
      #RT = rt mod p^k normal Newton
      # using x -> x-fp(x)//fp'(x) = x-fp(x) * IRT
      RT[j] = lift(ZX, Qt(pow[2] - eval_f*Qt(IRT[j])) % pgg)

      #before the reconstruction, we need to scale by den
      cf = lift(ZX, Qt(RT[j]*den) % pgg)
      ve = matrix(FlintZZ, 1, n, [coeff(cf, k) for k=0:n-1])
      _ve = ve*Mi
      mu = matrix(FlintZZ, 1, n,  [ round(_ve[1, k]//d) for k=1:n])
      ve = ve - mu*M
      z = ZX()
      for k=1:n
        setcoeff!(z, k-1, ve[1, k])
      end
      zz = K(z)*iden
      if res[j] == zz || i == length(pr)
        if f(zz) == 0
          push!(rs, zz)
          if length(rs) >= max_roots
            return rs
          end
          RT[j] = ZX(0)
        end
      else
        res[j] = zz
      end
    end
  end
  return rs
end

function ispower(a::Nemo.fq_nmod, m::Int)
  s = size(parent(a))
  if gcd(s-1, m) == 1
    return true, a^invmod(m, s-1)
  end
  St, t = PolynomialRing(parent(a), "t", cached=false)
  f = t^m-a
  rt = roots(f)
  if length(rt) > 0
    return true, rt[1]
  else
    return false, a
  end
end

function has_primitive_root_1(K::Nemo.FqNmodFiniteField, m::Int)
  @assert m>0
  if m == 1
    return K(1)
  end
  if size(K) % m != 1
    return false, K(1)
  end
  if m == 2
    return K(-1)
  end
  fm = factor(m)
  while true
    g = rand(K)
    if iszero(g)
      continue
    end
    if any(x -> isone(g^div(m, x)), keys(fm))
      continue
    end
    return true, g^div(size(K)-1, m)
  end
end

#as above, but for f = x^m-a, so m-th root of a

function _hensel(a::nf_elem, m::Int, p::Int, k::Int; max_roots::Int = m)
  k = max(k, 2)
  #@assert denominator(a) == 1
  #well, actually, denominator(a, maximal_order)==1 would be sufficient, but
  #hard to test...
  ZX, X = FlintZZ["X"]
  Rp = ResidueRing(FlintZZ, p, cached = false)
  Rpt, t = PolynomialRing(Rp, "t", cached = false)
  K = parent(a)

  gp = Rpt(K.pol)
  #to avoid embarrasment...
  @assert !iszero(discriminant(gp))

  #find the prime ideal - as I don't want to use orders, this is
  #fun (computing a max order just for this is wasteful)
  #fun fact: if g = prod g_i mod p^k, then P_i^k = <p^k, g_i>
  #so instead of powering, and simplify and such, lets write it down
  lp = factor(gp).fac
  if length(lp) > 1
    g1 = lift(ZX, first(keys(lp)))
    gg = hensel_lift(ZX(K.pol), g1, fmpz(p), k)
  else
    gg = ZX(K.pol)
  end

  # now for all i<= k, <p^i, K(gg)+p^i> is a normal presentation for
  #                                     the prime ideal power.
  #later we'll get the HNF matrix for selected powers as well

  #set up the mod p data:
  #need FiniteField as I need to factor (roots)
  S = FqNmodFiniteField(first(keys(lp)), :z, false)
  ST, T = PolynomialRing(S, "T", cached=false)
  fp = T^m - S(Rpt(a))
  rt = roots(fp)
  #OK, we're doing s.th. different
#  b = a^(m-1) #this dominates everything!!!, maybe it should be done in the
#              #residue ring as well.
  rt = [inv(x^(m-1)) for x = rt]

  # this is in the finite field, but now I need it in the res. ring.
  # in ZX for ease of transport.
  RT = [lift(ZX, lift(Rpt, x)) for x = rt]

  #the den: ala Kronnecker:
  den = ZX(derivative(K.pol)(gen(K)))
  iden = inv(derivative(K.pol)(gen(K)))

  #for the result, to check for stabilising

  res = nf_elem[K(0) for x = RT]
  rs = nf_elem[]

  #we'll be working in (Z/p^k)[t]/gg

  #an "optimal" lifting chain:
  pr = [k]
  while k>1
    k = div(k+1, 2)
    push!(pr, k)
  end
  pr = reverse(pr)

  #lets start:

  for i=2:length(pr)
    pp = fmpz(p)^pr[i]
    Q = ResidueRing(FlintZZ, pp, cached=false)
    Qt, t = PolynomialRing(Q, "t", cached = false)

    #possibly this should be done with max precision and then adjusted down
    #the poly mod P^??

    pgg = Qt(gg) #we'll do the reductions by hand - possibly not optimal

    #the lattice for reco:
    n = degree(K)
    M = zero_matrix(FlintZZ, n, n)
    for j=1:degree(pgg)
      M[j,j] = pp
    end
    pt = t^(degree(pgg)-1)
    for j=degree(pgg)+1:n
      pt = (pt*t) % pgg
      M[j,j] = 1
      for k=0:degree(pt)
        M[j, k+1] = -lift(coeff(pt, k))
      end
    end
    #this is (or should be) the HNF basis for P^??
    M = lll(M)
    Mi, d = pseudo_inv(M)

    ap = Qt(ZX(a))
    bp = powmod(ap, m-1, pgg)
    minv = invmod(fmpz(m), pp)

    for j=1:length(RT)
      if RT[j] == 0
        continue
      end

      RTjp = Qt(RT[j])
      RT[j] = lift(ZX, (RTjp*(1+minv) - bp*minv* powmod(RTjp, m+1, pgg)) % pgg)

      #before the reconstruction, we need to scale by den and by a
      cf = lift(ZX, (Qt(RT[j]*den) % pgg)*ap % pgg)
      ve = matrix(FlintZZ, 1, n, [coeff(cf, k) for k=0:n-1])
      _ve = ve*Mi
      mu = matrix(FlintZZ, 1, n,  [ round(_ve[1, k]//d) for k=1:n])
      ve = ve - mu*M
      z = ZX()
      for k=1:n
        setcoeff!(z, k-1, ve[1, k])
      end
      zz = K(z)*iden
      if res[j] == zz || i == length(pr)
        if zz^m == a
          push!(rs, zz)
          if length(rs) >= max_roots
            return rs
          end
          RT[j] = ZX(0)
        end
      else
        res[j] = zz
      end
    end
  end
  return rs
end

## Hensel lifting of roots
##todo: redo for equation order using the kronnecker basis (co-different)
##      to avoid denominators
##    : use double iteration to avoid division
#     : use exponent chain to not overshoot (too much) in lifting
#     : common case is f == K.pol. In this case we known a sharp T2-bound
#     : for torsion polynomials, embed torsion units faithully and
#     : lift only one root of maximal order
#mostly done for nf_elem, see next function
#missing: different (better) handling for x^n-a
# possibly based on https://scicomp.stackexchange.com/questions/18846/newton-iteration-for-cube-root-without-division
#
# g = x^-n -a^(n-1), and r = root(g), then r = a^(-(n-1)/n), hence
#                                         ar = a^(-(n-1)/n + 1) = a^(1/n)
# Here, the Newton-iteration is easy: (b = a^(n-1))
# g = x^-n - b, => g' = -nx^(-n-1)
#Newton: r -> r-g(r)/g'(r) = r - (r^(-n) -b)/(-n r^(-n-1))
#                          = r - r/(-n) - b/n r^(n+1)
#                          = r(1+1/n) - b/n r^(n+1)
# no division! and no double iteration
# however, for reconstruction: we don't want r (which is non-integral)
# but ar which is...

function _roots_hensel(f::Generic.Poly{NfOrdElem}, max_roots::Int = degree(f))
  # f must be squarefree
  # I should check that
  O = base_ring(f)
  n = degree(O)
  deg = degree(f)

  # First we find a prime ideal such that f is squarefree modulo P
  # (The discriminant of f has only finitely many divisors).

  p = degree(f)+1

  fder = derivative(f)

  found_prime = false

  # Dummy variable
  Q = NfOrdIdl(O)
  pi_F = NfOrdToFqNmodMor()
  rt = Array{fq_nmod, 1}()
  di = discriminant(O)

  while !found_prime
    p = next_prime(p)

    if di % p == 0
      continue
    end

    lP = prime_decomposition(O, p)

    for P in lP
      F, pi_F = ResidueField(O, P[1])

      fmodP = pi_F(f)

      if !issquarefree(fmodP)
        continue
      end

      rt = roots(fmodP)
      Q = P[1]
      found_prime = true
      break
    end
  end

  #println("prime $p $(Q.gen_two)")

  fmodQ = pi_F(f)

  # compute the lifting exponent a la Friedrich-Fieker

  R = ArbField(64, false)
  (r1, r2) = signature(O)

  bound_root = [ R(0) for i in 1:(r1 + r2) ]

  CC = AcbField(64, false)
  CCt, t = PolynomialRing(CC, "t", cached=false)
  conjugates_of_coeffs = [ conjugates_arb(coeff(f, i), 32) for i in 0:degree(f) ]

  for i in 1:r1
    g = CCt([ conjugates_of_coeffs[j + 1][i] for j in 0:degree(f) ])
    bound_root[i] = roots_upper_bound(g)
  end

  for i in 1:r2
    g = CCt([ conjugates_of_coeffs[j + 1][r1 + i] for j in 0:degree(f) ])
    bound_root[r1 + i] = roots_upper_bound(g)
  end

  ss = _lifting_expo(p, Q.splitting_type[2], order(Q), bound_root)
  s = Int(ss)

  #println(length(lin_factor))
  #println("lifting bound: ", s)

  rts = NfOrdElem[]
  #println("aiming for $max_roots roots")
  for j in 1:length(rt)
  #  println("have $(length(roots)) found, want $max_roots")
    if length(rts) >= max_roots
      break
    end

    zero_mod_Q = rt[j]

    #println(zero_mod_Q)

    #println(0, " ", pi_F\(zero_mod_Q))

    # The following should be a uniformizing element
    Q_pi = Q.gen_two

    @hassert :NfOrdQuoRing 1 fmodQ(zero_mod_Q) == 0

    # This is the first step

    I = Q^2
    R, pi_R = quo(O, I)

    t1 = divexact(pi_R(subst(f, pi_F\zero_mod_Q)), pi_R(Q_pi))
    t2 = -inv(pi_R(subst(fder,pi_F\zero_mod_Q)))
    new_a = pi_R\(pi_R(pi_F\zero_mod_Q) + t1*t2*pi_R(Q_pi))
    #return pi_R(f)

    #println(1, " ", new_a)

    old_a = new_a

    RR, pi_RR = R, pi_R

    # reconstruction
    B = basis_mat(I)
    L = lll(B)

    rhs = MatrixSpace(FlintZZ, degree(O), 1, false)(elem_in_basis(new_a)'')
    lhs = transpose(L)

    X, d = solve_rational(lhs, rhs)

    zz = [ fmpq(BigInt(X[l, 1])//BigInt(d) - round(BigInt(X[l, 1])//BigInt(d))) for l in 1:degree(O)]

    cden = denominator(zz[1])

    for l in 2:degree(O)
      cden = lcm(cden, denominator(zz[l]))
    end

    zz_num = [ numerator(cden*zz[l]) for l in 1:degree(O) ]

    v = matrix(FlintZZ, 1, degree(O), zz_num)

    w = v*L

    # There is no slower function

    reconstructed_new = O(fmpz[ divexact(w[1, l], cden) for l in 1:degree(O) ])
    # end reconstruction

    reconstructed_old = reconstructed_new

    stabilized = 0

    i = 1
    while 2^i < s
      if reconstructed_new == reconstructed_old
        stabilized = stabilized + 1
      else
        stabilized = 0
      end

      if stabilized >= 2
        if subst(f, reconstructed_new) == 0
          #push!(rts, reconstructed_new)
          break
        else
          stabilized = 0
        end
      end

      if i == s
        if iszero(subst(f, reconstructed_new))
          push!(rts, reconstructed_new)
        end
        break
      end

      reconstructed_old = reconstructed_new
      old_a = new_a
      R, pi_R = RR, pi_RR
      I = simplify(I^2)

      # From Q^i -> Q^(i+1)
      # From Q^(2^i) -> Q^(2^i+1)

      RR, pi_RR = quo(O, I)
      #t1 = divexact(pi_RR(subst(f, old_a)), pi_RR(Q_pi)^(2^i))
      #t2 = pi_RR(pi_F\(-inv(pi_F(subst(fder, old_a)))))
      #new_a = pi_RR\(pi_RR(old_a) + t1*t2*pi_RR((Q_pi))^(2^i))
      new_a = old_a - subst(f, old_a) * (pi_R\(divexact(one(R), pi_R(subst(fder, old_a)))))
      new_a = pi_RR\(pi_RR(new_a))
      #println(i + 1, " ", new_a)

      # Try to reconstruct:

      B = basis_mat(I)
      L = lll(B)

      rhs = MatrixSpace(FlintZZ, degree(O), 1, false)(elem_in_basis(new_a)'')
      lhs = transpose(L)

      X, d = solve_rational(lhs, rhs)

      zz = [ fmpq(BigInt(X[l, 1])//BigInt(d) - round(BigInt(X[l, 1])//BigInt(d))) for l in 1:degree(O)]

      cden = denominator(zz[1])

      for l in 2:degree(O)
        cden = lcm(cden, denominator(zz[l]))
      end

      zz_num = [ numerator(cden*zz[l]) for l in 1:degree(O) ]

      v = matrix(FlintZZ, 1, degree(O), zz_num)

      w = v*L

      # There is no slower function

      reconstructed_new = O(fmpz[ divexact(w[1, l], cden) for l in 1:degree(O) ])

      #println(i + 1, " ", reconstructed_new)

      i = i + 1
    end
    if f(reconstructed_new) == 0
      push!(rts, reconstructed_new)
    end
  end
  return rts
end

function _lifting_expo(p::Int, deg_p::Int, O::NfOrd, bnd::Array{arb, 1})
  # compute the lifting exponent a la Friedrich-Fieker
  #bnd has upper bounds on |x^{(i)}| 1<= i <= r1+r2 as arbs
  #we're using a prime ideal above p of intertia degree deg_p

  (c1, c2) = norm_change_const(O)
  r1, r2 = signature(O)
  R = parent(bnd[1])
  bd = R(0)
  n = degree(O)
  #so   |x|_mink  <= c_1 |x|_coeff
  #and  |x|_coeff <= c_2 |x|_mink

  for i in 1:r1
    bd += bnd[i]^2
  end

  for i=1:r2
    bd += 2*bnd[i+r1]^2
  end

  boundt2 = max(bd, R(1))

  #CF: there is a prob, in the paper wrt LLL bounds on |x| or |x|^2....
  boundk = R(n)*log(R(c1)*R(c2)*boundt2*exp((R(n*(n-1))//2 + 2)*log(R(2)))//n)//(2*deg_p*log(R(p)))

  ss = abs_upper_bound(boundk, fmpz)
  return ss
end

function _roots_hensel(f::Generic.Poly{nf_elem}, max_roots::Int = degree(f))
  # f must be squarefree
  # I should check that
  K = base_ring(f)
  if iszero(constant_coefficient(f))
    rs = [K(0)]
    f = div(f, gen(parent(f)))
  else
    rs = nf_elem[]
  end
  if length(rs) >= max_roots
    return rs
  end
  n = degree(K)
  deg = degree(f)

  # First we find a prime ideal such that f is squarefree modulo P
  # (The discriminant of f has only finitely many divisors).

  p = degree(f)+1
  deg_p = 1

  while true
    p = next_prime(p)

    Rp = ResidueRing(FlintZZ, p, cached=false)
    Rpt, t = PolynomialRing(Rp, "t", cached=false)
    gp = Rpt(K.pol)
    if iszero(discriminant(gp))
      continue
    end

    lp = factor(gp).fac
    #set up the mod p data:
    #need FiniteField as I need to factor (roots)
    deg_p = degree(first(keys(lp)))
    S = FqNmodFiniteField(first(keys(lp)), :z, false)
    ST, T = PolynomialRing(S, "T", cached=false)
    fp = ST([S(Rpt(coeff(f, i))) for i=0:degree(f)])
    if !issquarefree(fp)
      continue
    end
    rt = roots(fp)
    if length(rt) == 0
      return nf_elem[]
    end
    break
  end

  #println("prime $p ")

  # compute the lifting exponent a la Friedrich-Fieker

  R = ArbField(64, false)

  (r1, r2) = signature(K)

  # for Kronnecker:
  gsa = derivative(K.pol)(gen(K))
  gsa_con = conjugates_arb(gsa, 32)

  bound_root = [ R(0) for i in 1:(r1 + r2) ]

  CC = AcbField(64, false)
  CCt, t = PolynomialRing(CC, "t", cached=false)
  conjugates_of_coeffs = [ conjugates_arb(coeff(f, i), 32) for i in 0:degree(f) ]

  for i in 1:r1
    g = CCt([ conjugates_of_coeffs[j + 1][i] for j in 0:degree(f) ])
    bound_root[i] = roots_upper_bound(g) * abs(gsa_con[i])
  end

  for i in 1:r2
    g = CCt([ conjugates_of_coeffs[j + 1][r1 + i] for j in 0:degree(f) ])
    bound_root[r1 + i] = roots_upper_bound(g) * abs(gsa_con[r1+i])
  end

  ss = _lifting_expo(p, deg_p, EquationOrder(K), bound_root)

  rts = _hensel(f, p, Int(ss), max_roots = max_roots - length(rs))
  return vcat(rs, rts)
end

function _roots_hensel(a::nf_elem, m::Int, max_roots::Int = m)
  # f must be squarefree
  # I should check that
  K = parent(a)
  n = degree(K)

  # First we find a prime ideal such that f is squarefree modulo P
  # (The discriminant of f has only finitely many divisors).

  p = m+1
  deg_p = 1

  while true
    p = next_prime(p)

    Rp = ResidueRing(FlintZZ, p, cached=false)
    Rpt, t = PolynomialRing(Rp, "t", cached = false)
    gp = Rpt(K.pol)
    if iszero(discriminant(gp))
      continue
    end

    lp = factor(gp).fac
    #set up the mod p data:
    #need FiniteField as I need to factor (roots)
    deg_p = degree(first(keys(lp)))
    S = FqNmodFiniteField(first(keys(lp)), :z, false)
    ST, T = PolynomialRing(S, "T", cached=false)
    fp = T^m - S(Rpt(a))
    if !issquarefree(fp)
      continue
    end
    rt = roots(fp)
    if length(rt) == 0
      return nf_elem[]
    end
    break
  end

  #println("prime $p ")

  # compute the lifting exponent a la Friedrich-Fieker

  R = ArbField(64, false)
  (r1, r2) = signature(K)

  # for Kronnecker:
  gsa = derivative(K.pol)(gen(K))
  gsa_con = conjugates_arb(gsa, 32)
  a_con = conjugates_arb(a, 32)

  bound_root = [ R(0) for i in 1:(r1 + r2) ]

  for i in 1:r1+r2
    bound_root[i] = root(abs(a_con[i]), m) * abs(gsa_con[i])
  end

  ss = _lifting_expo(p, deg_p, EquationOrder(K), bound_root)

  rts = _hensel(a, m, p, Int(ss), max_roots = max_roots)
  return rts
end
#identical to hasroot - which one do we want?
function ispower(a::NfOrdElem, n::Int)
  Ox, x = PolynomialRing(parent(a), "x", cached=false)
  f = x^n - a
  r = _roots_hensel(f, 1)

  if length(r) == 0
    return false, parent(a)(0)
  else
    return true, r[1]
  end
end

function probablity(O::NfOrdQuoRing)
  p = 1.0
  I = O.ideal
  f = factor(norm(I))
  for (P, e) in f
    if valuation(I, P) > 0
      p = p * (1 - 1/Float64(norm(P)))
    end
  end
  return p
end

################################################################################
#
#  Group Structure
#
################################################################################

doc"""
***
    group_structure(Q::NfOrdQuoRing) -> GrpAbFinGenSnf

> Returns an abelian group with the structure of (Q,+).
"""
function group_structure(Q::NfOrdQuoRing)
  i = ideal(Q)
  fac = factor(i)
  structure = Vector{fmpz}()
  for (p,vp) in fac
    pnum = minimum(p)
    e = valuation(pnum,p)
    f = factor(norm(p))[pnum]
    q, r = divrem(vp+e-1,e)
    structure_pvp = [repeat([pnum^q],inner=[Int((r+1)*f)]) ; repeat([pnum^(q-1)],inner=[Int((e-r-1)*f)])]
    append!(structure,structure_pvp)
  end
  G = DiagonalGroup(structure)
  S, Smap = snf(G)
  return S
end


import Base: get, length, getindex, push!, append!
""""
    IndexedDict{T}

A data structure for vocabulary in NLP Models, or any other categorical inputs
details: http://www.ekinakyurek.com/indexeddict/

```JULIA
julia> data = "this is an example data with example words"
"this is an example data with example words"

julia> vocab = IndexedDict{String}()
IndexedDict{String}(Dict{String,Int64}(), String[])

julia> append!(vocab,split(data))
IndexedDict{String}(Dict("this"=>1,"is"=>2,"example"=>4,"data"=>5,"words"=>7,"with"=>6,"an"=>3), ["this", "is", "an", "example", "data", "with", "words"])

julia> vocab["example"]
4

julia> vocab[4]
"example"

julia> vocab[[1,2,3,4]]
4-element Array{String,1}:
 "this"
 "is"
 "an"
 "example"

julia> vocab[["example","this"]]
2-element Array{Int64,1}:
 4
 1

julia> push!(vocab,"a-new-word")
IndexedDict{String}(Dict("a-new-word"=>8,"this"=>1,"is"=>2,"example"=>4,"data"=>5,"words"=>7,"with"=>6,"an"=>3), ["this", "is", "an", "example", "data", "with", "words", "a-new-word"])

julia> vocab["a-new-word"]
8

julia> length(vocab)
8
```
"""
struct IndexedDict{T}
    toIndex::Dict{T,Int};
    toElement::Vector{T};

    IndexedDict{T}(toIndex,toElement) where T = new(toIndex,toElement)
    IndexedDict{T}(toIndex,toElement) where T<:Integer = error("Cannot Create IndexedDict of Integers")
    IndexedDict{T}(toIndex,toElement) where T<:AbstractArray = error("Cannot Create IndexedDict of Arrays")
end

IndexedDict{T}() where T = IndexedDict{T}(Dict{T,Int}(),T[])

function IndexedDict(toElement::Vector{T}) where T
    toIndex=Dict{T,Int}(v=>k for (k,v) in enumerate(toElement))
    IndexedDict{T}(toIndex, toElement)
end

function IndexedDict(toIndex::Dict{T,Int}) where T
    toElement=Vector{T}(undef,length(toIndex))
    for (k,v) in toIndex; toElement[v]=k; end
    IndexedDict{T}(toIndex,toElement)
end

get(d::IndexedDict,v,default) = get(d.toIndex,v,default)
length(d::IndexedDict) = length(d.toElement)
getindex(d::IndexedDict,inds::Integer) = d.toElement[inds]
getindex(d::IndexedDict{T},inds::T) where T = d.toIndex[inds]
getindex(d::IndexedDict{T}, elements::Array{T,1}) where T = map(e->d[e], elements)
getindex(d::IndexedDict, inds::Array{<:Integer}) = d.toElement[inds]
append!(d1::IndexedDict{T}, d2::IndexedDict{T}) where T = append!(d1,d2.toElement)

function push!(d::IndexedDict, element)
    if !haskey(d.toIndex,element)
        d.toIndex[element]=length(d)+1;
        push!(d.toElement,element)
    end
    return d
end

function append!(d::IndexedDict, elements)
     for element in elements
          push!(d,element)
     end
     return d
end


@testset "qm7b" begin
    raw = rawdata(QM7b())
    @test typeof(raw[:names]) == Vector{String}
    @test size(raw[:names]) == (14,)
    @test typeof(raw[:X]) == Array{Float32,3}
    @test size(raw[:X]) == (7211, 23, 23)
    @test typeof(raw[:T]) == Matrix{Float32}
    @test size(raw[:T]) == (7211, 14)
end


###### type definition ##########

import Base: convert, length, show, getindex, start, next, done, isempty

abstract AbstractTimeSeries

immutable TimeArray{T,N} <: AbstractTimeSeries

    timestamp::Vector{Date}
    values::Array{T,N}
    colnames::Vector{UTF8String}

    function TimeArray(timestamp::Vector{Date},
                       values::Array{T,N},
                       colnames::Vector{UTF8String})
                           nrow, ncol = size(values, 1), size(values, 2)
                           nrow != size(timestamp, 1) ? error("values must match length of timestamp"):
                           ncol != size(colnames,1) ? error("column names must match width of array"):
                           timestamp != unique(timestamp) ? error("there are duplicate dates"):
                           ~(flipud(timestamp) == sort(timestamp) || timestamp == sort(timestamp)) ? error("dates are mangled"):
                           flipud(timestamp) == sort(timestamp) ?
                           new(flipud(timestamp), flipud(values), colnames):
                           new(timestamp, values, colnames)
    end
end

TimeArray{T,N,S<:String}(d::Vector{Date}, v::Array{T,N}, c::Vector{S}) = TimeArray{T,N}(d,v,map(utf8,c))
TimeArray{T,N,S<:String}(d::Date, v::Array{T,N}, c::Array{S,1}) = TimeArray([d], v, map(utf8,c))

###### conversion ###############

convert(::Type{TimeArray{Float64,1}}, x::TimeArray{Bool,1}) = (TimeArray(x.timestamp, float64(x.values), x.colnames))
convert(::Type{TimeArray{Float64,2}}, x::TimeArray{Bool,2}) = (TimeArray(x.timestamp, float64(x.values), x.colnames))

convert(x::TimeArray{Bool,1}) = convert(TimeArray{Float64,1}, x::TimeArray{Bool,1})
convert(x::TimeArray{Bool,2}) = convert(TimeArray{Float64,2}, x::TimeArray{Bool,2})

###### length ###################

function length(ata::AbstractTimeSeries)
    length(ata.timestamp)
end

###### iterator protocol #########

start(ta::TimeArray)   = 1
next(ta::TimeArray,i)  = ((ta.timestamp[i],ta.values[i,:]),i+1)
done(ta::TimeArray,i)  = (i > length(ta))
isempty(ta::TimeArray) = (length(ta) == 0)

###### show #####################

function show{T,N}(io::IO, ta::TimeArray{T,N})

  # variables
  nrow          = size(ta.values, 1)
  ncol          = size(ta.values, 2)
  intcatcher    = falses(ncol)
  for c in 1:ncol
      rowcheck =  trunc(ta.values[:,c]) - ta.values[:,c] .== 0
      if sum(rowcheck) == length(rowcheck)
          intcatcher[c] = true
      end
  end
  spacetime     = strwidth(string(ta.timestamp[1])) + 3
  firstcolwidth = strwidth(ta.colnames[1])
  colwidth      = Int[]
      for m in 1:ncol
          T == Bool ?
          push!(colwidth, max(strwidth(ta.colnames[m]), 5)) :
          push!(colwidth, max(strwidth(ta.colnames[m]), strwidth(@sprintf("%.2f", maximum(ta.values[:,m]))) + DECIMALS - 2))
      end

  # summary line
  print(io, @sprintf("%dx%d %s %s to %s", nrow, ncol, typeof(ta), string(ta.timestamp[1]), string(ta.timestamp[nrow])))
  println(io, "")
  println(io, "")

  # row label line

   print(io, ^(" ", spacetime), ta.colnames[1], ^(" ", colwidth[1] + 2 -firstcolwidth))

   for p in 2:length(colwidth)
     print(io, ta.colnames[p], ^(" ", colwidth[p] - strwidth(ta.colnames[p]) + 2))
   end
   println(io, "")

  # timestamp and values line
    if nrow > 7
        for i in 1:4
            print(io, ta.timestamp[i], " | ")
        for j in 1:ncol
            T == Bool ?
            print(io, rpad(ta.values[i,j], colwidth[j] + 2, " ")) :
            intcatcher[j] & SHOWINT ?
            print(io, rpad(iround(ta.values[i,j]), colwidth[j] + 2, " ")) :
            print(io, rpad(round(ta.values[i,j], DECIMALS), colwidth[j] + 2, " "))
        end
        println(io, "")
        end

        println(io, '\u22EE')

        for i in nrow-3:nrow
            print(io, ta.timestamp[i], " | ")
        for j in 1:ncol
            T == Bool ?
            print(io, rpad(ta.values[i,j], colwidth[j] + 2, " ")) :
            intcatcher[j] & SHOWINT ?
            print(io, rpad(iround(ta.values[i,j]), colwidth[j] + 2, " ")) :
            print(io, rpad(round(ta.values[i,j], DECIMALS), colwidth[j] + 2, " "))
        end
        println(io, "")
        end
    else
        for i in 1:nrow
            print(io, ta.timestamp[i], " | ")
        for j in 1:ncol
            T == Bool ?
            print(io, rpad(ta.values[i,j], colwidth[j] + 2, " ")) :
            intcatcher[j] & SHOWINT ?
            print(io, rpad(iround(ta.values[i,j]), colwidth[j] + 2, " ")) :
            print(io, rpad(round(ta.values[i,j], DECIMALS), colwidth[j] + 2, " "))
        end
        println(io, "")
        end
    end
end

###### getindex #################

# single row
function getindex{T,N}(ta::TimeArray{T,N}, n::Int)
    TimeArray(ta.timestamp[n], ta.values[n,:], ta.colnames)
end

# single row 1d
function getindex{T}(ta::TimeArray{T,1}, n::Int)
    TimeArray(ta.timestamp[n], ta.values[[n]], ta.colnames)
end

# range of rows
function getindex{T,N}(ta::TimeArray{T,N}, r::Range1{Int})
    TimeArray(ta.timestamp[r], ta.values[r,:], ta.colnames)
end

# range of 1d rows
function getindex{T}(ta::TimeArray{T,1}, r::Range1{Int})
    TimeArray(ta.timestamp[r], ta.values[r], ta.colnames)
end

# array of rows
function getindex{T,N}(ta::TimeArray{T,N}, a::Array{Int})
    TimeArray(ta.timestamp[a], ta.values[a,:], ta.colnames)
end

# array of 1d rows
function getindex{T}(ta::TimeArray{T,1}, a::Array{Int})
    TimeArray(ta.timestamp[a], ta.values[a], ta.colnames)
end

# single column by name
function getindex{T,N}(ta::TimeArray{T,N}, s::String)
    n = findfirst(ta.colnames, s)
    TimeArray(ta.timestamp, ta.values[:, n], UTF8String[s])
end

# array of columns by name
function getindex{T,N}(ta::TimeArray{T,N}, args::String...)
    ns = [findfirst(ta.colnames, a) for a in args]
    TimeArray(ta.timestamp, ta.values[:,ns], UTF8String[a for a in args])
end

# single date
function getindex{T,N}(ta::TimeArray{T,N}, d::Date)
   for i in 1:length(ta)
     if [d] == ta[i].timestamp
       return ta[i]
     else
       nothing
     end
   end
 end

# range of dates
function getindex{T,N}(ta::TimeArray{T,N}, dates::Array{Date,1})
  counter = Int[]
#  counter = int(zeros(length(dates)))
  for i in 1:length(dates)
    if findfirst(ta.timestamp, dates[i]) != 0
      #counter[i] = findfirst(ta.timestamp, dates[i])
      push!(counter, findfirst(ta.timestamp, dates[i]))
    end
  end
  ta[counter]
end

function getindex{T,N}(ta::TimeArray{T,N}, r::StepRange{Date})
    ta[[r]]
end

# day of week
# getindex{T,N}(ta::TimeArray{T,N}, d::DAYOFWEEK) = ta[dayofweek(ta.timestamp) .== d]


Cassette.@context MetaheuristicLogging

function Cassette.overdub(ctx::MetaheuristicLogging, f::typeof(Polya.Δ), args...)
    logger = ctx.metadata
    ret = @timed f(args...)
    log!(logger, args[2], ret.value[2], ret.time)
    return ret.value
end


module TenXData

using CodecZlib,TranscodingStreams
using DataFrames,CSV
using SparseArrays # needed for mmread
using SparseArrays # needed for mmread
using LinearAlgebra # needed for mmread
using MatrixMarket: _parseint,find_splits,skewsymmetric!,symmetric!,hermitian! # needed for mmread

import MatrixMarket.mmread

include("MM.jl")

function loadData(countFn::String,geneFn::String,barcodeFn::String)
    extn = split(countFn,".")[end]
    if extn == "gz" || extn == "gzip"
        #fh = GZip.open(countFn)
        fh = GzipDecompressorStream(open(countFn))
        #M = TenX.mmread(fh)
        M = TenX.mmread(fh)
        gh = GzipDecompressorStream(open(geneFn))
        G = map(r->r[1],CSV.File(gh,header=0))
        bh = GzipDecompressorStream(open(barcodeFn))
        B = map(r->r[1],CSV.File(bh,header=0))
    else
        M = mmread(countFn)
        G = map(r->r[1],CSV.File(geneFn,header=0))
        B = map(r->r[1],CSV.File(barcodeFn,header=0))
    end
    df = rename!(DataFrame(M),B)
    insertcols!(df,1,"gene"=>G)
    df
end

# process files with standard names
function loadFolder(dirPath::String)
    fm = Dict(:gn1=>"features.tsv.gz",
              :gn2=>"features.tsv",
              :gn3=>"genes.tsv.gz",
              :gn4=>"genes.tsv",
              :bc=>"barcodes.tsv.gz",
              :bc2=>"barcodes.tsv",
              :cnt=>"matrix.mtx.gz",
              :cnt2=>"matrix.mtx")

    if isfile(joinpath(dirPath,get(fm,:gn1,"")))
        df = TenX.loadData(joinpath(dirPath,get(fm,:cnt,"")),
                           joinpath(dirPath,get(fm,:bc,"")),
                           joinpath(dirPath,get(fm,:gn1,"")))
    elseif isfile(joinpath(dirPath,get(fm,:gn2,"")))
        df = TenX.loadData(joinpath(dirPath,get(fm,:cnt2,"")),
                           joinpath(dirPath,get(fm,:bc2,"")),
                           joinpath(dirPath,get(fm,:gn2,"")))
    elseif isfile(joinpath(dirPath,get(fm,:gn3,"")))
        df = TenX.loadData(joinpath(dirPath,get(fm,:cnt,"")),
                           joinpath(dirPath,get(fm,:bc,"")),
                           joinpath(dirPath,get(fm,:gn3,"")))
    elseif isfile(joinpath(dirPath,get(fm,:gn4,"")))
        df = TenX.loadData(joinpath(dirPath,get(fm,:cnt2,"")),
                           joinpath(dirPath,get(fm,:bc2,"")),
                           joinpath(dirPath,get(fm,:gn4,"")))
    end
    df
end
end


"""
    mc_opts{T}

Storage type used for parameter options in implicit bounding routine.
* `lambda::T`: Affine weighting parameter
* `kmax::Int64`: Number of iterations run
* `style::String`: Type of contractor used
* `z_rnd::Bool`: Flag for rounding z on interval bounds
* `z_rnd_eps::T`: Amount for rounding z on interval bounds
* `z_rnd_all::Bool`: Flag for rounding z on all bounds
* `z_rnd_all_eps::T`: Amount for rounding z on all bounds
* `aff_rnd::Bool`: Flag for rounding affine relaxation on interval bounds
* `aff_rnd_eps::T`: Amount for rounding affine relaxation on interval bounds
* `aff_rnd_all::Bool`: Flag for rounding affine relaxation on all bounds
* `aff_rnd_all_eps::T`: Amount for rounding affine relaxation on all bounds
* `hhj_rnd::Bool`: Flag for rounding Jacobian on interval bounds
* `hhj_rnd_eps::T`: Amount for rounding Jacobian on interval bounds
* `hhj_rnd_all::Bool`: Flag for rounding Jacobian on all bounds
* `hhj_rnd_all_eps::T`: Amount for rounding Jacobian on all bounds
* `aff_correct_eps::T`: Affine correction tolerance
"""
type mc_opts{T<:AbstractFloat}
  lambda::T
  kmax::Int64
  style::String
  z_rnd::Bool
  z_rnd_eps::T
  z_rnd_all::Bool
  z_rnd_all_eps::T
  aff_rnd::Bool
  aff_rnd_eps::T
  aff_rnd_all::Bool
  aff_rnd_all_eps::T
  hhj_rnd::Bool
  hhj_rnd_eps::T
  hhj_rnd_all::Bool
  hhj_rnd_all_eps::T
  aff_correct_eps::T
end

"""
    mc_opts(T)

Initialization function for currently sets the weight `.lambda = 0.5`, the
contractor style `.style = KrawczykCW`, the number of iterations to `.kmax = 2`,
and the affine correction tolerance as `.aff_correct_eps = 1E-12`. Other rounding
options disabled.
"""
mc_opts(T) = mc_opts{T}(0.5,2,"KrawczykCW",
                        false,0.0,false,0.0,
                        false,0.0,false,0.0,
                        false,0.0,false,0.0,1E-12)

mc_opts() = mc_opts(Float64)
"""
    set_default!(x::mc_opts)

Restores `x` to default values. Sets the weight `.lambda = 0.5`, the
contractor style `.style = KrawczykCW`, the number of iterations to `.kmax = 2`,
and the affine correction tolerance as `.aff_correct_eps = 1E-12`. Other rounding
options disabled.
"""
function set_default!(x::mc_opts{T}) where {T<:AbstractFloat}
  x.lambda = 0.5
  x.kmax = 2
  x.style = "KrawczykCW"
  x.z_rnd = false
  x.z_rnd_eps = 0.0
  x.z_rnd_all= false
  x.z_rnd_all_eps = 0.0
  x.aff_rnd= false
  x.aff_rnd_eps = 0.0
  x.aff_rnd_all= false
  x.aff_rnd_all_eps = 0.0
  x.hhj_rnd= false
  x.hhj_rnd_eps = 0.0
  x.hhj_rnd_all= false
  x.hhj_rnd_all_eps = 0.0
  x.aff_correct_eps = 1E-12
end


@testset "DataFrames" begin
    df = DataFrame(x1 = rand(5), x2 = rand(5), y = [:a,:a,:b,:a,:b])

    @test_throws ArgumentError targets(df)

    @test targets(:y, df) == [:a,:a,:b,:a,:b]
    @test targets(row->row[1,:y], df) == [:a,:a,:b,:a,:b]
    @test typeof(targets(:y, df)) == Vector{Symbol}
    @test eltype(targets(x->x, df)) <: SubDataFrame

    @test nobs(@inferred(undersample(:y, df))) === 4

    @test @inferred(getobs(df, 2)) == df[2,:]
    @test @inferred(getobs(datasubset(df, 2))) == df[2,:]

    @test typeof(datasubset(df, 2)) <: SubDataFrame
    @test @inferred(datasubset(df, [1,2,3,5])) == view(df, [1,2,3,5])
    @test @inferred(datasubset(df, 2)) == view(df, 2)
    @test @inferred(datasubset(datasubset(df, 2:3), 2)) == view(df, 3)
    @test @inferred(datasubset(df)) == view(df, 1:5)
end


module BinTests

using Test

using PicoQuant
include("test_utils.jl")

@testset "Test executable files in PicoQuant/bin" begin
    gen_qft_circuit_executed = false
    qasm2tn_executed = false
    tn2plan_executed = false
    contract_tn_executed = false
    contract_qasm_executed = false

    n = 4 # number of qubits for qft circuit
    test_dir = "tmptest"
    if isdir(test_dir) rm(test_dir, recursive=true) end
    mkdir("tmptest")

    # Try executing the scripts
    try
        test_ARGS = ["--number-qubits", "$n", "-o", joinpath(test_dir, "qft_$n.qasm")]
        include("../bin/gen_qft_circuit.jl")
        main(test_ARGS)
        gen_qft_circuit_executed = isfile(joinpath(test_dir, "qft_$n.qasm"))

        test_ARGS = ["--qasm", joinpath(test_dir, "qft_$n.qasm")]
        include("../bin/qasm2tn.jl")
        main(test_ARGS)
        qasm2tn_executed = isfile(joinpath(test_dir, "qft_$n.json"))  &&
                           isfile(joinpath(test_dir, "qft_$n.tl")) &&
                           isfile(joinpath(test_dir, "qft_$n.h5"))

        test_ARGS = ["--tng", joinpath(test_dir, "qft_$n.json"),
                     "--tng_data", joinpath(test_dir, "qft_$n.h5"),
                     "--dsl", joinpath(test_dir, "qft_$(n)_contract"),
                     "--output_tng", joinpath(test_dir, "qft_$(n)_contract.json")]
        include("../bin/tn2dsl.jl")
        main(test_ARGS)
        tn2plan_executed = isfile(joinpath(test_dir, "qft_$(n)_contract.tl")) &&
                           isfile(joinpath(test_dir, "qft_$(n)_contract.h5")) &&
                           isfile(joinpath(test_dir, "qft_$(n)_contract.json"))

        test_ARGS = ["--dsl", joinpath(test_dir, "qft_$(n)_contract.tl"),
                     "--dsl_data", joinpath(test_dir, "qft_$(n)_contract.h5"),
                     "--output", joinpath(test_dir, "qft_$(n)_contract_out.h5")]
        include("../bin/contract_tn.jl")
        main(test_ARGS)
        contract_tn_executed = isfile(joinpath(test_dir, "qft_$(n)_contract_out.h5"))

        test_ARGS = ["--qasm", joinpath(test_dir, "qft_$n.qasm"),
                     "--output", joinpath(test_dir, "output.json"),
                     "--output_data", joinpath(test_dir, "output.h5")]
        include("../bin/contract_qasm.jl")
        main(test_ARGS)
        contract_qasm_executed = isfile(joinpath(test_dir, "output.json")) &&
                                 isfile(joinpath(test_dir, "output.h5"))

    finally
        # clean up any created files
        for fn in ["qft_$n.qasm",
                   "qft_$n.json",
                   "qft_$n.tl",
                   "qft_$n.h5",
                   "qft_$(n)_contract.tl",
                   "qft_$(n)_contract.h5",
                   "qft_$(n)_contract.json",
                   "qft_$(n)_contract_out.h5",
                   "output.json",
                   "output.h5"]
            rm(joinpath(test_dir, fn), force=true)
        end
        rm(test_dir, recursive=true)
    end

    # Test if scripts executed
    @test gen_qft_circuit_executed == true
    @test qasm2tn_executed == true
    @test tn2plan_executed == true
    @test contract_tn_executed == true
    @test contract_qasm_executed == true
end
end


function param_values()::Vector{Float64}
    p::Vector{Float64} = zeros(C.NUM)

    p[C.S] = 2.0
    p[C.kE] = 0.4
    p[C.kM] = 1.0
    p[C.kCD] = 0.03
    p[C.kCDS] = 0.45
    p[C.kR] = 0.18
    p[C.kRE] = 180
    p[C.kb] = 0.003
    p[C.KS] = 0.5
    p[C.kCE] = 0.35
    p[C.dM] = 0.7
    p[C.dE] = 0.25
    p[C.dCD] = 1.5
    p[C.dCE] = 1.5
    p[C.dR] = 0.06
    p[C.dRP] = 0.06
    p[C.dRE] = 0.03
    p[C.kP1] = 18.0
    p[C.kP2] = 18.0
    p[C.kDP] = 3.6
    p[C.KM] = 0.15
    p[C.KE] = 0.15
    p[C.KCD] = 0.92
    p[C.KCE] = 0.92
    p[C.KRP] = 0.01

    return p
end


# This file is part of Kpax3. License is MIT.

RecipesBase.@userplot Kpax3PlotP

RecipesBase.@recipe function plotp(g::Kpax3PlotP;
                                   clusterorder=:auto,
                                   clusterlabel=:auto)
  if (length(g.args) != 2) ||
     (typeof(g.args[1]) != Vector{Int}) ||
     (typeof(g.args[2]) != Matrix{Float64})
      throw(KInputError("Two arguments required. Got: $(typeof(g.args))"))
  end

  R, P = g.args

  n = length(R)
  k = maximum(R)

  if (clusterorder == :auto) || (length(clusterorder) != k)
    clusterorder = collect(1:k)
  end

  if (clusterlabel == :auto) || (length(clusterlabel) != k)
    clusterlabel = string.(clusterorder)
  end

  (ord, mid, sep) = reorderunits(R, P, clusterorder)

  mid .-= 0.5
  sep .-= 0.5

  N = n^2

  x = zeros(Int, N)
  y = zeros(Int, N)
  z = fill("", N)

  idx = 1
  pij = 0.0
  for j = 1:n, i = 1:n
    x[idx] = j
    y[idx] = i

    pij = P[ord[i], ord[j]]
    z[idx] = if pij <= 0.2
      "#FFFFFF"
    elseif pij <= 0.4
      "#F0F0F0"
    elseif pij <= 0.6
      "#BDBDBD"
    elseif pij <= 0.8
      "#636363"
    else
      "#000000"
    end

    idx += 1
  end

  # try to minimize the number of rectangles. We will do this greedly by
  # dividing the image into maximal adjacent rectangles
  # Hopefully the SVG file won't be too big
  #
  # TODO: improve the algorithm
  processed = falses(n, n)

  len = zero(Int)
  xmin = zeros(Int, len)
  xmax = zeros(Int, len)
  ymin = zeros(Int, len)
  ymax = zeros(Int, len)
  col = fill("", len)

  # elements on the diagonal have always p = 1.0
  val = "#000000"

  j = 1
  while j <= n
    jmax = expandsquarediag(j, n, val, z)

    push!(xmin, j - 1)
    push!(xmax, jmax)
    push!(ymin, j - 1)
    push!(ymax, jmax)
    push!(col, val)

    len += 1

    processed[j:jmax, j:jmax] .= true
    j = jmax + 1
  end

  # scan the lower triangular matrix
  j = 1
  while j < n
    # start a new column from the first non processed element
    i = j + 1
    while i <= n
      if !processed[i, j]
        val = z[LinearIndices((n, n))[i, j]]

        (imax, jmax) = expandrect(i, j, n, n, val, z, processed)

        # lower triangular
        push!(xmin, j - 1)
        push!(xmax, jmax)
        push!(ymin, i - 1)
        push!(ymax, imax)
        push!(col, val)

        len += 1

        # upper triangular (transpose)
        push!(xmin, i - 1)
        push!(xmax, imax)
        push!(ymin, j - 1)
        push!(ymax, jmax)
        push!(col, val)

        len += 1

        processed[i:imax, j:jmax] .= true
        i = imax + 1
      else
        while i <= n && processed[i, j]
          i += 1
        end
      end
    end

    j += 1
  end

  xborder = zeros(Float64, 5, k)
  yborder = zeros(Float64, 5, k)
  for i = 1:k
    xborder[:, i] = [sep[i]; sep[i]; sep[i + 1]; sep[i + 1]; sep[i]]
    yborder[:, i] = [sep[i]; sep[i + 1]; sep[i + 1]; sep[i]; sep[i]]
  end

  mcol = ["#FFFFFF" "#F0F0F0" "#BDBDBD" "#636363" "#000000"]
  mlab = ["[0.0, 0.2]" "(0.2, 0.4]" "(0.4, 0.6]" "(0.6, 0.8]" "(0.8, 1.0]"]

  rect_x = zeros(Float64, 5 * len)
  rect_y = zeros(Float64, 5 * len)
  for i = 1:len
    idx = LinearIndices((5, len))[:, i]
    rect_x[idx] .= [xmin[i]; xmax[i]; xmax[i]; xmin[i]; NaN]
    rect_y[idx] .= [ymin[i]; ymin[i]; ymax[i]; ymax[i]; NaN]
  end

  # start by plotting cluster borders
  RecipesBase.@series begin
    seriestype := :path
    linecolor := :black
    label := ""
    legend := :none

    xborder, yborder
  end

  # add the rectangles and fill them with color
  RecipesBase.@series begin
    seriestype := :shape
    fillcolor := col
    linealpha := 0.0
    linewidth := 0.0
    linestyle := :dot
    label := ""

    rect_x, rect_y
  end

  # complete the plot by adding the legend
  RecipesBase.@series begin
    seriestype := :scatter
    background_color := :white
    grid := false
    markershape := :rect
    markercolor := mcol
    label := mlab
    xlims := (0, n)
    xticks := (mid, clusterlabel)
    ylims := (0, n)
    yticks := (mid, clusterlabel)
    yflip := true

    html_output_format --> :svg
    size --> (800, 600)
    window_title --> ""
    title --> ""
    legend --> :right
    xlabel --> "Samples by cluster"
    ylabel --> "Samples by cluster"

    fill(-2.0, 1, length(mcol)), fill(-2.0, 1, length(mcol))
  end
end

RecipesBase.@userplot Kpax3PlotC

RecipesBase.@recipe function plotc(g::Kpax3PlotC)
  if (length(g.args) != 3) ||
     (typeof(g.args[1]) != Vector{Int}) ||
     (typeof(g.args[2]) != Vector{Float64}) ||
     (typeof(g.args[3]) != Matrix{Float64})
      throw(KInputError("Three arguments required. Got: $(typeof(g.args))"))
  end

  site, freq, C = g.args

  m = site[end]
  M = length(site)

  noise = zeros(Float64, m)
  weak = zeros(Float64, m)

  key = 1
  for b in 1:M
    if site[b] != key
      weak[key] += noise[key]
      key += 1
    end

    noise[key] += freq[b] * C[1, b]
    weak[key] += freq[b] * C[2, b]
  end

  weak[key] += noise[key]

  mcol = ["#FFFFFF" "#808080" "#000000"]
  mlab = ["Noise" "Weak signal" "Strong signal"]

  # start with a complete black plot (strong signal)
  RecipesBase.@series begin
    seriestype := :path
    linecolor := :black
    fill := (0, :black)
    label := ""
    legend := :none

    1:m, ones(Float64, m)
  end

  # add the weak signal on top
  RecipesBase.@series begin
    seriestype := :path
    linecolor := "#808080"
    fill := (0, "#808080")
    label :=""
    legend := :none

    1:m, weak
  end

  # finally add the white noise
  RecipesBase.@series begin
    seriestype := :path
    linecolor := :white
    fill := (0, :white)
    label :=""
    legend := :none

    1:m, noise
  end

  # add the legend
  RecipesBase.@series begin
    seriestype := :scatter
    background_color := :white
    grid := false
    markershape := :rect
    markercolor := mcol
    label := mlab
    legend := :right
    xlims := (1, m)
    ylims := (0, 1)

    html_output_format --> :svg
    size --> (800, 600)
    window_title --> ""
    title --> ""
    legend --> :right
    xlabel --> "Site"
    ylabel --> "Posterior probability"

    fill(-2, 1, length(mcol)), fill(-2, 1, length(mcol))
  end
end

RecipesBase.@userplot Kpax3PlotD

RecipesBase.@recipe function plotd(g::Kpax3PlotD;
                                   clusterorder=:auto,
                                   clusterlabel=:auto)
  if length(g.args) != 2
      throw(KInputError("Two arguments required. Got: $(typeof(g.args))"))
  end

  x, state = g.args

  (m, n) = size(x.data)
  M = length(x.ref)

  if (clusterorder == :auto) || (length(clusterorder) != state.k)
    clusterorder = collect(1:state.k)
  end

  if (clusterlabel == :auto) || length(clusterlabel) != state.k
    clusterlabel = string.(clusterorder)
  end

  v = zeros(Float64, state.k)
  for i in 1:n
    v[state.R[i]] += 1.0
  end
  v /= n

  v = [0.0; v[clusterorder]]

  cumsum!(v, v)

  xax = collect(1:M)

  yax = zeros(Float64, state.k)
  for g in 1:state.k
    yax[g] = (v[g] + v[g + 1]) / 2
  end
  round.(yax, digits=3)

  # colors
  colset = [("A", "Ala", "#FF3232");
            ("R", "Arg", "#FF7F00");
            ("N", "Asn", "#CAB2D6");
            ("D", "Asp", "#FDBF6F");
            ("C", "Cys", "#33A02C");
            ("E", "Glu", "#1F4E78");
            ("Q", "Gln", "#FF3300");
            ("G", "Gly", "#660066");
            ("H", "His", "#B2DF8A");
            ("I", "Ile", "#CC6600");
            ("L", "Leu", "#375623");
            ("K", "Lys", "#A6CEE3");
            ("M", "Met", "#CC3399");
            ("F", "Phe", "#FB9A99");
            ("P", "Pro", "#C81414");
            ("S", "Ser", "#1F78B4");
            ("T", "Thr", "#6A3D9A");
            ("W", "Trp", "#FFFF99");
            ("Y", "Tyr", "#FF66CC");
            ("V", "Val", "#993300");
            ("U", "Sec", "#00FFFF");
            ("O", "Pyl", "#FFFF00");
            ("B", "Asx", "#E4B9A3");
            ("Z", "Glx", "#8F413C");
            ("J", "Xle", "#825E12");
            ("X", "Xaa", "#000000");
            ("-",   "-", "#E0E0E0");
            ("*",   "*", "#000000");
            ( "",    "", "#FFFFFF")]

  coltable = Dict(map(a -> (a[1], a[3]), colset))

  # indices
  idx = 1
  s = 0
  t = 1
  b = 1
  g = 1
  h = 1

  # temporary values
  c = 0x01
  w = 0
  flag = false

  z = fill("#FFFFFF", state.k * M)

  for j in 1:M
    if x.ref[j] == UInt8('.')
      s += 1
      c = 0x01

      while (b <= m) && (x.key[b] == s)
        if state.C[state.cl[1], b] > c
          c = state.C[state.cl[1], b]
        end
        b += 1
      end

      if c > 0x02
        for g in 1:state.k
          h = clusterorder[g]
          w = t
          flag = false

          while !flag && w < b
            flag = state.C[state.cl[h], w] == 0x04
            w += 1
          end

          if flag
            z[idx] = coltable[uppercase(string(Char(x.val[w - 1])))]
          end

          idx += 1
        end
      else
        idx += state.k
      end

      t = b
    else
      idx += state.k
    end
  end

  # try to minimize the number of rectangles. We will do this greedly by
  # dividing the image into maximal adjacent rectangles
  # Hopefully the SVG file won't be too big
  #
  # TODO: improve the algorithm
  processed = falses(state.k, M)

  len = zero(Int)
  xmin = zeros(Int, len)
  xmax = zeros(Int, len)
  ymin = zeros(Float64, len)
  ymax = zeros(Float64, len)
  col = fill("", len)

  j = 1
  while j <= M
    # start a new column from the first non processed element
    i = 1
    while i <= state.k
      if !processed[i, j]
        val = z[LinearIndices((state.k, M))[i, j]]

        (imax, jmax) = expandrect(i, j, state.k, M, val, z, processed)

        push!(xmin, j - 1)
        push!(xmax, jmax)
        push!(ymin, v[i])
        push!(ymax, v[imax + 1])
        push!(col, val)

        len += 1

        processed[i:imax, j:jmax] .= true
        i = imax + 1
      else
        while i <= state.k && processed[i, j]
          i += 1
        end
      end
    end

    j += 1
  end

  mcol = reshape(map(a -> a[3], colset[1:26]), 1, 26)
  mlab = reshape(map(a -> a[2], colset[1:26]), 1, 26)

  rect_x = zeros(Float64, 5 * len)
  rect_y = zeros(Float64, 5 * len)
  for i = 1:len
    idx = LinearIndices((5, len))[:, i]
    rect_x[idx] .= [xmin[i]; xmax[i]; xmax[i]; xmin[i]; NaN]
    rect_y[idx] .= [ymin[i]; ymin[i]; ymax[i]; ymax[i]; NaN]
  end

  # draw the rectangles
  RecipesBase.@series begin
    seriestype := :shape
    fillcolor := col
    linecolor := col
    label := ""
    legend := :none

    rect_x, rect_y
  end

  # add cluster separators
  RecipesBase.@series begin
    seriestype := :hline
    linecolor := :black
    label := ""
    legend := :none

    v
  end

  # complete the plot by adding the legend
  RecipesBase.@series begin
    seriestype := :scatter
    background_color := :white
    grid := false
    markershape := :rect
    markercolor := mcol
    label := mlab
    xlims := (1, M)
    ylims := (0, 1)
    yticks := (yax, clusterlabel)
    yflip := true

    html_output_format --> :svg
    size --> (800, 600)
    window_title --> ""
    title --> ""
    legend --> :right
    xlabel --> "Site"
    ylabel --> "Cluster"

    fill(-2, 1, length(mcol)), fill(-2, 1, length(mcol))
  end
end


# This file is a part of ArraysOfArrays.jl, licensed under the MIT License (MIT).

using ArraysOfArrays
using Test

using StaticArrays

@testset "functions" begin
    function gen_nested()
        A11 = [1 2; 3 4]
        A21 = [4 5 6; 7 8 9]
        A12 = [10 11; 12 13; 14 15]
        A22 = [16 17; 18 19]
        hcat([A11, A21], [A12, A22])
    end

    @testset "flatview and nestedview" begin
        A = [(@SArray randn(3, 2, 4)) for i in 1:2, j in 1:2]
        @test @inferred(nestedview(flatview(A), Val(3))) == A

        B = rand(3, 2, 4)
        @test @inferred(nestedview(flatview(B), SVector{3})) == @inferred(nestedview(B, Val(1)))
    end

    @testset "innersize" begin
        @test @inferred(innersize([[1, 2, 3], [4, 5, 6]])) == (3,)
        @test @inferred(innersize([[]])) == (0,)
        @test_throws DimensionMismatch @inferred(innersize([[1, 2, 3], [4, 5]]))
    end

    @testset "deepgetindex" begin
        A = gen_nested()
        @test @inferred(deepgetindex(A, 1, 2)) === A[1, 2]
        @test @inferred(deepgetindex(A, 1, 2, 2, 1)) == A[1, 2][2, 1]
        @test @inferred(deepgetindex(A, 1, 2, 1:2, 1)) == A[1, 2][1:2, 1]
        @test @inferred(deepgetindex(A, 1, 1:2, 2, 1)) == [A[1, 1][2, 1], A[1, 2][2, 1]]
        @test @inferred(deepgetindex(A, 1, 1:2, 1:2, 2)) == [A[1, 1][1:2, 2], A[1, 2][1:2, 2]]
        @test_throws MethodError deepgetindex(A, 1, 2, 2, 1, 2)

        B = rand(3, 4, 5)
        @test @inferred(deepgetindex(B, 2, 3, 4)) === getindex(B, 2, 3, 4)
        @test_throws MethodError deepgetindex(B, 6)
        @test_throws MethodError deepgetindex(B, 2, 3)
    end

    @testset "deepsetindex!" begin
        B12 = [20 21 22; 23 24 25]
        X = [[41, 42], [43, 44]]

        A = gen_nested()
        @test @inferred(deepsetindex!(A, B12, 1, 2)) === A
        @test A[1, 2] == B12

        A = gen_nested()
        @test @inferred(deepsetindex!(A, 42, 1, 2, 2, 1)) === A
        @test A[1, 2][2, 1] == 42

        A = gen_nested()
        @test @inferred(deepsetindex!(A, X, 1, 1:2, 1:2, 2)) === A
        @test deepgetindex(A, 1, 1:2, 1:2, 2) == X
    end

    @testset "deepview" begin
        A = gen_nested()
        @test @inferred(deepview(A, 1, 2)) == view(A, 1, 2)
        @test @inferred(deepview(A, 1, 2, 2, 1)) == view(A[1, 2], 2, 1)
        @test @inferred(deepview(A, 1, 2, 1:2, 1)) == view(A[1, 2], 1:2, 1)
        @test_throws ArgumentError deepview(A, 1, 1:2, 2, 1)
        @test @inferred(deepview(A, 1, 1:2, 1:2, 2)) == [A[1, 1][1:2, 2], A[1, 2][1:2, 2]]
        @test_throws MethodError deepview(A, 1, 2, 2, 1, 2)

        B = rand(3, 4, 5)
        @test @inferred(deepview(B, 2, 3, 4)) === view(B, 2, 3, 4)
        @test_throws MethodError deepview(B, 6)
        @test_throws MethodError deepview(B, 2, 3)
    end

    @testset "abstract_nestedarray_type" begin
        @test @inferred(abstract_nestedarray_type(Int, Val(()))) == Int
        @test @inferred(abstract_nestedarray_type(Int, Val((2,)))) == AbstractArray{Int, 2}
        @test @inferred(abstract_nestedarray_type(Float32, Val((2,3,4)))) ==
            AbstractArray{<:AbstractArray{<:AbstractArray{Float32, 4}, 3}, 2}
    end
end


# Index constructors
NoIndex(t::Table) = NoIndex()

function UniqueIndex{names}(t::Table) where {names}
    # TODO check columns actually exist in table
    # TODO add uniqueness check? Maybe a keyword argument or `@checkbounds` or something
    return UniqueIndex{names}()
end

function SortIndex{names}(t::Table) where {names}
    order = sortperm(Project(names)(t))
    return SortIndex{names}(order)
end

function UniqueSortIndex{names}(t::Table) where {names}
    # TODO add uniqueness check? Maybe a keyword argument or `@checkbounds` or something
    order = sortperm(Project(names)(t))
    return UniqueSortIndex{names}(order)
end

function HashIndex{names}(t::Table) where {names}
    t_projected = Project(names)(t)
    d = Dict{_valuetype(eltype(t_projected)), Vector{Int}}()
    for i in keys(t_projected) # TODO make faster
        row = Tuple(t_projected[i])
        if haskey(d, row)
            push!(d[row], i)
        else
            d[row] = Int[i]
        end
    end
    return HashIndex{names}(d)
end

function UniqueHashIndex{names}(t::Table) where {names}
    t_projected = Project(names)(t)
    d = Dict{_valuetype(eltype(t_projected)), Int}()
    for i in keys(t_projected) # TODO make faster
        row = Tuple(t_projected[i])
        if haskey(d, row)
            error("Columns $names do not contain unique values")
        else
            d[row] = i
        end
    end
    return UniqueHashIndex{names}(d)
end

"""
    accelerate(table, AcceleratorType)

Returns a new table with an acceleration index of the specified type. The output table
may be faster at certain operations like `filter` with a relavent `Predicate`, however the
input table will remain unchanged.

See also `accelerate!`.
"""
function accelerate(t::Table{names}, ::Type{I}) where {I <: AbstractIndex, names}
    newindex = I(t)
    return Table{names}(getdata(t), (accelerators(t)..., newindex))
end

"""
    accelerate!(table, AcceleratorType)

Returns a new table with an acceleration index of the specified type. The output table may
be faster at certain operations like `filter` with a `Predicate`. The order of rows in the
input table may be changed in-place to optimize locality and further increase speed - for
example the rows may be ordered by one of the columns. However, the input table will not
have the knowledge of the acceleration index attached - take care to use the output table
after this operation.

See also `accelerate`.
"""
function accelerate!(t::Table{names}, ::Type{<:AbstractIndex})
    return accelerate(t, I)
end

function accelerate!(t::Table{<:Any,<:Any,Tuple{}}, ::Type{SortIndex{names}}) where {names}
    order = sortperm(Project(names)(t))
    foreach(col -> permute!(col, order), getdata(t))
    return Table{names}(getdata(t), (SortIndex{names}(order),))
end

function accelerate!(t::Table{<:Any,<:Any,Tuple{}}, ::Type{UniqueSortIndex{names}}) where {names}
    order = sortperm(Project(names)(t))
    foreach(col -> permute!(col, order), getdata(t))
    return Table{names}(getdata(t), (UniqueSortIndex{names}(order),))
end

# TODO tests for above