text: /*	$OpenBSD: noct.c,v 1.22 2011/01/12 17:01:26 deraadt Exp $	*/
  
  /*
   * Copyright (c) 2002 Jason L. Wright (jason@thought.net)
   * All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   * 1. Redistributions of source code must retain the above copyright
   *    notice, this list of conditions and the following disclaimer.
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice, this list of conditions and the following disclaimer in the
   *    documentation and/or other materials provided with the distribution.
   *
   * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
   * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
   * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
   * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
   * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
   * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   * POSSIBILITY OF SUCH DAMAGE.
   *
   * Effort sponsored in part by the Defense Advanced Research Projects
   * Agency (DARPA) and Air Force Research Laboratory, Air Force
   * Materiel Command, USAF, under agreement number F30602-01-2-0537.
   *
   */
  
  /*
   * Driver for the Netoctave NSP2000 security processor.
   */
  
  #include <sys/param.h>
  #include <sys/systm.h>
  #include <sys/proc.h>
  #include <sys/errno.h>
  #include <sys/malloc.h>
  #include <sys/kernel.h>
  #include <sys/mbuf.h>
  #include <sys/device.h>
  #include <sys/extent.h>
  #include <sys/kthread.h>
  
  #include <crypto/cryptodev.h>
  #include <dev/rndvar.h>
  
  #include <dev/pci/pcireg.h>
  #include <dev/pci/pcivar.h>
  #include <dev/pci/pcidevs.h>
  
  #include <dev/pci/noctreg.h>
  #include <dev/pci/noctvar.h>
  
  int noct_probe(struct device *, void *, void *);
  void noct_attach(struct device *, struct device *, void *);
  int noct_intr(void *);
  
  int noct_ram_size(struct noct_softc *);
  void noct_ram_write(struct noct_softc *, u_int32_t, u_int64_t);
  u_int64_t noct_ram_read(struct noct_softc *, u_int32_t);
  
  void noct_rng_enable(struct noct_softc *);
  void noct_rng_disable(struct noct_softc *);
  void noct_rng_init(struct noct_softc *);
  void noct_rng_intr(struct noct_softc *);
  void noct_rng_tick(void *);
  
  void noct_pkh_enable(struct noct_softc *);
  void noct_pkh_disable(struct noct_softc *);
  void noct_pkh_init(struct noct_softc *);
  void noct_pkh_intr(struct noct_softc *);
  void noct_pkh_freedesc(struct noct_softc *, int);
  u_int32_t noct_pkh_nfree(struct noct_softc *);
  int noct_kload(struct noct_softc *, struct crparam *, u_int32_t);
  void noct_kload_cb(struct noct_softc *, u_int32_t, int);
  void noct_modmul_cb(struct noct_softc *, u_int32_t, int);
  
  void noct_ea_enable(struct noct_softc *);
  void noct_ea_disable(struct noct_softc *);
  void noct_ea_init(struct noct_softc *);
  void noct_ea_intr(struct noct_softc *);
  void noct_ea_create_thread(void *);
  void noct_ea_thread(void *);
  u_int32_t noct_ea_nfree(struct noct_softc *);
  void noct_ea_start(struct noct_softc *, struct noct_workq *);
  void noct_ea_start_hash(struct noct_softc *, struct noct_workq *,
      struct cryptop *, struct cryptodesc *);
  void noct_ea_start_des(struct noct_softc *, struct noct_workq *,
      struct cryptop *, struct cryptodesc *);
  int noct_newsession(u_int32_t *, struct cryptoini *);
  int noct_freesession(u_int64_t);
  int noct_process(struct cryptop *);
  
  u_int32_t noct_read_4(struct noct_softc *, bus_size_t);
  void noct_write_4(struct noct_softc *, bus_size_t, u_int32_t);
  u_int64_t noct_read_8(struct noct_softc *, u_int32_t);
  void noct_write_8(struct noct_softc *, u_int32_t, u_int64_t);
  
  struct noct_softc *noct_kfind(struct cryptkop *);
  int noct_ksigbits(struct crparam *);
  int noct_kprocess(struct cryptkop *);
  int noct_kprocess_modexp(struct noct_softc *, struct cryptkop *);
  
  struct cfattach noct_ca = {
  	sizeof(struct noct_softc), noct_probe, noct_attach,
  };
  
  struct cfdriver noct_cd = {
  	0, "noct", DV_DULL
  };
  
  #define	SWAP32(x) (x) = htole32(ntohl((x)))
  
  int
  noct_probe(parent, match, aux)
  	struct device *parent;
  	void *match;
  	void *aux;
  {
  	struct pci_attach_args *pa = (struct pci_attach_args *) aux;
  
  	if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_NETOCTAVE &&
  	    PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_NETOCTAVE_NSP2K)
  		return (1);
  	return (0);
  }
  
  void 
  noct_attach(parent, self, aux)
  	struct device *parent, *self;
  	void *aux;
  {
  	struct noct_softc *sc = (struct noct_softc *)self;
  	struct pci_attach_args *pa = aux;
  	pci_chipset_tag_t pc = pa->pa_pc;
  	pci_intr_handle_t ih;
  	const char *intrstr = NULL;
  	bus_size_t iosize = 0;
  
  	if (pci_mapreg_map(pa, NOCT_BAR0, PCI_MAPREG_MEM_TYPE_64BIT, 0,
  	    &sc->sc_st, &sc->sc_sh, NULL, &iosize, 0)) {
  		printf(": can't map mem space\n");
  		goto fail;
  	}
  
  	/* Before we do anything else, put the chip in little endian mode */
  	NOCT_WRITE_4(sc, NOCT_BRDG_ENDIAN, 0);
  	sc->sc_rar_last = 0xffffffff;
  	sc->sc_waw_last = 0xffffffff;
  	sc->sc_dmat = pa->pa_dmat;
  
  	sc->sc_cid = crypto_get_driverid(0);
  	if (sc->sc_cid < 0) {
  		printf(": couldn't register cid\n");
  		goto fail;
  	}
  
  	if (pci_intr_map(pa, &ih)) {
  		printf(": couldn't map interrupt\n");
  		goto fail;
  	}
  	intrstr = pci_intr_string(pc, ih);
  	sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, noct_intr, sc,
  	    self->dv_xname);
  	if (sc->sc_ih == NULL) {
  		printf(": couldn't establish interrupt");
  		if (intrstr != NULL)
  			printf(" at %s", intrstr);
  		printf("\n");
  		goto fail;
  	}
  
  	if (noct_ram_size(sc))
  		goto fail;
  
  	printf(":");
  
  	noct_rng_init(sc);
  	noct_pkh_init(sc);
  	noct_ea_init(sc);
  
  	printf(", %uMB, %s\n", sc->sc_ramsize, intrstr);
  
  	return;
  
  fail:
  	if (iosize != 0)
  		bus_space_unmap(sc->sc_st, sc->sc_sh, iosize);
  }
  
  int 
  noct_intr(vsc)
  	void *vsc;
  {
  	struct noct_softc *sc = vsc;
  	u_int32_t reg;
  	int r = 0;
  
  	reg = NOCT_READ_4(sc, NOCT_BRDG_STAT);
  
  	if (reg & BRDGSTS_RNG_INT) {
  		r = 1;
  		noct_rng_intr(sc);
  	}
  
  	if (reg & BRDGSTS_PKP_INT) {
  		r = 1;
  		noct_pkh_intr(sc);
  	}
  
  	if (reg & BRDGSTS_CCH_INT) {
  		r = 1;
  		noct_ea_intr(sc);
  	}
  
  	return (r);
  }
  
  int
  noct_ram_size(sc)
  	struct noct_softc *sc;
  {
  	u_int64_t t;
  
  	noct_ram_write(sc, 0x000000, 64);
  	noct_ram_write(sc, 0x400000, 32);
  	t = noct_ram_read(sc, 0x000000);
  	noct_ram_write(sc, 0x000000, 128);
  	noct_ram_write(sc, 0x800000, t);
  	t = noct_ram_read(sc, 0x000000);
  
  	if (t != 32 && t != 64 && t != 128) {
  		printf(": invalid ram size %llx\n", (unsigned long long)t);
  		return (1);
  	}
  
  	sc->sc_ramsize = t;
  	return (0);
  }
  
  void
  noct_ram_write(sc, adr, dat)
  	struct noct_softc *sc;
  	u_int32_t adr;
  	u_int64_t dat;
  {
  	u_int32_t reg;
  
  	/* wait for pending writes to finish */
  	for (;;) {
  		reg = NOCT_READ_4(sc, NOCT_EA_CTX_ADDR);
  		if ((reg & EACTXADDR_WRITEPEND) == 0)
  			break;
  	}
  
  	NOCT_WRITE_4(sc, NOCT_EA_CTX_ADDR, adr);
  	NOCT_WRITE_4(sc, NOCT_EA_CTX_DAT_1, (dat >> 32) & 0xffffffff);
  	NOCT_WRITE_4(sc, NOCT_EA_CTX_DAT_0, (dat >>  0) & 0xffffffff);
  
  	for (;;) {
  		reg = NOCT_READ_4(sc, NOCT_EA_CTX_ADDR);
  		if ((reg & EACTXADDR_WRITEPEND) == 0)
  			break;
  	}
  }
  
  u_int64_t
  noct_ram_read(sc, adr)
  	struct noct_softc *sc;
  	u_int32_t adr;
  {
  	u_int64_t dat;
  	u_int32_t reg;
  
  	/* wait for pending reads to finish */
  	for (;;) {
  		reg = NOCT_READ_4(sc, NOCT_EA_CTX_ADDR);
  		if ((reg & EACTXADDR_READPEND) == 0)
  			break;
  	}
  
  	NOCT_WRITE_4(sc, NOCT_EA_CTX_ADDR, adr | EACTXADDR_READPEND);
  
  	for (;;) {
  		reg = NOCT_READ_4(sc, NOCT_EA_CTX_ADDR);
  		if ((reg & EACTXADDR_READPEND) == 0)
  			break;
  	}
  
  	dat = NOCT_READ_4(sc, NOCT_EA_CTX_DAT_1);
  	dat <<= 32;
  	dat |= NOCT_READ_4(sc, NOCT_EA_CTX_DAT_0);
  	return (dat);
  }
  
  void
  noct_pkh_disable(sc)
  	struct noct_softc *sc;
  {
  	u_int32_t r;
  
  	/* Turn off PK irq */
  	NOCT_WRITE_4(sc, NOCT_BRDG_CTL,
  	    NOCT_READ_4(sc, NOCT_BRDG_CTL) & ~(BRDGCTL_PKIRQ_ENA));
  
  	/* Turn off PK interrupts */
  	r = NOCT_READ_4(sc, NOCT_PKH_IER);
  	r &= ~(PKHIER_CMDSI | PKHIER_SKSWR | PKHIER_SKSOFF | PKHIER_PKHLEN |
  	    PKHIER_PKHOPCODE | PKHIER_BADQBASE | PKHIER_LOADERR |
  	    PKHIER_STOREERR | PKHIER_CMDERR | PKHIER_ILL | PKHIER_PKERESV |
  	    PKHIER_PKEWDT | PKHIER_PKENOTPRIME |
  	    PKHIER_PKE_B | PKHIER_PKE_A | PKHIER_PKE_M | PKHIER_PKE_R |
  	    PKHIER_PKEOPCODE);
  	NOCT_WRITE_4(sc, NOCT_PKH_IER, r);
  
  	/* Disable PK unit */
  	r = NOCT_READ_4(sc, NOCT_PKH_CSR);
  	r &= ~PKHCSR_PKH_ENA;
  	NOCT_WRITE_4(sc, NOCT_PKH_CSR, r);
  	for (;;) {
  		r = NOCT_READ_4(sc, NOCT_PKH_CSR);
  		if ((r & PKHCSR_PKH_BUSY) == 0)
  			break;
  	}
  
  	/* Clear status bits */
  	r |= PKHCSR_CMDSI | PKHCSR_SKSWR | PKHCSR_SKSOFF | PKHCSR_PKHLEN |
  	    PKHCSR_PKHOPCODE | PKHCSR_BADQBASE | PKHCSR_LOADERR |
  	    PKHCSR_STOREERR | PKHCSR_CMDERR | PKHCSR_ILL | PKHCSR_PKERESV |
  	    PKHCSR_PKEWDT | PKHCSR_PKENOTPRIME |
  	    PKHCSR_PKE_B | PKHCSR_PKE_A | PKHCSR_PKE_M | PKHCSR_PKE_R |
  	    PKHCSR_PKEOPCODE;
  	NOCT_WRITE_4(sc, NOCT_PKH_CSR, r);
  }
  
  void
  noct_pkh_enable(sc)
  	struct noct_softc *sc;
  {
  	u_int64_t adr;
  
  	sc->sc_pkhwp = 0;
  	sc->sc_pkhrp = 0;
  
  	adr = sc->sc_pkhmap->dm_segs[0].ds_addr;
  	NOCT_WRITE_4(sc, NOCT_PKH_Q_BASE_HI, (adr >> 32) & 0xffffffff);
  	NOCT_WRITE_4(sc, NOCT_PKH_Q_LEN, NOCT_PKH_QLEN);
  	NOCT_WRITE_4(sc, NOCT_PKH_Q_BASE_LO, (adr >> 0) & 0xffffffff);
  
  	NOCT_WRITE_4(sc, NOCT_PKH_IER,
  	    PKHIER_CMDSI | PKHIER_SKSWR | PKHIER_SKSOFF | PKHIER_PKHLEN |
  	    PKHIER_PKHOPCODE | PKHIER_BADQBASE | PKHIER_LOADERR |
  	    PKHIER_STOREERR | PKHIER_CMDERR | PKHIER_ILL | PKHIER_PKERESV |
  	    PKHIER_PKEWDT | PKHIER_PKENOTPRIME |
  	    PKHIER_PKE_B | PKHIER_PKE_A | PKHIER_PKE_M | PKHIER_PKE_R |
  	    PKHIER_PKEOPCODE);
  
  	NOCT_WRITE_4(sc, NOCT_PKH_CSR,
  	    NOCT_READ_4(sc, NOCT_PKH_CSR) | PKHCSR_PKH_ENA);
  
  	NOCT_WRITE_4(sc, NOCT_BRDG_CTL,
  	    NOCT_READ_4(sc, NOCT_BRDG_CTL) | BRDGCTL_PKIRQ_ENA);
  }
  
  void
  noct_pkh_init(sc)
  	struct noct_softc *sc;
  {
  	bus_dma_segment_t seg, bnseg;
  	int rseg, bnrseg;
  
  	sc->sc_pkh_bn = extent_create("noctbn", 0, 255, M_DEVBUF,
  	    NULL, NULL, EX_NOWAIT | EX_NOCOALESCE);
  	if (sc->sc_pkh_bn == NULL) {
  		printf("%s: failed pkh bn extent\n", sc->sc_dv.dv_xname);
  		goto fail;
  	}
  
  	if (bus_dmamem_alloc(sc->sc_dmat, NOCT_PKH_BUFSIZE,
  	    PAGE_SIZE, 0, &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
  		printf("%s: failed pkh buf alloc\n", sc->sc_dv.dv_xname);
  		goto fail;
  	}
  	if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, NOCT_PKH_BUFSIZE,
  	    (caddr_t *)&sc->sc_pkhcmd, BUS_DMA_NOWAIT)) {
  		printf("%s: failed pkh buf map\n", sc->sc_dv.dv_xname);
  		goto fail_1;
  	}
  	if (bus_dmamap_create(sc->sc_dmat, NOCT_PKH_BUFSIZE, rseg,
  	    NOCT_PKH_BUFSIZE, 0, BUS_DMA_NOWAIT, &sc->sc_pkhmap)) {
  		printf("%s: failed pkh map create\n", sc->sc_dv.dv_xname);
  		goto fail_2;
  	}
  	if (bus_dmamap_load_raw(sc->sc_dmat, sc->sc_pkhmap,
  	    &seg, rseg, NOCT_PKH_BUFSIZE, BUS_DMA_NOWAIT)) {
  		printf("%s: failed pkh buf load\n", sc->sc_dv.dv_xname);
  		goto fail_3;
  	}
  
  	/*
  	 * Allocate shadow big number cache.
  	 */
  	if (bus_dmamem_alloc(sc->sc_dmat, NOCT_BN_CACHE_SIZE, PAGE_SIZE, 0,
  	    &bnseg, 1, &bnrseg, BUS_DMA_NOWAIT)) {
  		printf("%s: failed bnc buf alloc\n", sc->sc_dv.dv_xname);
  		goto fail_4;
  	}
  	if (bus_dmamem_map(sc->sc_dmat, &bnseg, bnrseg, NOCT_BN_CACHE_SIZE,
  	    (caddr_t *)&sc->sc_bncache, BUS_DMA_NOWAIT)) {
  		printf("%s: failed bnc buf map\n", sc->sc_dv.dv_xname);
  		goto fail_5;
  	}
  	if (bus_dmamap_create(sc->sc_dmat, NOCT_BN_CACHE_SIZE, bnrseg,
  	    NOCT_BN_CACHE_SIZE, 0, BUS_DMA_NOWAIT, &sc->sc_bnmap)) {
  		printf("%s: failed bnc map create\n", sc->sc_dv.dv_xname);
  		goto fail_6;
  	}
  	if (bus_dmamap_load_raw(sc->sc_dmat, sc->sc_bnmap,
  	    &bnseg, bnrseg, NOCT_BN_CACHE_SIZE, BUS_DMA_NOWAIT)) {
  		printf("%s: failed bnc buf load\n", sc->sc_dv.dv_xname);
  		goto fail_7;
  	}
  
  	noct_pkh_disable(sc);
  	noct_pkh_enable(sc);
  
  #if 0
  	/*
  	 * XXX MODEXP is implemented as MODMUL for debugging, don't
  	 * XXX actually register.
  	 */
  	crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, noct_kprocess);
  	printf(" PK");
  #endif
  
  	return;
  
  fail_7:
  	bus_dmamap_destroy(sc->sc_dmat, sc->sc_bnmap);
  fail_6:
  	bus_dmamem_unmap(sc->sc_dmat,
  	    (caddr_t)sc->sc_pkhcmd, NOCT_PKH_BUFSIZE);
  fail_5:
  	bus_dmamem_free(sc->sc_dmat, &bnseg, bnrseg);
  fail_4:
  	bus_dmamap_unload(sc->sc_dmat, sc->sc_pkhmap);
  fail_3:
  	bus_dmamap_destroy(sc->sc_dmat, sc->sc_pkhmap);
  fail_2:
  	bus_dmamem_unmap(sc->sc_dmat,
  	    (caddr_t)sc->sc_pkhcmd, NOCT_PKH_BUFSIZE);
  fail_1:
  	bus_dmamem_free(sc->sc_dmat, &seg, rseg);
  fail:
  	if (sc->sc_pkh_bn != NULL) {
  		extent_destroy(sc->sc_pkh_bn);
  		sc->sc_pkh_bn = NULL;
  	}
  	sc->sc_pkhcmd = NULL;
  	sc->sc_pkhmap = NULL;
  }
  
  void
  noct_pkh_intr(sc)
  	struct noct_softc *sc;
  {
  	u_int32_t csr;
  	u_int32_t rp;
  
  	csr = NOCT_READ_4(sc, NOCT_PKH_CSR);
  	NOCT_WRITE_4(sc, NOCT_PKH_CSR, csr |
  	    PKHCSR_CMDSI | PKHCSR_SKSWR | PKHCSR_SKSOFF | PKHCSR_PKHLEN |
  	    PKHCSR_PKHOPCODE | PKHCSR_BADQBASE | PKHCSR_LOADERR |
  	    PKHCSR_STOREERR | PKHCSR_CMDERR | PKHCSR_ILL | PKHCSR_PKERESV |
  	    PKHCSR_PKEWDT | PKHCSR_PKENOTPRIME |
  	    PKHCSR_PKE_B | PKHCSR_PKE_A | PKHCSR_PKE_M | PKHCSR_PKE_R |
  	    PKHCSR_PKEOPCODE);
  
  	rp = (NOCT_READ_4(sc, NOCT_PKH_Q_PTR) & PKHQPTR_READ_M) >>
  	    PKHQPTR_READ_S;
  
  	while (sc->sc_pkhrp != rp) {
  		if (sc->sc_pkh_bnsw[sc->sc_pkhrp].bn_callback != NULL)
  			(*sc->sc_pkh_bnsw[sc->sc_pkhrp].bn_callback)(sc,
  			    sc->sc_pkhrp, 0);
  		if (++sc->sc_pkhrp == NOCT_PKH_ENTRIES)
  			sc->sc_pkhrp = 0;
  	}
  	sc->sc_pkhrp = rp;
  
  	if (csr & PKHCSR_CMDSI) {
  		/* command completed */
  	}
  
  	if (csr & PKHCSR_SKSWR)
  		printf("%s:%x: sks write error\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_SKSOFF)
  		printf("%s:%x: sks offset error\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKHLEN)
  		printf("%s:%x: pkh invalid length\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKHOPCODE)
  		printf("%s:%x: pkh bad opcode\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_BADQBASE)
  		printf("%s:%x: pkh base qbase\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_LOADERR)
  		printf("%s:%x: pkh load error\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_STOREERR)
  		printf("%s:%x: pkh store error\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_CMDERR)
  		printf("%s:%x: pkh command error\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_ILL)
  		printf("%s:%x: pkh illegal access\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKERESV)
  		printf("%s:%x: pke reserved error\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKEWDT)
  		printf("%s:%x: pke watchdog\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKENOTPRIME)
  		printf("%s:%x: pke not prime\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKE_B)
  		printf("%s:%x: pke bad 'b'\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKE_A)
  		printf("%s:%x: pke bad 'a'\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKE_M)
  		printf("%s:%x: pke bad 'm'\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKE_R)
  		printf("%s:%x: pke bad 'r'\n", sc->sc_dv.dv_xname, rp);
  	if (csr & PKHCSR_PKEOPCODE)
  		printf("%s:%x: pke bad opcode\n", sc->sc_dv.dv_xname, rp);
  }
  
  void
  noct_rng_disable(sc)
  	struct noct_softc *sc;
  {
  	u_int64_t csr;
  	u_int32_t r;
  
  	/* Turn off RN irq */
  	NOCT_WRITE_4(sc, NOCT_BRDG_CTL,
  	    NOCT_READ_4(sc, NOCT_BRDG_CTL) & ~(BRDGCTL_RNIRQ_ENA));
  
  	/* Turn off RNH interrupts */
  	r = NOCT_READ_4(sc, NOCT_RNG_CSR);
  	r &= ~(RNGCSR_INT_KEY | RNGCSR_INT_DUP |
  	    RNGCSR_INT_BUS | RNGCSR_INT_ACCESS);
  	NOCT_WRITE_4(sc, NOCT_RNG_CSR, r);
  
  	/* Turn off RN queue */
  	r = NOCT_READ_4(sc, NOCT_RNG_CSR);
  	r &= ~(RNGCSR_XFER_ENABLE | RNGCSR_INT_KEY | RNGCSR_INT_BUS |
  	    RNGCSR_INT_DUP | RNGCSR_INT_ACCESS);
  	NOCT_WRITE_4(sc, NOCT_RNG_CSR, r);
  
  	for (;;) {
  		r = NOCT_READ_4(sc, NOCT_RNG_CSR);
  		if ((r & RNGCSR_XFER_BUSY) == 0)
  			break;
  	}
  
  	/* Turn off RN generator */
  	csr = NOCT_READ_8(sc, NOCT_RNG_CTL);
  	csr &= ~RNGCTL_RNG_ENA;
  	NOCT_WRITE_8(sc, NOCT_RNG_CTL, csr);
  }
  
  void
  noct_rng_enable(sc)
  	struct noct_softc *sc;
  {
  	u_int64_t adr;
  	u_int32_t r;
  
  	adr = sc->sc_rngmap->dm_segs[0].ds_addr;
  	NOCT_WRITE_4(sc, NOCT_RNG_Q_BASE_HI, (adr >> 32) & 0xffffffff);
  	NOCT_WRITE_4(sc, NOCT_RNG_Q_LEN, NOCT_RNG_QLEN);
  	NOCT_WRITE_4(sc, NOCT_RNG_Q_BASE_LO, (adr >> 0 ) & 0xffffffff);
  
  	NOCT_WRITE_8(sc, NOCT_RNG_CTL,
  	    RNGCTL_RNG_ENA |
  	    RNGCTL_TOD_ENA |
  	    RNGCTL_BUFSRC_SEED |
  	    RNGCTL_SEEDSRC_INT |
  	    RNGCTL_EXTCLK_ENA |
  	    RNGCTL_DIAG |
  	    (100 & RNGCTL_ITERCNT));
  
  	/* Turn on interrupts and enable xfer */
  	r = RNGCSR_XFER_ENABLE | RNGCSR_INT_ACCESS |
  	    RNGCSR_INT_KEY | RNGCSR_INT_BUS | RNGCSR_INT_DUP;
  	NOCT_WRITE_4(sc, NOCT_RNG_CSR, r);
  
  	/* Turn on bridge/rng interrupts */
  	r = NOCT_READ_4(sc, NOCT_BRDG_CTL);
  	r |= BRDGCTL_RNIRQ_ENA;
  	NOCT_WRITE_4(sc, NOCT_BRDG_CTL, r);
  }
  
  void
  noct_rng_init(sc)
  	struct noct_softc *sc;
  {
  	bus_dma_segment_t seg;
  	int rseg;
  
  	if (bus_dmamem_alloc(sc->sc_dmat, NOCT_RNG_BUFSIZE,
  	    PAGE_SIZE, 0, &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
  		printf("%s: failed rng buf alloc\n", sc->sc_dv.dv_xname);
  		goto fail;
  	}
  	if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, NOCT_RNG_BUFSIZE,
  	    (caddr_t *)&sc->sc_rngbuf, BUS_DMA_NOWAIT)) {
  		printf("%s: failed rng buf map\n", sc->sc_dv.dv_xname);
  		goto fail_1;
  	}
  	if (bus_dmamap_create(sc->sc_dmat, NOCT_RNG_BUFSIZE, rseg,
  	    NOCT_RNG_BUFSIZE, 0, BUS_DMA_NOWAIT, &sc->sc_rngmap)) {
  		printf("%s: failed rng map create\n", sc->sc_dv.dv_xname);
  		goto fail_2;
  	}
  	if (bus_dmamap_load_raw(sc->sc_dmat, sc->sc_rngmap,
  	    &seg, rseg, NOCT_RNG_BUFSIZE, BUS_DMA_NOWAIT)) {
  		printf("%s: failed rng buf load\n", sc->sc_dv.dv_xname);
  		goto fail_3;
  	}
  
  	noct_rng_disable(sc);
  	noct_rng_enable(sc);
  
  	printf(" RNG");
  
  	if (hz > 100)
  		sc->sc_rngtick = hz/100;
  	else
  		sc->sc_rngtick = 1;
  	timeout_set(&sc->sc_rngto, noct_rng_tick, sc);
  	timeout_add(&sc->sc_rngto, sc->sc_rngtick);
  
  	return;
  
  fail_3:
  	bus_dmamap_destroy(sc->sc_dmat, sc->sc_rngmap);
  fail_2:
  	bus_dmamem_unmap(sc->sc_dmat,
  	    (caddr_t)sc->sc_rngbuf, NOCT_RNG_BUFSIZE);
  fail_1:
  	bus_dmamem_free(sc->sc_dmat, &seg, rseg);
  fail:
  	sc->sc_rngbuf = NULL;
  	sc->sc_rngmap = NULL;
  }
  
  void
  noct_rng_intr(sc)
  	struct noct_softc *sc;
  {
  	u_int32_t csr;
  	int enable = 1;
  
  	csr = NOCT_READ_4(sc, NOCT_RNG_CSR);
  	NOCT_WRITE_4(sc, NOCT_RNG_CSR, csr);
  
  	if (csr & RNGCSR_ERR_KEY) {
  		u_int32_t ctl;
  
  		enable = 0;
  		ctl = NOCT_READ_4(sc, NOCT_RNG_CTL);
  		printf("%s: rng bad key(s)", sc->sc_dv.dv_xname);
  		if (ctl & RNGCTL_KEY1PAR_ERR)
  			printf(", key1 parity");
  		if (ctl & RNGCTL_KEY2PAR_ERR)
  			printf(", key2 parity");
  		printf("\n");
  	}
  	if (csr & RNGCSR_ERR_BUS) {
  		enable = 0;
  		printf("%s: rng bus error\n", sc->sc_dv.dv_xname);
  	}
  	if (csr & RNGCSR_ERR_DUP) {
  		enable = 0;
  		printf("%s: rng duplicate block\n", sc->sc_dv.dv_xname);
  	}
  	if (csr & RNGCSR_ERR_ACCESS) {
  		enable = 0;
  		printf("%s: rng invalid access\n", sc->sc_dv.dv_xname);
  	}
  
  	if (!enable)
  		noct_rng_disable(sc);
  }
  
  void
  noct_rng_tick(vsc)
  	void *vsc;
  {
  	struct noct_softc *sc = vsc;
  	u_int64_t val;
  	u_int32_t reg, rd, wr;
  	int cons = 0;
  
  	reg = NOCT_READ_4(sc, NOCT_RNG_Q_PTR);
  	rd = (reg & RNGQPTR_READ_M) >> RNGQPTR_READ_S;
  	wr = (reg & RNGQPTR_WRITE_M) >> RNGQPTR_WRITE_S;
  
  	while (rd != wr && cons < 32) {
  		val = sc->sc_rngbuf[rd];
  		add_true_randomness((val >> 32) & 0xffffffff);
  		add_true_randomness((val >> 0) & 0xffffffff);
  		if (++rd == NOCT_RNG_ENTRIES)
  			rd = 0;
  		cons++;
  	}
  
  	if (cons != 0)
  		NOCT_WRITE_4(sc, NOCT_RNG_Q_PTR, rd);
  	timeout_add(&sc->sc_rngto, sc->sc_rngtick);
  }
  
  u_int32_t
  noct_ea_nfree(sc)
  	struct noct_softc *sc;
  {
  	if (sc->sc_eawp == sc->sc_earp)
  		return (NOCT_EA_ENTRIES);
  	if (sc->sc_eawp < sc->sc_earp)
  		return (sc->sc_earp - sc->sc_eawp - 1);
  	return (sc->sc_earp + NOCT_EA_ENTRIES - sc->sc_eawp - 1);
  }
  
  void
  noct_ea_disable(sc)
  	struct noct_softc *sc;
  {
  	u_int32_t r;
  
  	/* Turn off EA irq */
  	NOCT_WRITE_4(sc, NOCT_BRDG_CTL,
  	    NOCT_READ_4(sc, NOCT_BRDG_CTL) & ~(BRDGCTL_EAIRQ_ENA));
  
  	/* Turn off EA interrupts */
  	r = NOCT_READ_4(sc, NOCT_EA_IER);
  	r &= ~(EAIER_QALIGN | EAIER_CMDCMPL | EAIER_OPERR | EAIER_CMDREAD |
  	    EAIER_CMDWRITE | EAIER_DATAREAD | EAIER_DATAWRITE |
  	    EAIER_INTRNLLEN | EAIER_EXTRNLLEN | EAIER_DESBLOCK |
  	    EAIER_DESKEY | EAIER_ILL);
  	NOCT_WRITE_4(sc, NOCT_EA_IER, r);
  
  	/* Disable EA unit */
  	r = NOCT_READ_4(sc, NOCT_EA_CSR);
  	r &= ~EACSR_ENABLE;
  	NOCT_WRITE_4(sc, NOCT_EA_CSR, r);
  	for (;;) {
  		r = NOCT_READ_4(sc, NOCT_EA_CSR);
  		if ((r & EACSR_BUSY) == 0)
  			break;
  	}
  
  	/* Clear status bits */
  	r = NOCT_READ_4(sc, NOCT_EA_CSR);
  	r |= EACSR_QALIGN | EACSR_CMDCMPL | EACSR_OPERR | EACSR_CMDREAD |
  	    EACSR_CMDWRITE | EACSR_DATAREAD | EACSR_DATAWRITE |
  	    EACSR_INTRNLLEN | EACSR_EXTRNLLEN | EACSR_DESBLOCK |
  	    EACSR_DESKEY | EACSR_ILL;
  	NOCT_WRITE_4(sc, NOCT_EA_CSR, r);
  }
  
  void
  noct_ea_enable(sc)
  	struct noct_softc *sc;
  {
  	u_int64_t adr;
  
  	sc->sc_eawp = 0;
  	sc->sc_earp = 0;
  
  	adr = sc->sc_eamap->dm_segs[0].ds_addr;
  	NOCT_WRITE_4(sc, NOCT_EA_Q_BASE_HI, (adr >> 32) & 0xffffffff);
  	NOCT_WRITE_4(sc, NOCT_EA_Q_LEN, NOCT_EA_QLEN);
  	NOCT_WRITE_4(sc, NOCT_EA_Q_BASE_LO, (adr >> 0) & 0xffffffff);
  
  	NOCT_WRITE_4(sc, NOCT_EA_IER,
  	    EAIER_QALIGN | EAIER_CMDCMPL | EAIER_OPERR | EAIER_CMDREAD |
  	    EAIER_CMDWRITE | EAIER_DATAREAD | EAIER_DATAWRITE |
  	    EAIER_INTRNLLEN | EAIER_EXTRNLLEN | EAIER_DESBLOCK |
  	    EAIER_DESKEY | EAIER_ILL);
  
  	NOCT_WRITE_4(sc, NOCT_EA_CSR,
  	    NOCT_READ_4(sc, NOCT_EA_CSR) | EACSR_ENABLE);
  
  	NOCT_WRITE_4(sc, NOCT_BRDG_CTL,
  	    NOCT_READ_4(sc, NOCT_BRDG_CTL) | BRDGCTL_EAIRQ_ENA);
  }
  
  void
  noct_ea_init(sc)
  	struct noct_softc *sc;
  {
  	bus_dma_segment_t seg;
  	int rseg, algs[CRYPTO_ALGORITHM_MAX + 1];
  
  	if (bus_dmamem_alloc(sc->sc_dmat, NOCT_EA_BUFSIZE,
  	    PAGE_SIZE, 0, &seg, 1, &rseg, BUS_DMA_NOWAIT)) {
  		printf("%s: failed ea buf alloc\n", sc->sc_dv.dv_xname);
  		goto fail;
  	}
  	if (bus_dmamem_map(sc->sc_dmat, &seg, rseg, NOCT_EA_BUFSIZE,
  	    (caddr_t *)&sc->sc_eacmd, BUS_DMA_NOWAIT)) {
  		printf("%s: failed ea buf map\n", sc->sc_dv.dv_xname);
  		goto fail_1;
  	}
  	if (bus_dmamap_create(sc->sc_dmat, NOCT_EA_BUFSIZE, rseg,
  	    NOCT_EA_BUFSIZE, 0, BUS_DMA_NOWAIT, &sc->sc_eamap)) {
  		printf("%s: failed ea map create\n", sc->sc_dv.dv_xname);
  		goto fail_2;
  	}
  	if (bus_dmamap_load_raw(sc->sc_dmat, sc->sc_eamap,
  	    &seg, rseg, NOCT_EA_BUFSIZE, BUS_DMA_NOWAIT)) {
  		printf("%s: failed ea buf load\n", sc->sc_dv.dv_xname);
  		goto fail_3;
  	}
  
  	noct_ea_disable(sc);
  	noct_ea_enable(sc);
  
  	SIMPLEQ_INIT(&sc->sc_inq);
  	SIMPLEQ_INIT(&sc->sc_chipq);
  	SIMPLEQ_INIT(&sc->sc_outq);
  
  	bzero(algs, sizeof(algs));
  
  	algs[CRYPTO_MD5] = CRYPTO_ALG_FLAG_SUPPORTED;
  	algs[CRYPTO_SHA1] = CRYPTO_ALG_FLAG_SUPPORTED;
  	algs[CRYPTO_DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
  	algs[CRYPTO_3DES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
  
  	crypto_register(sc->sc_cid, algs, 
  	    noct_newsession, noct_freesession, noct_process);
  	printf(" MD5 SHA1 3DES");
  
  	kthread_create_deferred(noct_ea_create_thread, sc);
  
  	return;
  
  fail_3:
  	bus_dmamap_destroy(sc->sc_dmat, sc->sc_eamap);
  fail_2:
  	bus_dmamem_unmap(sc->sc_dmat,
  	    (caddr_t)sc->sc_eacmd, NOCT_EA_BUFSIZE);
  fail_1:
  	bus_dmamem_free(sc->sc_dmat, &seg, rseg);
  fail:
  	sc->sc_eacmd = NULL;
  	sc->sc_eamap = NULL;
  }
  
  void
  noct_ea_create_thread(vsc)
  	void *vsc;
  {
  	struct noct_softc *sc = vsc;
  
  	if (kthread_create(noct_ea_thread, sc, NULL,
  	    "%s", sc->sc_dv.dv_xname))
  		panic("%s: unable to create ea thread", sc->sc_dv.dv_xname);
  }
  
  void
  noct_ea_thread(vsc)
  	void *vsc;
  {
  	struct noct_softc *sc = vsc;
  	struct noct_workq *q;
  	struct cryptop *crp;
  	struct cryptodesc *crd;
  	int s, rseg;
  	u_int32_t len;
  
  	for (;;) {
  		tsleep(&sc->sc_eawp, PWAIT, "noctea", 0);
  
  		/* Handle output queue */
  		s = splnet();
  		while (!SIMPLEQ_EMPTY(&sc->sc_outq)) {
  			q = SIMPLEQ_FIRST(&sc->sc_outq);
  			SIMPLEQ_REMOVE_HEAD(&sc->sc_outq, q_next);
  			splx(s);
  
  			crp = q->q_crp;
  			crd = crp->crp_desc;
  			switch (crd->crd_alg) {
  			case CRYPTO_MD5:
  				len = 16;
  				break;
  			case CRYPTO_SHA1:
  				len = 20;
  				break;
  			default:
  				len = 0;
  				break;
  			}
  
  			bus_dmamap_sync(sc->sc_dmat, q->q_dmamap,
  			    0, q->q_dmamap->dm_mapsize,
  			    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
  
  			if (len != 0) {
  				if (crp->crp_flags & CRYPTO_F_IMBUF)
  					m_copyback((struct mbuf *)crp->crp_buf,
  					    crd->crd_inject, len,
  					    q->q_macbuf, M_NOWAIT);
  				else if (crp->crp_flags & CRYPTO_F_IOV)
  					bcopy(q->q_macbuf, crp->crp_mac, len);
  			}
  
  			if (crd->crd_alg == CRYPTO_DES_CBC ||
  			    crd->crd_alg == CRYPTO_3DES_CBC) {
  				if (crp->crp_flags & CRYPTO_F_IMBUF)
  					m_copyback((struct mbuf *)crp->crp_buf,
  					    crd->crd_skip, crd->crd_len,
  					    q->q_buf, M_NOWAIT);
  				else if (crp->crp_flags & CRYPTO_F_IOV)
  					cuio_copyback((struct uio *)crp->crp_buf,
  					    crd->crd_skip, crd->crd_len,
  					    q->q_buf);
  			}
  
  			bus_dmamap_unload(sc->sc_dmat, q->q_dmamap);
  			bus_dmamap_destroy(sc->sc_dmat, q->q_dmamap);
  			bus_dmamem_unmap(sc->sc_dmat, q->q_buf, crd->crd_len);
  			bus_dmamem_free(sc->sc_dmat, &q->q_dmaseg, rseg);
  			crp->crp_etype = 0;
  			free(q, M_DEVBUF);
  			s = splnet();
  			crypto_done(crp);
  		}
  		splx(s);
  
  		/* Handle input queue */
  		s = splnet();
  		while (!SIMPLEQ_EMPTY(&sc->sc_inq)) {
  			q = SIMPLEQ_FIRST(&sc->sc_inq);
  			SIMPLEQ_REMOVE_HEAD(&sc->sc_inq, q_next);
  			splx(s);
  
  			noct_ea_start(sc, q);
  			s = splnet();
  		}
  		splx(s);
  	}
  }
  
  void
  noct_ea_start(sc, q)
  	struct noct_softc *sc;
  	struct noct_workq *q;
  {
  	struct cryptop *crp;
  	struct cryptodesc *crd;
  	int s, err;
  
  	crp = q->q_crp;
  	crd = crp->crp_desc;
  
  	/* XXX Can't handle multiple ops yet */
  	if (crd->crd_next != NULL) {
  		err = EOPNOTSUPP;
  		goto errout;
  	}
  
  	switch (crd->crd_alg) {
  	case CRYPTO_MD5:
  	case CRYPTO_SHA1:
  		noct_ea_start_hash(sc, q, crp, crd);
  		break;
  	case CRYPTO_DES_CBC:
  	case CRYPTO_3DES_CBC:
  		noct_ea_start_des(sc, q, crp, crd);
  		break;
  	default:
  		err = EOPNOTSUPP;
  		goto errout;
  	}
  
  	return;
  
  errout:
  	crp->crp_etype = err;
  	free(q, M_DEVBUF);
  	s = splnet();
  	crypto_done(crp);
  	splx(s);
  }
  
  void
  noct_ea_start_hash(sc, q, crp, crd)
  	struct noct_softc *sc;
  	struct noct_workq *q;
  	struct cryptop *crp;
  	struct cryptodesc *crd;
  {
  	u_int64_t adr;
  	int s, err, i, rseg;
  	u_int32_t wp;
  
  	if (crd->crd_len > 0x4800) {
  		err = ERANGE;
  		goto errout;
  	}
  
  	if ((err = bus_dmamem_alloc(sc->sc_dmat, crd->crd_len, PAGE_SIZE, 0,
  	    &q->q_dmaseg, 1, &rseg, BUS_DMA_WAITOK | BUS_DMA_STREAMING)) != 0)
  		goto errout;
  
  	if ((err = bus_dmamem_map(sc->sc_dmat, &q->q_dmaseg, rseg,
  	    crd->crd_len, (caddr_t *)&q->q_buf, BUS_DMA_WAITOK)) != 0)
  		goto errout_dmafree;
  
  	if ((err = bus_dmamap_create(sc->sc_dmat, crd->crd_len, 1,
  	    crd->crd_len, 0, BUS_DMA_WAITOK, &q->q_dmamap)) != 0)
  		goto errout_dmaunmap;
  
  	if ((err = bus_dmamap_load_raw(sc->sc_dmat, q->q_dmamap, &q->q_dmaseg,
  	    rseg, crd->crd_len, BUS_DMA_WAITOK)) != 0)
  		goto errout_dmadestroy;
  
  	if (crp->crp_flags & CRYPTO_F_IMBUF)
  		m_copydata((struct mbuf *)crp->crp_buf,
  		    crd->crd_skip, crd->crd_len, q->q_buf);
  	else if (crp->crp_flags & CRYPTO_F_IOV)
  		cuio_copydata((struct uio *)crp->crp_buf,
  		    crd->crd_skip, crd->crd_len, q->q_buf);
  	else {
  		err = EINVAL;
  		goto errout_dmaunload;
  	}
  
  	bus_dmamap_sync(sc->sc_dmat, q->q_dmamap, 0, q->q_dmamap->dm_mapsize,
  	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
  
  	s = splnet();
  	if (noct_ea_nfree(sc) < 1) {
  		err = ENOMEM;
  		goto errout_dmaunload;
  	}
  	wp = sc->sc_eawp;
  	if (++sc->sc_eawp == NOCT_EA_ENTRIES)
  		sc->sc_eawp = 0;
  	for (i = 0; i < EA_CMD_WORDS; i++)
  		sc->sc_eacmd[wp].buf[i] = 0;
  	sc->sc_eacmd[wp].buf[0] = EA_0_SI;
  	switch (crd->crd_alg) {
  	case CRYPTO_MD5:
  		sc->sc_eacmd[wp].buf[1] = htole32(EA_OP_MD5);
  		break;
  	case CRYPTO_SHA1:
  		sc->sc_eacmd[wp].buf[1] = htole32(EA_OP_SHA1);
  		break;
  	}
  
  	/* Source, new buffer just allocated */
  	sc->sc_eacmd[wp].buf[1] |= htole32(crd->crd_len);
  	adr = q->q_dmamap->dm_segs[0].ds_addr;
  	sc->sc_eacmd[wp].buf[2] = htole32(adr >> 32);
  	sc->sc_eacmd[wp].buf[3] = htole32(adr & 0xffffffff);
  
  	/* Dest, hide it in the descriptor */
  	adr = sc->sc_eamap->dm_segs[0].ds_addr +
  	    (wp * sizeof(struct noct_ea_cmd)) +
  	    offsetof(struct noct_ea_cmd, buf[6]);
  	sc->sc_eacmd[wp].buf[4] = htole32(adr >> 32);
  	sc->sc_eacmd[wp].buf[5] = htole32(adr & 0xffffffff);
  
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_eamap,
  	    (wp * sizeof(struct noct_ea_cmd)), sizeof(struct noct_ea_cmd),
  	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
  
  	if (++wp == NOCT_EA_ENTRIES)
  		wp = 0;
  	NOCT_WRITE_4(sc, NOCT_EA_Q_PTR, wp);
  	sc->sc_eawp = wp;
  
  	SIMPLEQ_INSERT_TAIL(&sc->sc_chipq, q, q_next);
  	splx(s);
  
  	return;
  
  errout_dmaunload:
  	bus_dmamap_unload(sc->sc_dmat, q->q_dmamap);
  errout_dmadestroy:
  	bus_dmamap_destroy(sc->sc_dmat, q->q_dmamap);
  errout_dmaunmap:
  	bus_dmamem_unmap(sc->sc_dmat, q->q_buf, crd->crd_len);
  errout_dmafree:
  	bus_dmamem_free(sc->sc_dmat, &q->q_dmaseg, rseg);
  errout:
  	crp->crp_etype = err;
  	free(q, M_DEVBUF);
  	s = splnet();
  	crypto_done(crp);
  	splx(s);
  }
  
  void
  noct_ea_start_des(sc, q, crp, crd)
  	struct noct_softc *sc;
  	struct noct_workq *q;
  	struct cryptop *crp;
  	struct cryptodesc *crd;
  {
  	u_int64_t adr;
  	volatile u_int8_t *pb;
  	int s, err, i, rseg;
  	u_int32_t wp;
  	u_int8_t iv[8], key[24];
  
  	if (crd->crd_len > 0x4800) {
  		err = ERANGE;
  		goto errout;
  	}
  
  	if ((crd->crd_len & 3) != 0) {
  		err = ERANGE;
  		goto errout;
  	}
  
  	if (crd->crd_alg == CRYPTO_DES_CBC) {
  		for (i = 0; i < 8; i++)
  			key[i] = key[i + 8] = key[i + 16] = crd->crd_key[i];
  	} else {
  		for (i = 0; i < 24; i++)
  			key[i] = crd->crd_key[i];
  	}
  
  	if (crd->crd_flags & CRD_F_ENCRYPT) {
  		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
  			bcopy(crd->crd_iv, iv, 8);
  		else
  			arc4random_buf(iv, sizeof(iv));
  
  		if (!(crd->crd_flags & CRD_F_IV_PRESENT)) {
  			if (crp->crp_flags & CRYPTO_F_IMBUF)
  				m_copyback((struct mbuf *)crp->crp_buf,
  				    crd->crd_inject, 8, iv, M_NOWAIT);
  			else if (crp->crp_flags & CRYPTO_F_IOV)
  				cuio_copyback((struct uio *)crp->crp_buf,
  				    crd->crd_inject, 8, iv);
  		}
  	} else {
  		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
  			bcopy(crd->crd_iv, iv, 8);
  		else if (crp->crp_flags & CRYPTO_F_IMBUF)
  			m_copydata((struct mbuf *)crp->crp_buf,
  			    crd->crd_inject, 8, iv);
  		else if (crp->crp_flags & CRYPTO_F_IOV)
  			cuio_copydata((struct uio *)crp->crp_buf,
  			    crd->crd_inject, 8, iv);
  	}
  
  	if ((err = bus_dmamem_alloc(sc->sc_dmat, crd->crd_len, PAGE_SIZE, 0,
  	    &q->q_dmaseg, 1, &rseg, BUS_DMA_WAITOK | BUS_DMA_STREAMING)) != 0)
  		goto errout;
  
  	if ((err = bus_dmamem_map(sc->sc_dmat, &q->q_dmaseg, rseg,
  	    crd->crd_len, (caddr_t *)&q->q_buf, BUS_DMA_WAITOK)) != 0)
  		goto errout_dmafree;
  
  	if ((err = bus_dmamap_create(sc->sc_dmat, crd->crd_len, 1,
  	    crd->crd_len, 0, BUS_DMA_WAITOK, &q->q_dmamap)) != 0)
  		goto errout_dmaunmap;
  
  	if ((err = bus_dmamap_load_raw(sc->sc_dmat, q->q_dmamap, &q->q_dmaseg,
  	    rseg, crd->crd_len, BUS_DMA_WAITOK)) != 0)
  		goto errout_dmadestroy;
  
  	if (crp->crp_flags & CRYPTO_F_IMBUF)
  		m_copydata((struct mbuf *)crp->crp_buf,
  		    crd->crd_skip, crd->crd_len, q->q_buf);
  	else if (crp->crp_flags & CRYPTO_F_IOV)
  		cuio_copydata((struct uio *)crp->crp_buf,
  		    crd->crd_skip, crd->crd_len, q->q_buf);
  	else {
  		err = EINVAL;
  		goto errout_dmaunload;
  	}
  
  	bus_dmamap_sync(sc->sc_dmat, q->q_dmamap, 0, q->q_dmamap->dm_mapsize,
  	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
  
  	s = splnet();
  	if (noct_ea_nfree(sc) < 1) {
  		err = ENOMEM;
  		goto errout_dmaunload;
  	}
  	wp = sc->sc_eawp;
  	if (++sc->sc_eawp == NOCT_EA_ENTRIES)
  		sc->sc_eawp = 0;
  
  	for (i = 0; i < EA_CMD_WORDS; i++)
  		sc->sc_eacmd[wp].buf[i] = 0;
  
  	sc->sc_eacmd[wp].buf[0] = EA_0_SI;
  
  	if (crd->crd_flags & CRD_F_ENCRYPT)
  		sc->sc_eacmd[wp].buf[1] = htole32(EA_OP_3DESCBCE);
  	else
  		sc->sc_eacmd[wp].buf[1] = htole32(EA_OP_3DESCBCD);
  
  	/* Source, new buffer just allocated */
  	sc->sc_eacmd[wp].buf[1] |= htole32(crd->crd_len);
  	adr = q->q_dmamap->dm_segs[0].ds_addr;
  	sc->sc_eacmd[wp].buf[2] = htole32(adr >> 32);
  	sc->sc_eacmd[wp].buf[3] = htole32(adr & 0xffffffff);
  
  	/* Dest, same as source. */
  	sc->sc_eacmd[wp].buf[4] = htole32(adr >> 32);
  	sc->sc_eacmd[wp].buf[5] = htole32(adr & 0xffffffff);
  
  	/* IV and key */
  	pb = (volatile u_int8_t *)&sc->sc_eacmd[wp].buf[20];
  	for (i = 0; i < 8; i++)
  		pb[i] = iv[i];
  	SWAP32(sc->sc_eacmd[wp].buf[20]);
  	SWAP32(sc->sc_eacmd[wp].buf[21]);
  	pb = (volatile u_int8_t *)&sc->sc_eacmd[wp].buf[24];
  	for (i = 0; i < 24; i++)
  		pb[i] = key[i];
  	SWAP32(sc->sc_eacmd[wp].buf[24]);
  	SWAP32(sc->sc_eacmd[wp].buf[25]);
  	SWAP32(sc->sc_eacmd[wp].buf[26]);
  	SWAP32(sc->sc_eacmd[wp].buf[27]);
  	SWAP32(sc->sc_eacmd[wp].buf[28]);
  	SWAP32(sc->sc_eacmd[wp].buf[29]);
  
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_eamap,
  	    (wp * sizeof(struct noct_ea_cmd)), sizeof(struct noct_ea_cmd),
  	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
  
  	if (++wp == NOCT_EA_ENTRIES)
  		wp = 0;
  	NOCT_WRITE_4(sc, NOCT_EA_Q_PTR, wp);
  	sc->sc_eawp = wp;
  
  	SIMPLEQ_INSERT_TAIL(&sc->sc_chipq, q, q_next);
  	splx(s);
  
  	return;
  
  errout_dmaunload:
  	bus_dmamap_unload(sc->sc_dmat, q->q_dmamap);
  errout_dmadestroy:
  	bus_dmamap_destroy(sc->sc_dmat, q->q_dmamap);
  errout_dmaunmap:
  	bus_dmamem_unmap(sc->sc_dmat, q->q_buf, crd->crd_len);
  errout_dmafree:
  	bus_dmamem_free(sc->sc_dmat, &q->q_dmaseg, rseg);
  errout:
  	crp->crp_etype = err;
  	free(q, M_DEVBUF);
  	s = splnet();
  	crypto_done(crp);
  	splx(s);
  }
  
  void
  noct_ea_intr(sc)
  	struct noct_softc *sc;
  {
  	struct noct_workq *q;
  	u_int32_t csr, rp;
  
  	csr = NOCT_READ_4(sc, NOCT_EA_CSR);
  	NOCT_WRITE_4(sc, NOCT_EA_CSR, csr |
  	    EACSR_QALIGN | EACSR_CMDCMPL | EACSR_OPERR | EACSR_CMDREAD |
  	    EACSR_CMDWRITE | EACSR_DATAREAD | EACSR_DATAWRITE |
  	    EACSR_INTRNLLEN | EACSR_EXTRNLLEN | EACSR_DESBLOCK |
  	    EACSR_DESKEY | EACSR_ILL);
  
  	rp = (NOCT_READ_4(sc, NOCT_EA_Q_PTR) & EAQPTR_READ_M) >>
  	    EAQPTR_READ_S;
  	while (sc->sc_earp != rp) {
  		if (SIMPLEQ_EMPTY(&sc->sc_chipq))
  			panic("%s: empty chipq", sc->sc_dv.dv_xname);
  		q = SIMPLEQ_FIRST(&sc->sc_chipq);
  		SIMPLEQ_REMOVE_HEAD(&sc->sc_chipq, q_next);
  		SIMPLEQ_INSERT_TAIL(&sc->sc_outq, q, q_next);
  
  		bus_dmamap_sync(sc->sc_dmat, sc->sc_eamap,
  		    (sc->sc_earp * sizeof(struct noct_ea_cmd)),
  		    sizeof(struct noct_ea_cmd),
  		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
  		bcopy((u_int8_t *)&sc->sc_eacmd[sc->sc_earp].buf[6],
  		    q->q_macbuf, 20);
  
  		NOCT_WAKEUP(sc);
  		if (++sc->sc_earp == NOCT_EA_ENTRIES)
  			sc->sc_earp = 0;
  	}
  	sc->sc_earp = rp;
  
  	if (csr & EACSR_QALIGN)
  		printf("%s: ea bad queue alignment\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_OPERR)
  		printf("%s: ea bad opcode\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_CMDREAD)
  		printf("%s: ea command read error\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_CMDWRITE)
  		printf("%s: ea command write error\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_DATAREAD)
  		printf("%s: ea data read error\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_DATAWRITE)
  		printf("%s: ea data write error\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_INTRNLLEN)
  		printf("%s: ea bad internal len\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_EXTRNLLEN)
  		printf("%s: ea bad external len\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_DESBLOCK)
  		printf("%s: ea bad des block\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_DESKEY)
  		printf("%s: ea bad des key\n", sc->sc_dv.dv_xname);
  	if (csr & EACSR_ILL)
  		printf("%s: ea illegal access\n", sc->sc_dv.dv_xname);
  }
  
  void
  noct_write_8(sc, reg, val)
  	struct noct_softc *sc;
  	u_int32_t reg;
  	u_int64_t val;
  {
  	NOCT_WRITE_4(sc, reg, (val >> 32) & 0xffffffff);
  	NOCT_WRITE_4(sc, reg + 4, (val >> 0) & 0xffffffff);
  }
  
  u_int64_t
  noct_read_8(sc, reg)
  	struct noct_softc *sc;
  	u_int32_t reg;
  {
  	u_int64_t ret;
  
  	ret = NOCT_READ_4(sc, reg);
  	ret <<= 32;
  	ret |= NOCT_READ_4(sc, reg + 4);
  	return (ret);
  }
  
  /*
   * NSP2000 has a nifty bug, writes or reads to consecutive addresses
   * can be coalesced by a PCI bridge and executed as a burst read or
   * write which NSP2000's AMBA bridge doesn't grok.  Avoid the hazard.
   */
  u_int32_t
  noct_read_4(sc, off)
  	struct noct_softc *sc;
  	bus_size_t off;
  {
  	if (sc->sc_rar_last == off - 4 ||
  	    sc->sc_rar_last == off + 4) {
  		bus_space_write_4(sc->sc_st, sc->sc_sh, NOCT_BRDG_TEST, 0);
  		sc->sc_rar_last = off;
  		sc->sc_waw_last = 0xffffffff;
  	}
  	return (bus_space_read_4(sc->sc_st, sc->sc_sh, off));
  }
  
  void
  noct_write_4(sc, off, val)
  	struct noct_softc *sc;
  	bus_size_t off;
  	u_int32_t val;
  {
  	if (sc->sc_waw_last == off - 4 ||
  	    sc->sc_waw_last == off + 4) {
  		bus_space_read_4(sc->sc_st, sc->sc_sh, NOCT_BRDG_TEST);
  		sc->sc_waw_last = off;
  		sc->sc_rar_last = 0xffffffff;
  	}
  	bus_space_write_4(sc->sc_st, sc->sc_sh, off, val);
  }
  
  struct noct_softc *
  noct_kfind(krp)
  	struct cryptkop *krp;
  {
  	struct noct_softc *sc;
  	int i;
  
  	for (i = 0; i < noct_cd.cd_ndevs; i++) {
  		sc = noct_cd.cd_devs[i];
  		if (sc == NULL)
  			continue;
  		if (sc->sc_cid == krp->krp_hid)
  			return (sc);
  	}
  	return (NULL);
  }
  
  int
  noct_kprocess(krp)
  	struct cryptkop *krp;
  {
  	struct noct_softc *sc;
  
  	if (krp == NULL || krp->krp_callback == NULL)
  		return (EINVAL);
  	if ((sc = noct_kfind(krp)) == NULL) {
  		krp->krp_status = EINVAL;
  		crypto_kdone(krp);
  		return (0);
  	}
  
  	switch (krp->krp_op) {
  	case CRK_MOD_EXP:
  		noct_kprocess_modexp(sc, krp);
  		break;
  	default:
  		printf("%s: kprocess: invalid op 0x%x\n",
  		    sc->sc_dv.dv_xname, krp->krp_op);
  		krp->krp_status = EOPNOTSUPP;
  		crypto_kdone(krp);
  		break;
  	}
  	return (0);
  }
  
  u_int32_t
  noct_pkh_nfree(sc)
  	struct noct_softc *sc;
  {
  	if (sc->sc_pkhwp == sc->sc_pkhrp)
  		return (NOCT_PKH_ENTRIES);
  	if (sc->sc_pkhwp < sc->sc_pkhrp)
  		return (sc->sc_pkhrp - sc->sc_pkhwp - 1);
  	return (sc->sc_pkhrp + NOCT_PKH_ENTRIES - sc->sc_pkhwp - 1);
  }
  
  int
  noct_kprocess_modexp(sc, krp)
  	struct noct_softc *sc;
  	struct cryptkop *krp;
  {
  	int s, err;
  	u_int32_t wp, aidx, bidx, midx;
  	u_int64_t adr;
  	union noct_pkh_cmd *cmd;
  	int i, bits, mbits, digits, rmodidx, mmulidx;
  
  	s = splnet();
  	if (noct_pkh_nfree(sc) < 7) {
  		/* Need 7 entries: 3 loads, 1 store, 3 ops */
  		splx(s);
  		return (ENOMEM);
  	}
  
  	/* Load M */
  	midx = wp = sc->sc_pkhwp;
  	mbits = bits = noct_ksigbits(&krp->krp_param[2]);
  	if (bits > 4096) {
  		err = ERANGE;
  		goto errout;
  	}
  	sc->sc_pkh_bnsw[midx].bn_siz = (bits + 127) / 128;
  	if (extent_alloc(sc->sc_pkh_bn, sc->sc_pkh_bnsw[midx].bn_siz,
  	    EX_NOALIGN, 0, EX_NOBOUNDARY, EX_NOWAIT,
  	    &sc->sc_pkh_bnsw[midx].bn_off)) {
  		err = ENOMEM;
  		goto errout;
  	}
  	cmd = &sc->sc_pkhcmd[midx];
  	cmd->cache.op = htole32(PKH_OP_CODE_LOAD);
  	cmd->cache.r = htole32(sc->sc_pkh_bnsw[midx].bn_off);
  	adr = sc->sc_bnmap->dm_segs[0].ds_addr +
  	    (sc->sc_pkh_bnsw[midx].bn_off * 16);
  	cmd->cache.addrhi = htole32((adr >> 32) & 0xffffffff);
  	cmd->cache.addrlo = htole32((adr >> 0 ) & 0xffffffff);
  	cmd->cache.len = htole32(sc->sc_pkh_bnsw[midx].bn_siz);
  	cmd->cache.unused[0] = cmd->cache.unused[1] = cmd->cache.unused[2] = 0;
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_pkhmap,
  	    midx * sizeof(union noct_pkh_cmd), sizeof(union noct_pkh_cmd),
  	    BUS_DMASYNC_PREWRITE);
  	for (i = 0; i < (digits * 16); i++)
  		sc->sc_bncache[(sc->sc_pkh_bnsw[midx].bn_off * 16) + i] = 0;
  	for (i = 0; i < ((bits + 7) / 8); i++)
  		sc->sc_bncache[(sc->sc_pkh_bnsw[midx].bn_off * 16) +
  		    (digits * 16) - 1 - i] = krp->krp_param[2].crp_p[i];
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_bnmap,
  	    sc->sc_pkh_bnsw[midx].bn_off * 16, digits * 16,
  	    BUS_DMASYNC_PREWRITE);
  	if (++wp == NOCT_PKH_ENTRIES)
  		wp = 0;
  
  	/* Store RMOD(m) -> location tmp1 */
  	rmodidx = wp;
  	sc->sc_pkh_bnsw[rmodidx].bn_siz = sc->sc_pkh_bnsw[midx].bn_siz;
  	if (extent_alloc(sc->sc_pkh_bn, sc->sc_pkh_bnsw[rmodidx].bn_siz,
  	    EX_NOALIGN, 0, EX_NOBOUNDARY, EX_NOWAIT,
  	    &sc->sc_pkh_bnsw[rmodidx].bn_off)) {
  		err = ENOMEM;
  		goto errout_m;
  	}
  	cmd = &sc->sc_pkhcmd[rmodidx];
  	cmd->arith.op = htole32(PKH_OP_CODE_RMOD);
  	cmd->arith.r = htole32(sc->sc_pkh_bnsw[rmodidx].bn_off);
  	cmd->arith.m = htole32(sc->sc_pkh_bnsw[midx].bn_off |
  	    (sc->sc_pkh_bnsw[midx].bn_siz << 16));
  	cmd->arith.a = cmd->arith.b = cmd->arith.c = cmd->arith.unused[0] =
  	    cmd->arith.unused[1] = 0;
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_pkhmap,
  	    rmodidx * sizeof(union noct_pkh_cmd), sizeof(union noct_pkh_cmd),
  	    BUS_DMASYNC_PREWRITE);
  	if (++wp == NOCT_PKH_ENTRIES)
  		wp = 0;
  
  	/* Load A XXX deal with A < M padding ... */
  	aidx = wp = sc->sc_pkhwp;
  	bits = noct_ksigbits(&krp->krp_param[0]);
  	if (bits > 4096 || bits > mbits) {
  		err = ERANGE;
  		goto errout_rmod;
  	}
  	sc->sc_pkh_bnsw[aidx].bn_siz = (bits + 127) / 128;
  	if (extent_alloc(sc->sc_pkh_bn, sc->sc_pkh_bnsw[aidx].bn_siz,
  	    EX_NOALIGN, 0, EX_NOBOUNDARY, EX_NOWAIT,
  	    &sc->sc_pkh_bnsw[aidx].bn_off)) {
  		err = ENOMEM;
  		goto errout_rmod;
  	}
  	cmd = &sc->sc_pkhcmd[aidx];
  	cmd->cache.op = htole32(PKH_OP_CODE_LOAD);
  	cmd->cache.r = htole32(sc->sc_pkh_bnsw[aidx].bn_off);
  	adr = sc->sc_bnmap->dm_segs[0].ds_addr +
  	    (sc->sc_pkh_bnsw[aidx].bn_off * 16);
  	cmd->cache.addrhi = htole32((adr >> 32) & 0xffffffff);
  	cmd->cache.addrlo = htole32((adr >> 0 ) & 0xffffffff);
  	cmd->cache.len = htole32(sc->sc_pkh_bnsw[aidx].bn_siz);
  	cmd->cache.unused[0] = cmd->cache.unused[1] = cmd->cache.unused[2] = 0;
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_pkhmap,
  	    aidx * sizeof(union noct_pkh_cmd), sizeof(union noct_pkh_cmd),
  	    BUS_DMASYNC_PREWRITE);
  	for (i = 0; i < (digits * 16); i++)
  		sc->sc_bncache[(sc->sc_pkh_bnsw[aidx].bn_off * 16) + i] = 0;
  	for (i = 0; i < ((bits + 7) / 8); i++)
  		sc->sc_bncache[(sc->sc_pkh_bnsw[aidx].bn_off * 16) +
  		    (digits * 16) - 1 - i] = krp->krp_param[2].crp_p[i];
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_bnmap,
  	    sc->sc_pkh_bnsw[aidx].bn_off * 16, digits * 16,
  	    BUS_DMASYNC_PREWRITE);
  	if (++wp == NOCT_PKH_ENTRIES)
  		wp = 0;
  
  	/* Compute (A * tmp1) mod m -> A */
  	mmulidx = wp;
  	sc->sc_pkh_bnsw[mmulidx].bn_siz = 0;
  	sc->sc_pkh_bnsw[mmulidx].bn_off = 0;
  	cmd = &sc->sc_pkhcmd[mmulidx];
  	cmd->arith.op = htole32(PKH_OP_CODE_MUL);
  	cmd->arith.r = htole32(sc->sc_pkh_bnsw[aidx].bn_off);
  	cmd->arith.m = htole32(sc->sc_pkh_bnsw[midx].bn_off |
  	    (sc->sc_pkh_bnsw[midx].bn_siz << 16));
  	cmd->arith.a = htole32(sc->sc_pkh_bnsw[aidx].bn_off |
  	    (sc->sc_pkh_bnsw[aidx].bn_siz << 16));
  	cmd->arith.b = htole32(sc->sc_pkh_bnsw[rmodidx].bn_off |
  	    (sc->sc_pkh_bnsw[rmodidx].bn_siz << 16));
  	cmd->arith.c = cmd->arith.unused[0] = cmd->arith.unused[1] = 0;
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_pkhmap,
  	    rmodidx * sizeof(union noct_pkh_cmd), sizeof(union noct_pkh_cmd),
  	    BUS_DMASYNC_PREWRITE);
  	if (++wp == NOCT_PKH_ENTRIES)
  		wp = 0;
  
  	/* Load B */
  	bidx = wp = sc->sc_pkhwp;
  	bits = noct_ksigbits(&krp->krp_param[1]);
  	if (bits > 4096) {
  		err = ERANGE;
  		goto errout_a;
  	}
  	sc->sc_pkh_bnsw[bidx].bn_siz = (bits + 127) / 128;
  	if (extent_alloc(sc->sc_pkh_bn, sc->sc_pkh_bnsw[bidx].bn_siz,
  	    EX_NOALIGN, 0, EX_NOBOUNDARY, EX_NOWAIT,
  	    &sc->sc_pkh_bnsw[bidx].bn_off)) {
  		err = ENOMEM;
  		goto errout_a;
  	}
  	cmd = &sc->sc_pkhcmd[bidx];
  	cmd->cache.op = htole32(PKH_OP_CODE_LOAD);
  	cmd->cache.r = htole32(sc->sc_pkh_bnsw[bidx].bn_off);
  	adr = sc->sc_bnmap->dm_segs[0].ds_addr +
  	    (sc->sc_pkh_bnsw[bidx].bn_off * 16);
  	cmd->cache.addrhi = htole32((adr >> 32) & 0xffffffff);
  	cmd->cache.addrlo = htole32((adr >> 0 ) & 0xffffffff);
  	cmd->cache.len = htole32(sc->sc_pkh_bnsw[bidx].bn_siz);
  	cmd->cache.unused[0] = cmd->cache.unused[1] = cmd->cache.unused[2] = 0;
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_pkhmap,
  	    bidx * sizeof(union noct_pkh_cmd), sizeof(union noct_pkh_cmd),
  	    BUS_DMASYNC_PREWRITE);
  	for (i = 0; i < (digits * 16); i++)
  		sc->sc_bncache[(sc->sc_pkh_bnsw[bidx].bn_off * 16) + i] = 0;
  	for (i = 0; i < ((bits + 7) / 8); i++)
  		sc->sc_bncache[(sc->sc_pkh_bnsw[bidx].bn_off * 16) +
  		    (digits * 16) - 1 - i] = krp->krp_param[2].crp_p[i];
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_bnmap,
  	    sc->sc_pkh_bnsw[bidx].bn_off * 16, digits * 16,
  	    BUS_DMASYNC_PREWRITE);
  	if (++wp == NOCT_PKH_ENTRIES)
  		wp = 0;
  
  	NOCT_WRITE_4(sc, NOCT_PKH_Q_PTR, wp);
  	sc->sc_pkhwp = wp;
  
  	splx(s);
  
  	return (0);
  
  errout_a:
  	extent_free(sc->sc_pkh_bn, sc->sc_pkh_bnsw[aidx].bn_off,
  	    sc->sc_pkh_bnsw[aidx].bn_siz, EX_NOWAIT);
  errout_rmod:
  	extent_free(sc->sc_pkh_bn, sc->sc_pkh_bnsw[rmodidx].bn_off,
  	    sc->sc_pkh_bnsw[rmodidx].bn_siz, EX_NOWAIT);
  errout_m:
  	extent_free(sc->sc_pkh_bn, sc->sc_pkh_bnsw[midx].bn_off,
  	    sc->sc_pkh_bnsw[midx].bn_siz, EX_NOWAIT);
  errout:
  	splx(s);
  	krp->krp_status = err;
  	crypto_kdone(krp);
  	return (1);
  }
  
  void
  noct_pkh_freedesc(sc, idx)
  	struct noct_softc *sc;
  	int idx;
  {
  	if (sc->sc_pkh_bnsw[idx].bn_callback != NULL)
  		(*sc->sc_pkh_bnsw[idx].bn_callback)(sc, idx, 0);
  }
  
  /*
   * Return the number of significant bits of a big number.
   */
  int
  noct_ksigbits(cr)
  	struct crparam *cr;
  {
  	u_int plen = (cr->crp_nbits + 7) / 8;
  	int i, sig = plen * 8;
  	u_int8_t c, *p = cr->crp_p;
  
  	for (i = plen - 1; i >= 0; i--) {
  		c = p[i];
  		if (c != 0) {
  			while ((c & 0x80) == 0) {
  				sig--;
  				c <<= 1;
  			}
  			break;
  		}
  		sig -= 8;
  	}
  	return (sig);
  }
  
  int
  noct_kload(sc, cr, wp)
  	struct noct_softc *sc;
  	struct crparam *cr;
  	u_int32_t wp;
  {
  	u_int64_t adr;
  	union noct_pkh_cmd *cmd;
  	u_long off;
  	int bits, digits, i;
  	u_int32_t wpnext;
  
  	wpnext = wp + 1;
  	if (wpnext == NOCT_PKH_ENTRIES)
  		wpnext = 0;
  	if (wpnext == sc->sc_pkhrp)
  		return (ENOMEM);
  
  	bits = noct_ksigbits(cr);
  	if (bits > 4096)
  		return (E2BIG);
  
  	digits = (bits + 127) / 128;
  
  	if (extent_alloc(sc->sc_pkh_bn, digits, EX_NOALIGN, 0, EX_NOBOUNDARY,
  	    EX_NOWAIT, &off))
  		return (ENOMEM);
  
  	cmd = &sc->sc_pkhcmd[wp];
  	cmd->cache.op = htole32(PKH_OP_CODE_LOAD);
  	cmd->cache.r = htole32(off);
  	adr = sc->sc_bnmap->dm_segs[0].ds_addr + (off * 16);
  	cmd->cache.addrhi = htole32((adr >> 32) & 0xffffffff);
  	cmd->cache.addrlo = htole32((adr >> 0 ) & 0xffffffff);
  	cmd->cache.len = htole32(digits * 16);
  	cmd->cache.unused[0] = cmd->cache.unused[1] = cmd->cache.unused[2] = 0;
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_pkhmap,
  	    wp * sizeof(union noct_pkh_cmd), sizeof(union noct_pkh_cmd),
  	    BUS_DMASYNC_PREWRITE);
  
  	for (i = 0; i < (digits * 16); i++)
  		sc->sc_bncache[(off * 16) + i] = 0;
  	for (i = 0; i < ((bits + 7) / 8); i++)
  		sc->sc_bncache[(off * 16) + (digits * 16) - 1 - i] =
  		    cr->crp_p[i];
  	bus_dmamap_sync(sc->sc_dmat, sc->sc_bnmap, off * 16, digits * 16,
  	    BUS_DMASYNC_PREWRITE);
  
  	sc->sc_pkh_bnsw[wp].bn_off = off;
  	sc->sc_pkh_bnsw[wp].bn_siz = digits;
  	sc->sc_pkh_bnsw[wp].bn_callback = noct_kload_cb;
  	return (0);
  }
  
  void
  noct_kload_cb(sc, wp, err)
  	struct noct_softc *sc;
  	u_int32_t wp;
  	int err;
  {
  	struct noct_bnc_sw *sw = &sc->sc_pkh_bnsw[wp];
  
  	extent_free(sc->sc_pkh_bn, sw->bn_off, sw->bn_siz, EX_NOWAIT);
  	explicit_bzero(&sc->sc_bncache[sw->bn_off * 16], sw->bn_siz * 16);
  }
  
  void
  noct_modmul_cb(sc, wp, err)
  	struct noct_softc *sc;
  	u_int32_t wp;
  	int err;
  {
  	struct noct_bnc_sw *sw = &sc->sc_pkh_bnsw[wp];
  	struct cryptkop *krp = sw->bn_krp;
  	int i, j;
  
  	if (err)
  		goto out;
  
  	i = (sw->bn_off * 16) + (sw->bn_siz * 16) - 1;
  	for (j = 0; j < (krp->krp_param[3].crp_nbits + 7) / 8; j++) {
  		krp->krp_param[3].crp_p[j] = sc->sc_bncache[i];
  		i--;
  	}
  
  out:
  	extent_free(sc->sc_pkh_bn, sw->bn_off, sw->bn_siz, EX_NOWAIT);
  	explicit_bzero(&sc->sc_bncache[sw->bn_off * 16], sw->bn_siz * 16);
  	krp->krp_status = err;
  	crypto_kdone(krp);
  }
  
  static const u_int8_t noct_odd_parity[] = {
  	0x01, 0x01, 0x02, 0x02, 0x04, 0x04, 0x07, 0x07,
  	0x08, 0x08, 0x0b, 0x0b, 0x0d, 0x0d, 0x0e, 0x0e,
  	0x10, 0x10, 0x13, 0x13, 0x15, 0x15, 0x16, 0x16,
  	0x19, 0x19, 0x1a, 0x1a, 0x1c, 0x1c, 0x1f, 0x1f,
  	0x20, 0x20, 0x23, 0x23, 0x25, 0x25, 0x26, 0x26,
  	0x29, 0x29, 0x2a, 0x2a, 0x2c, 0x2c, 0x2f, 0x2f,
  	0x31, 0x31, 0x32, 0x32, 0x34, 0x34, 0x37, 0x37,
  	0x38, 0x38, 0x3b, 0x3b, 0x3d, 0x3d, 0x3e, 0x3e,
  	0x40, 0x40, 0x43, 0x43, 0x45, 0x45, 0x46, 0x46,
  	0x49, 0x49, 0x4a, 0x4a, 0x4c, 0x4c, 0x4f, 0x4f,
  	0x51, 0x51, 0x52, 0x52, 0x54, 0x54, 0x57, 0x57,
  	0x58, 0x58, 0x5b, 0x5b, 0x5d, 0x5d, 0x5e, 0x5e,
  	0x61, 0x61, 0x62, 0x62, 0x64, 0x64, 0x67, 0x67,
  	0x68, 0x68, 0x6b, 0x6b, 0x6d, 0x6d, 0x6e, 0x6e,
  	0x70, 0x70, 0x73, 0x73, 0x75, 0x75, 0x76, 0x76,
  	0x79, 0x79, 0x7a, 0x7a, 0x7c, 0x7c, 0x7f, 0x7f,
  	0x80, 0x80, 0x83, 0x83, 0x85, 0x85, 0x86, 0x86,
  	0x89, 0x89, 0x8a, 0x8a, 0x8c, 0x8c, 0x8f, 0x8f,
  	0x91, 0x91, 0x92, 0x92, 0x94, 0x94, 0x97, 0x97,
  	0x98, 0x98, 0x9b, 0x9b, 0x9d, 0x9d, 0x9e, 0x9e,
  	0xa1, 0xa1, 0xa2, 0xa2, 0xa4, 0xa4, 0xa7, 0xa7,
  	0xa8, 0xa8, 0xab, 0xab, 0xad, 0xad, 0xae, 0xae,
  	0xb0, 0xb0, 0xb3, 0xb3, 0xb5, 0xb5, 0xb6, 0xb6,
  	0xb9, 0xb9, 0xba, 0xba, 0xbc, 0xbc, 0xbf, 0xbf,
  	0xc1, 0xc1, 0xc2, 0xc2, 0xc4, 0xc4, 0xc7, 0xc7,
  	0xc8, 0xc8, 0xcb, 0xcb, 0xcd, 0xcd, 0xce, 0xce,
  	0xd0, 0xd0, 0xd3, 0xd3, 0xd5, 0xd5, 0xd6, 0xd6,
  	0xd9, 0xd9, 0xda, 0xda, 0xdc, 0xdc, 0xdf, 0xdf,
  	0xe0, 0xe0, 0xe3, 0xe3, 0xe5, 0xe5, 0xe6, 0xe6,
  	0xe9, 0xe9, 0xea, 0xea, 0xec, 0xec, 0xef, 0xef,
  	0xf1, 0xf1, 0xf2, 0xf2, 0xf4, 0xf4, 0xf7, 0xf7,
  	0xf8, 0xf8, 0xfb, 0xfb, 0xfd, 0xfd, 0xfe, 0xfe,
  };
  
  int
  noct_newsession(sidp, cri)
  	u_int32_t *sidp;
  	struct cryptoini *cri;
  {
  	struct noct_softc *sc;
  	int i;
  
  	for (i = 0; i < noct_cd.cd_ndevs; i++) {
  		sc = noct_cd.cd_devs[i];
  		if (sc == NULL || sc->sc_cid == (*sidp))
  			break;
  	}
  	if (sc == NULL)
  		return (EINVAL);
  
  	/* XXX Can only handle single operations */
  	if (cri->cri_next != NULL)
  		return (EINVAL);
  
  	if (cri->cri_alg == CRYPTO_DES_CBC || cri->cri_alg == CRYPTO_3DES_CBC) {
  		u_int8_t key[24];
  
  		if (cri->cri_alg == CRYPTO_DES_CBC) {
  			if (cri->cri_klen != 64)
  				return (EINVAL);
  			for (i = 0; i < 8; i++)
  				key[i] = key[i + 8] = key[i + 16] =
  				    cri->cri_key[i];
  		} else {
  			if (cri->cri_klen != 192)
  				return (EINVAL);
  			for (i = 0; i < 24; i++)
  				key[i] = cri->cri_key[i];
  		}
  
  		/* Verify key parity */
  		for (i = 0; i < 24; i++)
  			if (key[i] != noct_odd_parity[key[i]])
  				return (ENOEXEC);
  	}
  
  	*sidp = NOCT_SID(sc->sc_dv.dv_unit, 0);
  	return (0);
  }
  
  int
  noct_freesession(tid)
  	u_int64_t tid;
  {
  	int card;
  	u_int32_t sid = ((u_int32_t)tid) & 0xffffffff;
  
  	card = NOCT_CARD(sid);
  	if (card >= noct_cd.cd_ndevs || noct_cd.cd_devs[card] == NULL)
  		return (EINVAL);
  	return (0);
  }
  
  int
  noct_process(crp)
  	struct cryptop *crp;
  {
  	struct noct_softc *sc;
  	struct noct_workq *q = NULL;
  	int card, err, s;
  
  	if (crp == NULL || crp->crp_callback == NULL)
  		return (EINVAL);
  
  	card = NOCT_CARD(crp->crp_sid);
  	if (card >= noct_cd.cd_ndevs || noct_cd.cd_devs[card] == NULL)
  		return (EINVAL);
  	sc = noct_cd.cd_devs[card];
  
  	q = (struct noct_workq *)malloc(sizeof(struct noct_workq),
  	    M_DEVBUF, M_NOWAIT);
  	if (q == NULL) {
  		err = ENOMEM;
  		goto errout;
  	}
  	q->q_crp = crp;
  
  	s = splnet();
  	SIMPLEQ_INSERT_TAIL(&sc->sc_inq, q, q_next);
  	splx(s);
  	NOCT_WAKEUP(sc);
  	return (0);
  
  errout:
  	if (q != NULL)
  		free(q, M_DEVBUF);
  	crp->crp_etype = err;
  	crypto_done(crp);
  	return (0);
  }
  