xref: /freebsd/sys/dev/sec/sec.c (revision 661ee6eea523d27a894f2f6869db2661d64bd44e)
102b553caSRafal Jaworowski /*-
202b553caSRafal Jaworowski  * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik
302b553caSRafal Jaworowski  * All rights reserved.
402b553caSRafal Jaworowski  *
502b553caSRafal Jaworowski  * Redistribution and use in source and binary forms, with or without
602b553caSRafal Jaworowski  * modification, are permitted provided that the following conditions
702b553caSRafal Jaworowski  * are met:
802b553caSRafal Jaworowski  * 1. Redistributions of source code must retain the above copyright
902b553caSRafal Jaworowski  *    notice, this list of conditions and the following disclaimer.
1002b553caSRafal Jaworowski  * 2. Redistributions in binary form must reproduce the above copyright
1102b553caSRafal Jaworowski  *    notice, this list of conditions and the following disclaimer in the
1202b553caSRafal Jaworowski  *    documentation and/or other materials provided with the distribution.
1302b553caSRafal Jaworowski  *
1402b553caSRafal Jaworowski  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
1502b553caSRafal Jaworowski  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
1602b553caSRafal Jaworowski  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
1702b553caSRafal Jaworowski  * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
1802b553caSRafal Jaworowski  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
1902b553caSRafal Jaworowski  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
2002b553caSRafal Jaworowski  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
2102b553caSRafal Jaworowski  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
2202b553caSRafal Jaworowski  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
2302b553caSRafal Jaworowski  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2402b553caSRafal Jaworowski  */
2502b553caSRafal Jaworowski 
2602b553caSRafal Jaworowski /*
2702b553caSRafal Jaworowski  * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and
2802b553caSRafal Jaworowski  * 3.0 are supported.
2902b553caSRafal Jaworowski  */
3002b553caSRafal Jaworowski 
3102b553caSRafal Jaworowski #include <sys/cdefs.h>
3202b553caSRafal Jaworowski __FBSDID("$FreeBSD$");
3302b553caSRafal Jaworowski 
3402b553caSRafal Jaworowski #include <sys/param.h>
3502b553caSRafal Jaworowski #include <sys/systm.h>
3602b553caSRafal Jaworowski #include <sys/bus.h>
3702b553caSRafal Jaworowski #include <sys/endian.h>
3802b553caSRafal Jaworowski #include <sys/kernel.h>
3902b553caSRafal Jaworowski #include <sys/lock.h>
4002b553caSRafal Jaworowski #include <sys/malloc.h>
4102b553caSRafal Jaworowski #include <sys/mbuf.h>
4202b553caSRafal Jaworowski #include <sys/module.h>
4302b553caSRafal Jaworowski #include <sys/mutex.h>
4402b553caSRafal Jaworowski #include <sys/random.h>
4502b553caSRafal Jaworowski #include <sys/rman.h>
4602b553caSRafal Jaworowski 
4702b553caSRafal Jaworowski #include <machine/bus.h>
4802b553caSRafal Jaworowski #include <machine/ocpbus.h>
4902b553caSRafal Jaworowski #include <machine/resource.h>
5002b553caSRafal Jaworowski 
5102b553caSRafal Jaworowski #include <opencrypto/cryptodev.h>
5202b553caSRafal Jaworowski #include "cryptodev_if.h"
5302b553caSRafal Jaworowski 
5402b553caSRafal Jaworowski #include <dev/sec/sec.h>
5502b553caSRafal Jaworowski 
5602b553caSRafal Jaworowski static int	sec_probe(device_t dev);
5702b553caSRafal Jaworowski static int	sec_attach(device_t dev);
5802b553caSRafal Jaworowski static int	sec_detach(device_t dev);
5902b553caSRafal Jaworowski static int	sec_suspend(device_t dev);
6002b553caSRafal Jaworowski static int	sec_resume(device_t dev);
61661ee6eeSRafal Jaworowski static int	sec_shutdown(device_t dev);
6202b553caSRafal Jaworowski static void	sec_primary_intr(void *arg);
6302b553caSRafal Jaworowski static void	sec_secondary_intr(void *arg);
6402b553caSRafal Jaworowski static int	sec_setup_intr(struct sec_softc *sc, struct resource **ires,
6502b553caSRafal Jaworowski     void **ihand, int *irid, driver_intr_t handler, const char *iname);
6602b553caSRafal Jaworowski static void	sec_release_intr(struct sec_softc *sc, struct resource *ires,
6702b553caSRafal Jaworowski     void *ihand, int irid, const char *iname);
6802b553caSRafal Jaworowski static int	sec_controller_reset(struct sec_softc *sc);
6902b553caSRafal Jaworowski static int	sec_channel_reset(struct sec_softc *sc, int channel, int full);
7002b553caSRafal Jaworowski static int	sec_init(struct sec_softc *sc);
7102b553caSRafal Jaworowski static int	sec_alloc_dma_mem(struct sec_softc *sc,
7202b553caSRafal Jaworowski     struct sec_dma_mem *dma_mem, bus_size_t size);
7302b553caSRafal Jaworowski static int	sec_desc_map_dma(struct sec_softc *sc,
7402b553caSRafal Jaworowski     struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type,
7502b553caSRafal Jaworowski     struct sec_desc_map_info *sdmi);
7602b553caSRafal Jaworowski static void	sec_free_dma_mem(struct sec_dma_mem *dma_mem);
7702b553caSRafal Jaworowski static void	sec_enqueue(struct sec_softc *sc);
7802b553caSRafal Jaworowski static int	sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc,
7902b553caSRafal Jaworowski     int channel);
8002b553caSRafal Jaworowski static int	sec_eu_channel(struct sec_softc *sc, int eu);
8102b553caSRafal Jaworowski static int	sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
8202b553caSRafal Jaworowski     u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype);
8302b553caSRafal Jaworowski static int	sec_make_pointer_direct(struct sec_softc *sc,
8402b553caSRafal Jaworowski     struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
8502b553caSRafal Jaworowski static int	sec_alloc_session(struct sec_softc *sc);
8602b553caSRafal Jaworowski static int	sec_newsession(device_t dev, u_int32_t *sidp,
8702b553caSRafal Jaworowski     struct cryptoini *cri);
8802b553caSRafal Jaworowski static int	sec_freesession(device_t dev, uint64_t tid);
8902b553caSRafal Jaworowski static int	sec_process(device_t dev, struct cryptop *crp, int hint);
9002b553caSRafal Jaworowski static int	sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
9102b553caSRafal Jaworowski     struct cryptoini **mac);
9202b553caSRafal Jaworowski static int	sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
9302b553caSRafal Jaworowski     struct cryptodesc **mac);
9402b553caSRafal Jaworowski static int	sec_build_common_ns_desc(struct sec_softc *sc,
9502b553caSRafal Jaworowski     struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
9602b553caSRafal Jaworowski     struct cryptodesc *enc, int buftype);
9702b553caSRafal Jaworowski static int	sec_build_common_s_desc(struct sec_softc *sc,
9802b553caSRafal Jaworowski     struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
9902b553caSRafal Jaworowski     struct cryptodesc *enc, struct cryptodesc *mac, int buftype);
10002b553caSRafal Jaworowski 
10102b553caSRafal Jaworowski static struct sec_session *sec_get_session(struct sec_softc *sc, u_int sid);
10202b553caSRafal Jaworowski static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
10302b553caSRafal Jaworowski 
10402b553caSRafal Jaworowski /* AESU */
10502b553caSRafal Jaworowski static int	sec_aesu_newsession(struct sec_softc *sc,
10602b553caSRafal Jaworowski     struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
10702b553caSRafal Jaworowski static int	sec_aesu_make_desc(struct sec_softc *sc,
10802b553caSRafal Jaworowski     struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
10902b553caSRafal Jaworowski     int buftype);
11002b553caSRafal Jaworowski 
11102b553caSRafal Jaworowski /* DEU */
11202b553caSRafal Jaworowski static int	sec_deu_newsession(struct sec_softc *sc,
11302b553caSRafal Jaworowski     struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
11402b553caSRafal Jaworowski static int	sec_deu_make_desc(struct sec_softc *sc,
11502b553caSRafal Jaworowski     struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
11602b553caSRafal Jaworowski     int buftype);
11702b553caSRafal Jaworowski 
11802b553caSRafal Jaworowski /* MDEU */
11902b553caSRafal Jaworowski static int	sec_mdeu_can_handle(u_int alg);
12002b553caSRafal Jaworowski static int	sec_mdeu_config(struct cryptodesc *crd,
12102b553caSRafal Jaworowski     u_int *eu, u_int *mode, u_int *hashlen);
12202b553caSRafal Jaworowski static int	sec_mdeu_newsession(struct sec_softc *sc,
12302b553caSRafal Jaworowski     struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
12402b553caSRafal Jaworowski static int	sec_mdeu_make_desc(struct sec_softc *sc,
12502b553caSRafal Jaworowski     struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
12602b553caSRafal Jaworowski     int buftype);
12702b553caSRafal Jaworowski 
12802b553caSRafal Jaworowski static device_method_t sec_methods[] = {
12902b553caSRafal Jaworowski 	/* Device interface */
13002b553caSRafal Jaworowski 	DEVMETHOD(device_probe,		sec_probe),
13102b553caSRafal Jaworowski 	DEVMETHOD(device_attach,	sec_attach),
13202b553caSRafal Jaworowski 	DEVMETHOD(device_detach,	sec_detach),
13302b553caSRafal Jaworowski 
13402b553caSRafal Jaworowski 	DEVMETHOD(device_suspend,	sec_suspend),
13502b553caSRafal Jaworowski 	DEVMETHOD(device_resume,	sec_resume),
13602b553caSRafal Jaworowski 	DEVMETHOD(device_shutdown,	sec_shutdown),
13702b553caSRafal Jaworowski 
13802b553caSRafal Jaworowski 	/* Bus interface */
13902b553caSRafal Jaworowski 	DEVMETHOD(bus_print_child,	bus_generic_print_child),
14002b553caSRafal Jaworowski 	DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
14102b553caSRafal Jaworowski 
14202b553caSRafal Jaworowski 	/* Crypto methods */
14302b553caSRafal Jaworowski 	DEVMETHOD(cryptodev_newsession,	sec_newsession),
14402b553caSRafal Jaworowski 	DEVMETHOD(cryptodev_freesession,sec_freesession),
14502b553caSRafal Jaworowski 	DEVMETHOD(cryptodev_process,	sec_process),
14602b553caSRafal Jaworowski 
14702b553caSRafal Jaworowski 	{ 0, 0 }
14802b553caSRafal Jaworowski };
14902b553caSRafal Jaworowski static driver_t sec_driver = {
15002b553caSRafal Jaworowski 	"sec",
15102b553caSRafal Jaworowski 	sec_methods,
15202b553caSRafal Jaworowski 	sizeof(struct sec_softc),
15302b553caSRafal Jaworowski };
15402b553caSRafal Jaworowski 
15502b553caSRafal Jaworowski static devclass_t sec_devclass;
15602b553caSRafal Jaworowski DRIVER_MODULE(sec, ocpbus, sec_driver, sec_devclass, 0, 0);
15702b553caSRafal Jaworowski MODULE_DEPEND(sec, crypto, 1, 1, 1);
15802b553caSRafal Jaworowski 
15902b553caSRafal Jaworowski static struct sec_eu_methods sec_eus[] = {
16002b553caSRafal Jaworowski 	{
16102b553caSRafal Jaworowski 		sec_aesu_newsession,
16202b553caSRafal Jaworowski 		sec_aesu_make_desc,
16302b553caSRafal Jaworowski 	},
16402b553caSRafal Jaworowski 	{
16502b553caSRafal Jaworowski 		sec_deu_newsession,
16602b553caSRafal Jaworowski 		sec_deu_make_desc,
16702b553caSRafal Jaworowski 	},
16802b553caSRafal Jaworowski 	{
16902b553caSRafal Jaworowski 		sec_mdeu_newsession,
17002b553caSRafal Jaworowski 		sec_mdeu_make_desc,
17102b553caSRafal Jaworowski 	},
17202b553caSRafal Jaworowski 	{ NULL, NULL }
17302b553caSRafal Jaworowski };
17402b553caSRafal Jaworowski 
17502b553caSRafal Jaworowski static inline void
17602b553caSRafal Jaworowski sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op)
17702b553caSRafal Jaworowski {
17802b553caSRafal Jaworowski 
17902b553caSRafal Jaworowski 	/* Sync only if dma memory is valid */
18002b553caSRafal Jaworowski 	if (dma_mem->dma_vaddr != NULL)
18102b553caSRafal Jaworowski 		bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op);
18202b553caSRafal Jaworowski }
18302b553caSRafal Jaworowski 
18402b553caSRafal Jaworowski static inline void
18502b553caSRafal Jaworowski sec_free_session(struct sec_softc *sc, struct sec_session *ses)
18602b553caSRafal Jaworowski {
18702b553caSRafal Jaworowski 
18802b553caSRafal Jaworowski 	SEC_LOCK(sc, sessions);
18902b553caSRafal Jaworowski 	ses->ss_used = 0;
19002b553caSRafal Jaworowski 	SEC_UNLOCK(sc, sessions);
19102b553caSRafal Jaworowski }
19202b553caSRafal Jaworowski 
19302b553caSRafal Jaworowski static inline void *
19402b553caSRafal Jaworowski sec_get_pointer_data(struct sec_desc *desc, u_int n)
19502b553caSRafal Jaworowski {
19602b553caSRafal Jaworowski 
19702b553caSRafal Jaworowski 	return (desc->sd_ptr_dmem[n].dma_vaddr);
19802b553caSRafal Jaworowski }
19902b553caSRafal Jaworowski 
20002b553caSRafal Jaworowski static int
20102b553caSRafal Jaworowski sec_probe(device_t dev)
20202b553caSRafal Jaworowski {
20302b553caSRafal Jaworowski 	struct sec_softc *sc;
20402b553caSRafal Jaworowski 	device_t parent;
20502b553caSRafal Jaworowski 	uintptr_t devtype;
20602b553caSRafal Jaworowski 	uint64_t id;
20702b553caSRafal Jaworowski 	int error;
20802b553caSRafal Jaworowski 
20902b553caSRafal Jaworowski 	parent = device_get_parent(dev);
21002b553caSRafal Jaworowski 	error = BUS_READ_IVAR(parent, dev, OCPBUS_IVAR_DEVTYPE, &devtype);
21102b553caSRafal Jaworowski 	if (error)
21202b553caSRafal Jaworowski 		return (error);
21302b553caSRafal Jaworowski 
21402b553caSRafal Jaworowski 	if (devtype != OCPBUS_DEVTYPE_SEC)
21502b553caSRafal Jaworowski 		return (ENXIO);
21602b553caSRafal Jaworowski 
21702b553caSRafal Jaworowski 	sc = device_get_softc(dev);
21802b553caSRafal Jaworowski 
21902b553caSRafal Jaworowski 	sc->sc_rrid = 0;
22002b553caSRafal Jaworowski 	sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid,
22102b553caSRafal Jaworowski 	    0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE);
22202b553caSRafal Jaworowski 
22302b553caSRafal Jaworowski 	if (sc->sc_rres == NULL)
22402b553caSRafal Jaworowski 		return (ENXIO);
22502b553caSRafal Jaworowski 
22602b553caSRafal Jaworowski 	sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
22702b553caSRafal Jaworowski 	sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
22802b553caSRafal Jaworowski 
22902b553caSRafal Jaworowski 	id = SEC_READ(sc, SEC_ID);
23002b553caSRafal Jaworowski 
23102b553caSRafal Jaworowski 	bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
23202b553caSRafal Jaworowski 
23302b553caSRafal Jaworowski 	switch (id) {
23402b553caSRafal Jaworowski 	case SEC_20_ID:
23502b553caSRafal Jaworowski 		device_set_desc(dev, "Freescale Security Engine 2.0");
23602b553caSRafal Jaworowski 		sc->sc_version = 2;
23702b553caSRafal Jaworowski 		break;
23802b553caSRafal Jaworowski 	case SEC_30_ID:
23902b553caSRafal Jaworowski 		device_set_desc(dev, "Freescale Security Engine 3.0");
24002b553caSRafal Jaworowski 		sc->sc_version = 3;
24102b553caSRafal Jaworowski 		break;
24202b553caSRafal Jaworowski 	default:
24302b553caSRafal Jaworowski 		device_printf(dev, "unknown SEC ID 0x%016llx!\n", id);
24402b553caSRafal Jaworowski 		return (ENXIO);
24502b553caSRafal Jaworowski 	}
24602b553caSRafal Jaworowski 
24702b553caSRafal Jaworowski 	return (0);
24802b553caSRafal Jaworowski }
24902b553caSRafal Jaworowski 
25002b553caSRafal Jaworowski static int
25102b553caSRafal Jaworowski sec_attach(device_t dev)
25202b553caSRafal Jaworowski {
25302b553caSRafal Jaworowski 	struct sec_softc *sc;
25402b553caSRafal Jaworowski 	struct sec_hw_lt *lt;
25502b553caSRafal Jaworowski 	int error = 0;
25602b553caSRafal Jaworowski 	int i;
25702b553caSRafal Jaworowski 
25802b553caSRafal Jaworowski 	sc = device_get_softc(dev);
25902b553caSRafal Jaworowski 	sc->sc_dev = dev;
26002b553caSRafal Jaworowski 	sc->sc_blocked = 0;
26102b553caSRafal Jaworowski 	sc->sc_shutdown = 0;
26202b553caSRafal Jaworowski 
26302b553caSRafal Jaworowski 	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
26402b553caSRafal Jaworowski 	if (sc->sc_cid < 0) {
26502b553caSRafal Jaworowski 		device_printf(dev, "could not get crypto driver ID!\n");
26602b553caSRafal Jaworowski 		return (ENXIO);
26702b553caSRafal Jaworowski 	}
26802b553caSRafal Jaworowski 
26902b553caSRafal Jaworowski 	/* Init locks */
27002b553caSRafal Jaworowski 	mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev),
27102b553caSRafal Jaworowski 	    "SEC Controller lock", MTX_DEF);
27202b553caSRafal Jaworowski 	mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev),
27302b553caSRafal Jaworowski 	    "SEC Descriptors lock", MTX_DEF);
27402b553caSRafal Jaworowski 	mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
27502b553caSRafal Jaworowski 	    "SEC Sessions lock", MTX_DEF);
27602b553caSRafal Jaworowski 
27702b553caSRafal Jaworowski 	/* Allocate I/O memory for SEC registers */
27802b553caSRafal Jaworowski 	sc->sc_rrid = 0;
27902b553caSRafal Jaworowski 	sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid,
28002b553caSRafal Jaworowski 	    0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE);
28102b553caSRafal Jaworowski 
28202b553caSRafal Jaworowski 	if (sc->sc_rres == NULL) {
28302b553caSRafal Jaworowski 		device_printf(dev, "could not allocate I/O memory!\n");
28402b553caSRafal Jaworowski 		goto fail1;
28502b553caSRafal Jaworowski 	}
28602b553caSRafal Jaworowski 
28702b553caSRafal Jaworowski 	sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
28802b553caSRafal Jaworowski 	sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
28902b553caSRafal Jaworowski 
29002b553caSRafal Jaworowski 	/* Setup interrupts */
29102b553caSRafal Jaworowski 	sc->sc_pri_irid = 0;
29202b553caSRafal Jaworowski 	error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand,
29302b553caSRafal Jaworowski 	    &sc->sc_pri_irid, sec_primary_intr, "primary");
29402b553caSRafal Jaworowski 
29502b553caSRafal Jaworowski 	if (error)
29602b553caSRafal Jaworowski 		goto fail2;
29702b553caSRafal Jaworowski 
29802b553caSRafal Jaworowski 	sc->sc_sec_irid = 1;
29902b553caSRafal Jaworowski 	error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand,
30002b553caSRafal Jaworowski 	    &sc->sc_sec_irid, sec_secondary_intr, "secondary");
30102b553caSRafal Jaworowski 
30202b553caSRafal Jaworowski 	if (error)
30302b553caSRafal Jaworowski 		goto fail3;
30402b553caSRafal Jaworowski 
30502b553caSRafal Jaworowski 	/* Alloc DMA memory for descriptors and link tables */
30602b553caSRafal Jaworowski 	error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem),
30702b553caSRafal Jaworowski 	    SEC_DESCRIPTORS * sizeof(struct sec_hw_desc));
30802b553caSRafal Jaworowski 
30902b553caSRafal Jaworowski 	if (error)
31002b553caSRafal Jaworowski 		goto fail4;
31102b553caSRafal Jaworowski 
31202b553caSRafal Jaworowski 	error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem),
31302b553caSRafal Jaworowski 	    (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt));
31402b553caSRafal Jaworowski 
31502b553caSRafal Jaworowski 	if (error)
31602b553caSRafal Jaworowski 		goto fail5;
31702b553caSRafal Jaworowski 
31802b553caSRafal Jaworowski 	/* Fill in descriptors and link tables */
31902b553caSRafal Jaworowski 	for (i = 0; i < SEC_DESCRIPTORS; i++) {
32002b553caSRafal Jaworowski 		sc->sc_desc[i].sd_desc =
32102b553caSRafal Jaworowski 		    (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i;
32202b553caSRafal Jaworowski 		sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr +
32302b553caSRafal Jaworowski 		    (i * sizeof(struct sec_hw_desc));
32402b553caSRafal Jaworowski 	}
32502b553caSRafal Jaworowski 
32602b553caSRafal Jaworowski 	for (i = 0; i < SEC_LT_ENTRIES + 1; i++) {
32702b553caSRafal Jaworowski 		sc->sc_lt[i].sl_lt =
32802b553caSRafal Jaworowski 		    (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i;
32902b553caSRafal Jaworowski 		sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr +
33002b553caSRafal Jaworowski 		    (i * sizeof(struct sec_hw_lt));
33102b553caSRafal Jaworowski 	}
33202b553caSRafal Jaworowski 
33302b553caSRafal Jaworowski 	/* Last entry in link table is used to create a circle */
33402b553caSRafal Jaworowski 	lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt;
33502b553caSRafal Jaworowski 	lt->shl_length = 0;
33602b553caSRafal Jaworowski 	lt->shl_r = 0;
33702b553caSRafal Jaworowski 	lt->shl_n = 1;
33802b553caSRafal Jaworowski 	lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr;
33902b553caSRafal Jaworowski 
34002b553caSRafal Jaworowski 	/* Init descriptor and link table queues pointers */
34102b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS);
34202b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS);
34302b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS);
34402b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS);
34502b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS);
34602b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS);
34702b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES);
34802b553caSRafal Jaworowski 	SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES);
34902b553caSRafal Jaworowski 
35002b553caSRafal Jaworowski 	/* Create masks for fast checks */
35102b553caSRafal Jaworowski 	sc->sc_int_error_mask = 0;
35202b553caSRafal Jaworowski 	for (i = 0; i < SEC_CHANNELS; i++)
35302b553caSRafal Jaworowski 		sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i));
35402b553caSRafal Jaworowski 
35502b553caSRafal Jaworowski 	switch (sc->sc_version) {
35602b553caSRafal Jaworowski 	case 2:
35702b553caSRafal Jaworowski 		sc->sc_channel_idle_mask =
35802b553caSRafal Jaworowski 		    (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) |
35902b553caSRafal Jaworowski 		    (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) |
36002b553caSRafal Jaworowski 		    (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) |
36102b553caSRafal Jaworowski 		    (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S);
36202b553caSRafal Jaworowski 		break;
36302b553caSRafal Jaworowski 	case 3:
36402b553caSRafal Jaworowski 		sc->sc_channel_idle_mask =
36502b553caSRafal Jaworowski 		    (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) |
36602b553caSRafal Jaworowski 		    (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) |
36702b553caSRafal Jaworowski 		    (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) |
36802b553caSRafal Jaworowski 		    (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S);
36902b553caSRafal Jaworowski 		break;
37002b553caSRafal Jaworowski 	}
37102b553caSRafal Jaworowski 
37202b553caSRafal Jaworowski 	/* Init hardware */
37302b553caSRafal Jaworowski 	error = sec_init(sc);
37402b553caSRafal Jaworowski 
37502b553caSRafal Jaworowski 	if (error)
37602b553caSRafal Jaworowski 		goto fail6;
37702b553caSRafal Jaworowski 
37802b553caSRafal Jaworowski 	/* Register in OCF (AESU) */
37902b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
38002b553caSRafal Jaworowski 
38102b553caSRafal Jaworowski 	/* Register in OCF (DEU) */
38202b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
38302b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
38402b553caSRafal Jaworowski 
38502b553caSRafal Jaworowski 	/* Register in OCF (MDEU) */
38602b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
38702b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
38802b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
38902b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
39002b553caSRafal Jaworowski 	crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
39102b553caSRafal Jaworowski 	if (sc->sc_version >= 3) {
39202b553caSRafal Jaworowski 		crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
39302b553caSRafal Jaworowski 		crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
39402b553caSRafal Jaworowski 	}
39502b553caSRafal Jaworowski 
39602b553caSRafal Jaworowski 	return (0);
39702b553caSRafal Jaworowski 
39802b553caSRafal Jaworowski fail6:
39902b553caSRafal Jaworowski 	sec_free_dma_mem(&(sc->sc_lt_dmem));
40002b553caSRafal Jaworowski fail5:
40102b553caSRafal Jaworowski 	sec_free_dma_mem(&(sc->sc_desc_dmem));
40202b553caSRafal Jaworowski fail4:
40302b553caSRafal Jaworowski 	sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
40402b553caSRafal Jaworowski 	    sc->sc_sec_irid, "secondary");
40502b553caSRafal Jaworowski fail3:
40602b553caSRafal Jaworowski 	sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
40702b553caSRafal Jaworowski 	    sc->sc_pri_irid, "primary");
40802b553caSRafal Jaworowski fail2:
40902b553caSRafal Jaworowski 	bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
41002b553caSRafal Jaworowski fail1:
41102b553caSRafal Jaworowski 	mtx_destroy(&sc->sc_controller_lock);
41202b553caSRafal Jaworowski 	mtx_destroy(&sc->sc_descriptors_lock);
41302b553caSRafal Jaworowski 	mtx_destroy(&sc->sc_sessions_lock);
41402b553caSRafal Jaworowski 
41502b553caSRafal Jaworowski 	return (ENXIO);
41602b553caSRafal Jaworowski }
41702b553caSRafal Jaworowski 
41802b553caSRafal Jaworowski static int
41902b553caSRafal Jaworowski sec_detach(device_t dev)
42002b553caSRafal Jaworowski {
42102b553caSRafal Jaworowski 	struct sec_softc *sc = device_get_softc(dev);
42202b553caSRafal Jaworowski 	int i, error, timeout = SEC_TIMEOUT;
42302b553caSRafal Jaworowski 
42402b553caSRafal Jaworowski 	/* Prepare driver to shutdown */
42502b553caSRafal Jaworowski 	SEC_LOCK(sc, descriptors);
42602b553caSRafal Jaworowski 	sc->sc_shutdown = 1;
42702b553caSRafal Jaworowski 	SEC_UNLOCK(sc, descriptors);
42802b553caSRafal Jaworowski 
42902b553caSRafal Jaworowski 	/* Wait until all queued processing finishes */
43002b553caSRafal Jaworowski 	while (1) {
43102b553caSRafal Jaworowski 		SEC_LOCK(sc, descriptors);
43202b553caSRafal Jaworowski 		i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc);
43302b553caSRafal Jaworowski 		SEC_UNLOCK(sc, descriptors);
43402b553caSRafal Jaworowski 
43502b553caSRafal Jaworowski 		if (i == 0)
43602b553caSRafal Jaworowski 			break;
43702b553caSRafal Jaworowski 
43802b553caSRafal Jaworowski 		if (timeout < 0) {
43902b553caSRafal Jaworowski 			device_printf(dev, "queue flush timeout!\n");
44002b553caSRafal Jaworowski 
44102b553caSRafal Jaworowski 			/* DMA can be still active - stop it */
44202b553caSRafal Jaworowski 			for (i = 0; i < SEC_CHANNELS; i++)
44302b553caSRafal Jaworowski 				sec_channel_reset(sc, i, 1);
44402b553caSRafal Jaworowski 
44502b553caSRafal Jaworowski 			break;
44602b553caSRafal Jaworowski 		}
44702b553caSRafal Jaworowski 
44802b553caSRafal Jaworowski 		timeout -= 1000;
44902b553caSRafal Jaworowski 		DELAY(1000);
45002b553caSRafal Jaworowski 	}
45102b553caSRafal Jaworowski 
45202b553caSRafal Jaworowski 	/* Disable interrupts */
45302b553caSRafal Jaworowski 	SEC_WRITE(sc, SEC_IER, 0);
45402b553caSRafal Jaworowski 
45502b553caSRafal Jaworowski 	/* Unregister from OCF */
45602b553caSRafal Jaworowski 	crypto_unregister_all(sc->sc_cid);
45702b553caSRafal Jaworowski 
45802b553caSRafal Jaworowski 	/* Free DMA memory */
45902b553caSRafal Jaworowski 	for (i = 0; i < SEC_DESCRIPTORS; i++)
46002b553caSRafal Jaworowski 		SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i]));
46102b553caSRafal Jaworowski 
46202b553caSRafal Jaworowski 	sec_free_dma_mem(&(sc->sc_lt_dmem));
46302b553caSRafal Jaworowski 	sec_free_dma_mem(&(sc->sc_desc_dmem));
46402b553caSRafal Jaworowski 
46502b553caSRafal Jaworowski 	/* Release interrupts */
46602b553caSRafal Jaworowski 	sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
46702b553caSRafal Jaworowski 	    sc->sc_pri_irid, "primary");
46802b553caSRafal Jaworowski 	sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
46902b553caSRafal Jaworowski 	    sc->sc_sec_irid, "secondary");
47002b553caSRafal Jaworowski 
47102b553caSRafal Jaworowski 	/* Release memory */
47202b553caSRafal Jaworowski 	if (sc->sc_rres) {
47302b553caSRafal Jaworowski 		error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
47402b553caSRafal Jaworowski 		    sc->sc_rres);
47502b553caSRafal Jaworowski 		if (error)
47602b553caSRafal Jaworowski 			device_printf(dev, "bus_release_resource() failed for"
47702b553caSRafal Jaworowski 			    " I/O memory, error %d\n", error);
47802b553caSRafal Jaworowski 
47902b553caSRafal Jaworowski 		sc->sc_rres = NULL;
48002b553caSRafal Jaworowski 	}
48102b553caSRafal Jaworowski 
48202b553caSRafal Jaworowski 	mtx_destroy(&sc->sc_controller_lock);
48302b553caSRafal Jaworowski 	mtx_destroy(&sc->sc_descriptors_lock);
48402b553caSRafal Jaworowski 	mtx_destroy(&sc->sc_sessions_lock);
48502b553caSRafal Jaworowski 
48602b553caSRafal Jaworowski 	return (0);
48702b553caSRafal Jaworowski }
48802b553caSRafal Jaworowski 
48902b553caSRafal Jaworowski static int
49002b553caSRafal Jaworowski sec_suspend(device_t dev)
49102b553caSRafal Jaworowski {
49202b553caSRafal Jaworowski 
49302b553caSRafal Jaworowski 	return (0);
49402b553caSRafal Jaworowski }
49502b553caSRafal Jaworowski 
49602b553caSRafal Jaworowski static int
49702b553caSRafal Jaworowski sec_resume(device_t dev)
49802b553caSRafal Jaworowski {
49902b553caSRafal Jaworowski 
50002b553caSRafal Jaworowski 	return (0);
50102b553caSRafal Jaworowski }
50202b553caSRafal Jaworowski 
503661ee6eeSRafal Jaworowski static int
50402b553caSRafal Jaworowski sec_shutdown(device_t dev)
50502b553caSRafal Jaworowski {
506661ee6eeSRafal Jaworowski 
507661ee6eeSRafal Jaworowski 	return (0);
50802b553caSRafal Jaworowski }
50902b553caSRafal Jaworowski 
51002b553caSRafal Jaworowski static int
51102b553caSRafal Jaworowski sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand,
51202b553caSRafal Jaworowski     int *irid, driver_intr_t handler, const char *iname)
51302b553caSRafal Jaworowski {
51402b553caSRafal Jaworowski 	int error;
51502b553caSRafal Jaworowski 
51602b553caSRafal Jaworowski 	(*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid,
51702b553caSRafal Jaworowski 	    RF_ACTIVE);
51802b553caSRafal Jaworowski 
51902b553caSRafal Jaworowski 	if ((*ires) == NULL) {
52002b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname);
52102b553caSRafal Jaworowski 		return (ENXIO);
52202b553caSRafal Jaworowski 	}
52302b553caSRafal Jaworowski 
52402b553caSRafal Jaworowski 	error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET,
52502b553caSRafal Jaworowski 	    NULL, handler, sc, ihand);
52602b553caSRafal Jaworowski 
52702b553caSRafal Jaworowski 	if (error) {
52802b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname);
52902b553caSRafal Jaworowski 		if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires))
53002b553caSRafal Jaworowski 			device_printf(sc->sc_dev, "could not release %s IRQ\n",
53102b553caSRafal Jaworowski 			    iname);
53202b553caSRafal Jaworowski 
53302b553caSRafal Jaworowski 		(*ires) = NULL;
53402b553caSRafal Jaworowski 		return (error);
53502b553caSRafal Jaworowski 	}
53602b553caSRafal Jaworowski 
53702b553caSRafal Jaworowski 	return (0);
53802b553caSRafal Jaworowski }
53902b553caSRafal Jaworowski 
54002b553caSRafal Jaworowski static void
54102b553caSRafal Jaworowski sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand,
54202b553caSRafal Jaworowski     int irid, const char *iname)
54302b553caSRafal Jaworowski {
54402b553caSRafal Jaworowski 	int error;
54502b553caSRafal Jaworowski 
54602b553caSRafal Jaworowski 	if (ires == NULL)
54702b553caSRafal Jaworowski 		return;
54802b553caSRafal Jaworowski 
54902b553caSRafal Jaworowski 	error = bus_teardown_intr(sc->sc_dev, ires, ihand);
55002b553caSRafal Jaworowski 	if (error)
55102b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s"
55202b553caSRafal Jaworowski 		    " IRQ, error %d\n", iname, error);
55302b553caSRafal Jaworowski 
55402b553caSRafal Jaworowski 	error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires);
55502b553caSRafal Jaworowski 	if (error)
55602b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "bus_release_resource() failed for %s"
55702b553caSRafal Jaworowski 		    " IRQ, error %d\n", iname, error);
55802b553caSRafal Jaworowski }
55902b553caSRafal Jaworowski 
56002b553caSRafal Jaworowski static void
56102b553caSRafal Jaworowski sec_primary_intr(void *arg)
56202b553caSRafal Jaworowski {
56302b553caSRafal Jaworowski 	struct sec_softc *sc = arg;
56402b553caSRafal Jaworowski 	struct sec_desc *desc;
56502b553caSRafal Jaworowski 	uint64_t isr;
56602b553caSRafal Jaworowski 	int i, wakeup = 0;
56702b553caSRafal Jaworowski 
56802b553caSRafal Jaworowski 	SEC_LOCK(sc, controller);
56902b553caSRafal Jaworowski 
57002b553caSRafal Jaworowski 	/* Check for errors */
57102b553caSRafal Jaworowski 	isr = SEC_READ(sc, SEC_ISR);
57202b553caSRafal Jaworowski 	if (isr & sc->sc_int_error_mask) {
57302b553caSRafal Jaworowski 		/* Check each channel for error */
57402b553caSRafal Jaworowski 		for (i = 0; i < SEC_CHANNELS; i++) {
57502b553caSRafal Jaworowski 			if ((isr & SEC_INT_CH_ERR(i)) == 0)
57602b553caSRafal Jaworowski 				continue;
57702b553caSRafal Jaworowski 
57802b553caSRafal Jaworowski 			device_printf(sc->sc_dev,
57902b553caSRafal Jaworowski 			    "I/O error on channel %i!\n", i);
58002b553caSRafal Jaworowski 
58102b553caSRafal Jaworowski 			/* Find and mark problematic descriptor */
58202b553caSRafal Jaworowski 			desc = sec_find_desc(sc, SEC_READ(sc,
58302b553caSRafal Jaworowski 			    SEC_CHAN_CDPR(i)));
58402b553caSRafal Jaworowski 
58502b553caSRafal Jaworowski 			if (desc != NULL)
58602b553caSRafal Jaworowski 				desc->sd_error = EIO;
58702b553caSRafal Jaworowski 
58802b553caSRafal Jaworowski 			/* Do partial channel reset */
58902b553caSRafal Jaworowski 			sec_channel_reset(sc, i, 0);
59002b553caSRafal Jaworowski 		}
59102b553caSRafal Jaworowski 	}
59202b553caSRafal Jaworowski 
59302b553caSRafal Jaworowski 	/* ACK interrupt */
59402b553caSRafal Jaworowski 	SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL);
59502b553caSRafal Jaworowski 
59602b553caSRafal Jaworowski 	SEC_UNLOCK(sc, controller);
59702b553caSRafal Jaworowski 	SEC_LOCK(sc, descriptors);
59802b553caSRafal Jaworowski 
59902b553caSRafal Jaworowski 	/* Handle processed descriptors */
60002b553caSRafal Jaworowski 	SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
60102b553caSRafal Jaworowski 
60202b553caSRafal Jaworowski 	while (SEC_QUEUED_DESC_CNT(sc) > 0) {
60302b553caSRafal Jaworowski 		desc = SEC_GET_QUEUED_DESC(sc);
60402b553caSRafal Jaworowski 
60502b553caSRafal Jaworowski 		if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) {
60602b553caSRafal Jaworowski 			SEC_PUT_BACK_QUEUED_DESC(sc);
60702b553caSRafal Jaworowski 			break;
60802b553caSRafal Jaworowski 		}
60902b553caSRafal Jaworowski 
61002b553caSRafal Jaworowski 		SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
61102b553caSRafal Jaworowski 		    BUS_DMASYNC_PREWRITE);
61202b553caSRafal Jaworowski 
61302b553caSRafal Jaworowski 		desc->sd_crp->crp_etype = desc->sd_error;
61402b553caSRafal Jaworowski 		crypto_done(desc->sd_crp);
61502b553caSRafal Jaworowski 
61602b553caSRafal Jaworowski 		SEC_DESC_FREE_POINTERS(desc);
61702b553caSRafal Jaworowski 		SEC_DESC_FREE_LT(sc, desc);
61802b553caSRafal Jaworowski 		SEC_DESC_QUEUED2FREE(sc);
61902b553caSRafal Jaworowski 	}
62002b553caSRafal Jaworowski 
62102b553caSRafal Jaworowski 	SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
62202b553caSRafal Jaworowski 
62302b553caSRafal Jaworowski 	if (!sc->sc_shutdown) {
62402b553caSRafal Jaworowski 		wakeup = sc->sc_blocked;
62502b553caSRafal Jaworowski 		sc->sc_blocked = 0;
62602b553caSRafal Jaworowski 	}
62702b553caSRafal Jaworowski 
62802b553caSRafal Jaworowski 	SEC_UNLOCK(sc, descriptors);
62902b553caSRafal Jaworowski 
63002b553caSRafal Jaworowski 	/* Enqueue ready descriptors in hardware */
63102b553caSRafal Jaworowski 	sec_enqueue(sc);
63202b553caSRafal Jaworowski 
63302b553caSRafal Jaworowski 	if (wakeup)
63402b553caSRafal Jaworowski 		crypto_unblock(sc->sc_cid, wakeup);
63502b553caSRafal Jaworowski }
63602b553caSRafal Jaworowski 
63702b553caSRafal Jaworowski static void
63802b553caSRafal Jaworowski sec_secondary_intr(void *arg)
63902b553caSRafal Jaworowski {
64002b553caSRafal Jaworowski 	struct sec_softc *sc = arg;
64102b553caSRafal Jaworowski 
64202b553caSRafal Jaworowski 	device_printf(sc->sc_dev, "spurious secondary interrupt!\n");
64302b553caSRafal Jaworowski 	sec_primary_intr(arg);
64402b553caSRafal Jaworowski }
64502b553caSRafal Jaworowski 
64602b553caSRafal Jaworowski static int
64702b553caSRafal Jaworowski sec_controller_reset(struct sec_softc *sc)
64802b553caSRafal Jaworowski {
64902b553caSRafal Jaworowski 	int timeout = SEC_TIMEOUT;
65002b553caSRafal Jaworowski 
65102b553caSRafal Jaworowski 	/* Reset Controller */
65202b553caSRafal Jaworowski 	SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR);
65302b553caSRafal Jaworowski 
65402b553caSRafal Jaworowski 	while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) {
65502b553caSRafal Jaworowski 		DELAY(1000);
65602b553caSRafal Jaworowski 		timeout -= 1000;
65702b553caSRafal Jaworowski 
65802b553caSRafal Jaworowski 		if (timeout < 0) {
65902b553caSRafal Jaworowski 			device_printf(sc->sc_dev, "timeout while waiting for "
66002b553caSRafal Jaworowski 			    "device reset!\n");
66102b553caSRafal Jaworowski 			return (ETIMEDOUT);
66202b553caSRafal Jaworowski 		}
66302b553caSRafal Jaworowski 	}
66402b553caSRafal Jaworowski 
66502b553caSRafal Jaworowski 	return (0);
66602b553caSRafal Jaworowski }
66702b553caSRafal Jaworowski 
66802b553caSRafal Jaworowski static int
66902b553caSRafal Jaworowski sec_channel_reset(struct sec_softc *sc, int channel, int full)
67002b553caSRafal Jaworowski {
67102b553caSRafal Jaworowski 	int timeout = SEC_TIMEOUT;
67202b553caSRafal Jaworowski 	uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON;
67302b553caSRafal Jaworowski 	uint64_t reg;
67402b553caSRafal Jaworowski 
67502b553caSRafal Jaworowski 	/* Reset Channel */
67602b553caSRafal Jaworowski 	reg = SEC_READ(sc, SEC_CHAN_CCR(channel));
67702b553caSRafal Jaworowski 	SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit);
67802b553caSRafal Jaworowski 
67902b553caSRafal Jaworowski 	while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) {
68002b553caSRafal Jaworowski 		DELAY(1000);
68102b553caSRafal Jaworowski 		timeout -= 1000;
68202b553caSRafal Jaworowski 
68302b553caSRafal Jaworowski 		if (timeout < 0) {
68402b553caSRafal Jaworowski 			device_printf(sc->sc_dev, "timeout while waiting for "
68502b553caSRafal Jaworowski 			    "channel reset!\n");
68602b553caSRafal Jaworowski 			return (ETIMEDOUT);
68702b553caSRafal Jaworowski 		}
68802b553caSRafal Jaworowski 	}
68902b553caSRafal Jaworowski 
69002b553caSRafal Jaworowski 	if (full) {
69102b553caSRafal Jaworowski 		reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS;
69202b553caSRafal Jaworowski 
69302b553caSRafal Jaworowski 		switch(sc->sc_version) {
69402b553caSRafal Jaworowski 		case 2:
69502b553caSRafal Jaworowski 			reg |= SEC_CHAN_CCR_CDWE;
69602b553caSRafal Jaworowski 			break;
69702b553caSRafal Jaworowski 		case 3:
69802b553caSRafal Jaworowski 			reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN;
69902b553caSRafal Jaworowski 			break;
70002b553caSRafal Jaworowski 		}
70102b553caSRafal Jaworowski 
70202b553caSRafal Jaworowski 		SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg);
70302b553caSRafal Jaworowski 	}
70402b553caSRafal Jaworowski 
70502b553caSRafal Jaworowski 	return (0);
70602b553caSRafal Jaworowski }
70702b553caSRafal Jaworowski 
70802b553caSRafal Jaworowski static int
70902b553caSRafal Jaworowski sec_init(struct sec_softc *sc)
71002b553caSRafal Jaworowski {
71102b553caSRafal Jaworowski 	uint64_t reg;
71202b553caSRafal Jaworowski 	int error, i;
71302b553caSRafal Jaworowski 
71402b553caSRafal Jaworowski 	/* Reset controller twice to clear all pending interrupts */
71502b553caSRafal Jaworowski 	error = sec_controller_reset(sc);
71602b553caSRafal Jaworowski 	if (error)
71702b553caSRafal Jaworowski 		return (error);
71802b553caSRafal Jaworowski 
71902b553caSRafal Jaworowski 	error = sec_controller_reset(sc);
72002b553caSRafal Jaworowski 	if (error)
72102b553caSRafal Jaworowski 		return (error);
72202b553caSRafal Jaworowski 
72302b553caSRafal Jaworowski 	/* Reset channels */
72402b553caSRafal Jaworowski 	for (i = 0; i < SEC_CHANNELS; i++) {
72502b553caSRafal Jaworowski 		error = sec_channel_reset(sc, i, 1);
72602b553caSRafal Jaworowski 		if (error)
72702b553caSRafal Jaworowski 			return (error);
72802b553caSRafal Jaworowski 	}
72902b553caSRafal Jaworowski 
73002b553caSRafal Jaworowski 	/* Enable Interrupts */
73102b553caSRafal Jaworowski 	reg = SEC_INT_ITO;
73202b553caSRafal Jaworowski 	for (i = 0; i < SEC_CHANNELS; i++)
73302b553caSRafal Jaworowski 		reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i);
73402b553caSRafal Jaworowski 
73502b553caSRafal Jaworowski 	SEC_WRITE(sc, SEC_IER, reg);
73602b553caSRafal Jaworowski 
73702b553caSRafal Jaworowski 	return (error);
73802b553caSRafal Jaworowski }
73902b553caSRafal Jaworowski 
74002b553caSRafal Jaworowski static void
74102b553caSRafal Jaworowski sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
74202b553caSRafal Jaworowski {
74302b553caSRafal Jaworowski 	struct sec_dma_mem *dma_mem = arg;
74402b553caSRafal Jaworowski 
74502b553caSRafal Jaworowski 	if (error)
74602b553caSRafal Jaworowski 		return;
74702b553caSRafal Jaworowski 
74802b553caSRafal Jaworowski 	KASSERT(nseg == 1, ("Wrong number of segments, should be 1"));
74902b553caSRafal Jaworowski 	dma_mem->dma_paddr = segs->ds_addr;
75002b553caSRafal Jaworowski }
75102b553caSRafal Jaworowski 
75202b553caSRafal Jaworowski static void
75302b553caSRafal Jaworowski sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg,
75402b553caSRafal Jaworowski     int error)
75502b553caSRafal Jaworowski {
75602b553caSRafal Jaworowski 	struct sec_desc_map_info *sdmi = arg;
75702b553caSRafal Jaworowski 	struct sec_softc *sc = sdmi->sdmi_sc;
75802b553caSRafal Jaworowski 	struct sec_lt *lt = NULL;
75902b553caSRafal Jaworowski 	bus_addr_t addr;
76002b553caSRafal Jaworowski 	bus_size_t size;
76102b553caSRafal Jaworowski 	int i;
76202b553caSRafal Jaworowski 
76302b553caSRafal Jaworowski 	SEC_LOCK_ASSERT(sc, descriptors);
76402b553caSRafal Jaworowski 
76502b553caSRafal Jaworowski 	if (error)
76602b553caSRafal Jaworowski 		return;
76702b553caSRafal Jaworowski 
76802b553caSRafal Jaworowski 	for (i = 0; i < nseg; i++) {
76902b553caSRafal Jaworowski 		addr = segs[i].ds_addr;
77002b553caSRafal Jaworowski 		size = segs[i].ds_len;
77102b553caSRafal Jaworowski 
77202b553caSRafal Jaworowski 		/* Skip requested offset */
77302b553caSRafal Jaworowski 		if (sdmi->sdmi_offset >= size) {
77402b553caSRafal Jaworowski 			sdmi->sdmi_offset -= size;
77502b553caSRafal Jaworowski 			continue;
77602b553caSRafal Jaworowski 		}
77702b553caSRafal Jaworowski 
77802b553caSRafal Jaworowski 		addr += sdmi->sdmi_offset;
77902b553caSRafal Jaworowski 		size -= sdmi->sdmi_offset;
78002b553caSRafal Jaworowski 		sdmi->sdmi_offset = 0;
78102b553caSRafal Jaworowski 
78202b553caSRafal Jaworowski 		/* Do not link more than requested */
78302b553caSRafal Jaworowski 		if (sdmi->sdmi_size < size)
78402b553caSRafal Jaworowski 			size = sdmi->sdmi_size;
78502b553caSRafal Jaworowski 
78602b553caSRafal Jaworowski 		lt = SEC_ALLOC_LT_ENTRY(sc);
78702b553caSRafal Jaworowski 		lt->sl_lt->shl_length = size;
78802b553caSRafal Jaworowski 		lt->sl_lt->shl_r = 0;
78902b553caSRafal Jaworowski 		lt->sl_lt->shl_n = 0;
79002b553caSRafal Jaworowski 		lt->sl_lt->shl_ptr = addr;
79102b553caSRafal Jaworowski 
79202b553caSRafal Jaworowski 		if (sdmi->sdmi_lt_first == NULL)
79302b553caSRafal Jaworowski 			sdmi->sdmi_lt_first = lt;
79402b553caSRafal Jaworowski 
79502b553caSRafal Jaworowski 		sdmi->sdmi_lt_used += 1;
79602b553caSRafal Jaworowski 
79702b553caSRafal Jaworowski 		if ((sdmi->sdmi_size -= size) == 0)
79802b553caSRafal Jaworowski 			break;
79902b553caSRafal Jaworowski 	}
80002b553caSRafal Jaworowski 
80102b553caSRafal Jaworowski 	sdmi->sdmi_lt_last = lt;
80202b553caSRafal Jaworowski }
80302b553caSRafal Jaworowski 
80402b553caSRafal Jaworowski static void
80502b553caSRafal Jaworowski sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
80602b553caSRafal Jaworowski     bus_size_t size, int error)
80702b553caSRafal Jaworowski {
80802b553caSRafal Jaworowski 
80902b553caSRafal Jaworowski 	sec_dma_map_desc_cb(arg, segs, nseg, error);
81002b553caSRafal Jaworowski }
81102b553caSRafal Jaworowski 
81202b553caSRafal Jaworowski static int
81302b553caSRafal Jaworowski sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
81402b553caSRafal Jaworowski     bus_size_t size)
81502b553caSRafal Jaworowski {
81602b553caSRafal Jaworowski 	int error;
81702b553caSRafal Jaworowski 
81802b553caSRafal Jaworowski 	if (dma_mem->dma_vaddr != NULL)
81902b553caSRafal Jaworowski 		return (EBUSY);
82002b553caSRafal Jaworowski 
82102b553caSRafal Jaworowski 	error = bus_dma_tag_create(NULL,	/* parent */
82202b553caSRafal Jaworowski 		SEC_DMA_ALIGNMENT, 0,		/* alignment, boundary */
82302b553caSRafal Jaworowski 		BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
82402b553caSRafal Jaworowski 		BUS_SPACE_MAXADDR,		/* highaddr */
82502b553caSRafal Jaworowski 		NULL, NULL,			/* filtfunc, filtfuncarg */
82602b553caSRafal Jaworowski 		size, 1,			/* maxsize, nsegments */
82702b553caSRafal Jaworowski 		size, 0,			/* maxsegsz, flags */
82802b553caSRafal Jaworowski 		NULL, NULL,			/* lockfunc, lockfuncarg */
82902b553caSRafal Jaworowski 		&(dma_mem->dma_tag));		/* dmat */
83002b553caSRafal Jaworowski 
83102b553caSRafal Jaworowski 	if (error) {
83202b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
83302b553caSRafal Jaworowski 		    " %i!\n", error);
83402b553caSRafal Jaworowski 		goto err1;
83502b553caSRafal Jaworowski 	}
83602b553caSRafal Jaworowski 
83702b553caSRafal Jaworowski 	error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr),
83802b553caSRafal Jaworowski 	    BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map));
83902b553caSRafal Jaworowski 
84002b553caSRafal Jaworowski 	if (error) {
84102b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "failed to allocate DMA safe"
84202b553caSRafal Jaworowski 		    " memory, error %i!\n", error);
84302b553caSRafal Jaworowski 		goto err2;
84402b553caSRafal Jaworowski 	}
84502b553caSRafal Jaworowski 
84602b553caSRafal Jaworowski 	error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
84702b553caSRafal Jaworowski 		    dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem,
84802b553caSRafal Jaworowski 		    BUS_DMA_NOWAIT);
84902b553caSRafal Jaworowski 
85002b553caSRafal Jaworowski 	if (error) {
85102b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "cannot get address of the DMA"
85202b553caSRafal Jaworowski 		    " memory, error %i\n", error);
85302b553caSRafal Jaworowski 		goto err3;
85402b553caSRafal Jaworowski 	}
85502b553caSRafal Jaworowski 
85602b553caSRafal Jaworowski 	dma_mem->dma_is_map = 0;
85702b553caSRafal Jaworowski 	return (0);
85802b553caSRafal Jaworowski 
85902b553caSRafal Jaworowski err3:
86002b553caSRafal Jaworowski 	bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map);
86102b553caSRafal Jaworowski err2:
86202b553caSRafal Jaworowski 	bus_dma_tag_destroy(dma_mem->dma_tag);
86302b553caSRafal Jaworowski err1:
86402b553caSRafal Jaworowski 	dma_mem->dma_vaddr = NULL;
86502b553caSRafal Jaworowski 	return(error);
86602b553caSRafal Jaworowski }
86702b553caSRafal Jaworowski 
86802b553caSRafal Jaworowski static int
86902b553caSRafal Jaworowski sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
87002b553caSRafal Jaworowski     bus_size_t size, int type, struct sec_desc_map_info *sdmi)
87102b553caSRafal Jaworowski {
87202b553caSRafal Jaworowski 	int error;
87302b553caSRafal Jaworowski 
87402b553caSRafal Jaworowski 	if (dma_mem->dma_vaddr != NULL)
87502b553caSRafal Jaworowski 		return (EBUSY);
87602b553caSRafal Jaworowski 
87702b553caSRafal Jaworowski 	switch (type) {
87802b553caSRafal Jaworowski 	case SEC_MEMORY:
87902b553caSRafal Jaworowski 		break;
88002b553caSRafal Jaworowski 	case SEC_UIO:
88102b553caSRafal Jaworowski 		size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
88202b553caSRafal Jaworowski 		break;
88302b553caSRafal Jaworowski 	case SEC_MBUF:
88402b553caSRafal Jaworowski 		size = m_length((struct mbuf*)mem, NULL);
88502b553caSRafal Jaworowski 		break;
88602b553caSRafal Jaworowski 	default:
88702b553caSRafal Jaworowski 		return (EINVAL);
88802b553caSRafal Jaworowski 	}
88902b553caSRafal Jaworowski 
89002b553caSRafal Jaworowski 	error = bus_dma_tag_create(NULL,	/* parent */
89102b553caSRafal Jaworowski 		SEC_DMA_ALIGNMENT, 0,		/* alignment, boundary */
89202b553caSRafal Jaworowski 		BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
89302b553caSRafal Jaworowski 		BUS_SPACE_MAXADDR,		/* highaddr */
89402b553caSRafal Jaworowski 		NULL, NULL,			/* filtfunc, filtfuncarg */
89502b553caSRafal Jaworowski 		size,				/* maxsize */
89602b553caSRafal Jaworowski 		SEC_FREE_LT_CNT(sc),		/* nsegments */
89702b553caSRafal Jaworowski 		SEC_MAX_DMA_BLOCK_SIZE, 0,	/* maxsegsz, flags */
89802b553caSRafal Jaworowski 		NULL, NULL,			/* lockfunc, lockfuncarg */
89902b553caSRafal Jaworowski 		&(dma_mem->dma_tag));		/* dmat */
90002b553caSRafal Jaworowski 
90102b553caSRafal Jaworowski 	if (error) {
90202b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
90302b553caSRafal Jaworowski 		    " %i!\n", error);
90402b553caSRafal Jaworowski 		dma_mem->dma_vaddr = NULL;
90502b553caSRafal Jaworowski 		return (error);
90602b553caSRafal Jaworowski 	}
90702b553caSRafal Jaworowski 
90802b553caSRafal Jaworowski 	error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map));
90902b553caSRafal Jaworowski 
91002b553caSRafal Jaworowski 	if (error) {
91102b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "failed to create DMA map, error %i!"
91202b553caSRafal Jaworowski 		    "\n", error);
91302b553caSRafal Jaworowski 		bus_dma_tag_destroy(dma_mem->dma_tag);
91402b553caSRafal Jaworowski 		return (error);
91502b553caSRafal Jaworowski 	}
91602b553caSRafal Jaworowski 
91702b553caSRafal Jaworowski 	switch (type) {
91802b553caSRafal Jaworowski 	case SEC_MEMORY:
91902b553caSRafal Jaworowski 		error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
92002b553caSRafal Jaworowski 		    mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
92102b553caSRafal Jaworowski 		break;
92202b553caSRafal Jaworowski 	case SEC_UIO:
92302b553caSRafal Jaworowski 		error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map,
92402b553caSRafal Jaworowski 		    mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
92502b553caSRafal Jaworowski 		break;
92602b553caSRafal Jaworowski 	case SEC_MBUF:
92702b553caSRafal Jaworowski 		error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map,
92802b553caSRafal Jaworowski 		    mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
92902b553caSRafal Jaworowski 		break;
93002b553caSRafal Jaworowski 	}
93102b553caSRafal Jaworowski 
93202b553caSRafal Jaworowski 	if (error) {
93302b553caSRafal Jaworowski 		device_printf(sc->sc_dev, "cannot get address of the DMA"
93402b553caSRafal Jaworowski 		    " memory, error %i!\n", error);
93502b553caSRafal Jaworowski 		bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
93602b553caSRafal Jaworowski 		bus_dma_tag_destroy(dma_mem->dma_tag);
93702b553caSRafal Jaworowski 		return (error);
93802b553caSRafal Jaworowski 	}
93902b553caSRafal Jaworowski 
94002b553caSRafal Jaworowski 	dma_mem->dma_is_map = 1;
94102b553caSRafal Jaworowski 	dma_mem->dma_vaddr = mem;
94202b553caSRafal Jaworowski 
94302b553caSRafal Jaworowski 	return (0);
94402b553caSRafal Jaworowski }
94502b553caSRafal Jaworowski 
94602b553caSRafal Jaworowski static void
94702b553caSRafal Jaworowski sec_free_dma_mem(struct sec_dma_mem *dma_mem)
94802b553caSRafal Jaworowski {
94902b553caSRafal Jaworowski 
95002b553caSRafal Jaworowski 	/* Check for double free */
95102b553caSRafal Jaworowski 	if (dma_mem->dma_vaddr == NULL)
95202b553caSRafal Jaworowski 		return;
95302b553caSRafal Jaworowski 
95402b553caSRafal Jaworowski 	bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map);
95502b553caSRafal Jaworowski 
95602b553caSRafal Jaworowski 	if (dma_mem->dma_is_map)
95702b553caSRafal Jaworowski 		bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
95802b553caSRafal Jaworowski 	else
95902b553caSRafal Jaworowski 		bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr,
96002b553caSRafal Jaworowski 		    dma_mem->dma_map);
96102b553caSRafal Jaworowski 
96202b553caSRafal Jaworowski 	bus_dma_tag_destroy(dma_mem->dma_tag);
96302b553caSRafal Jaworowski 	dma_mem->dma_vaddr = NULL;
96402b553caSRafal Jaworowski }
96502b553caSRafal Jaworowski 
96602b553caSRafal Jaworowski static int
96702b553caSRafal Jaworowski sec_eu_channel(struct sec_softc *sc, int eu)
96802b553caSRafal Jaworowski {
96902b553caSRafal Jaworowski 	uint64_t reg;
97002b553caSRafal Jaworowski 	int channel = 0;
97102b553caSRafal Jaworowski 
97202b553caSRafal Jaworowski 	SEC_LOCK_ASSERT(sc, controller);
97302b553caSRafal Jaworowski 
97402b553caSRafal Jaworowski 	reg = SEC_READ(sc, SEC_EUASR);
97502b553caSRafal Jaworowski 
97602b553caSRafal Jaworowski 	switch (eu) {
97702b553caSRafal Jaworowski 	case SEC_EU_AFEU:
97802b553caSRafal Jaworowski 		channel = SEC_EUASR_AFEU(reg);
97902b553caSRafal Jaworowski 		break;
98002b553caSRafal Jaworowski 	case SEC_EU_DEU:
98102b553caSRafal Jaworowski 		channel = SEC_EUASR_DEU(reg);
98202b553caSRafal Jaworowski 		break;
98302b553caSRafal Jaworowski 	case SEC_EU_MDEU_A:
98402b553caSRafal Jaworowski 	case SEC_EU_MDEU_B:
98502b553caSRafal Jaworowski 		channel = SEC_EUASR_MDEU(reg);
98602b553caSRafal Jaworowski 		break;
98702b553caSRafal Jaworowski 	case SEC_EU_RNGU:
98802b553caSRafal Jaworowski 		channel = SEC_EUASR_RNGU(reg);
98902b553caSRafal Jaworowski 		break;
99002b553caSRafal Jaworowski 	case SEC_EU_PKEU:
99102b553caSRafal Jaworowski 		channel = SEC_EUASR_PKEU(reg);
99202b553caSRafal Jaworowski 		break;
99302b553caSRafal Jaworowski 	case SEC_EU_AESU:
99402b553caSRafal Jaworowski 		channel = SEC_EUASR_AESU(reg);
99502b553caSRafal Jaworowski 		break;
99602b553caSRafal Jaworowski 	case SEC_EU_KEU:
99702b553caSRafal Jaworowski 		channel = SEC_EUASR_KEU(reg);
99802b553caSRafal Jaworowski 		break;
99902b553caSRafal Jaworowski 	case SEC_EU_CRCU:
100002b553caSRafal Jaworowski 		channel = SEC_EUASR_CRCU(reg);
100102b553caSRafal Jaworowski 		break;
100202b553caSRafal Jaworowski 	}
100302b553caSRafal Jaworowski 
100402b553caSRafal Jaworowski 	return (channel - 1);
100502b553caSRafal Jaworowski }
100602b553caSRafal Jaworowski 
100702b553caSRafal Jaworowski static int
100802b553caSRafal Jaworowski sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel)
100902b553caSRafal Jaworowski {
101002b553caSRafal Jaworowski 	u_int fflvl = SEC_MAX_FIFO_LEVEL;
101102b553caSRafal Jaworowski 	uint64_t reg;
101202b553caSRafal Jaworowski 	int i;
101302b553caSRafal Jaworowski 
101402b553caSRafal Jaworowski 	SEC_LOCK_ASSERT(sc, controller);
101502b553caSRafal Jaworowski 
101602b553caSRafal Jaworowski 	/* Find free channel if have not got one */
101702b553caSRafal Jaworowski 	if (channel < 0) {
101802b553caSRafal Jaworowski 		for (i = 0; i < SEC_CHANNELS; i++) {
101902b553caSRafal Jaworowski 			reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
102002b553caSRafal Jaworowski 
102102b553caSRafal Jaworowski 			if ((reg & sc->sc_channel_idle_mask) == 0) {
102202b553caSRafal Jaworowski 				channel = i;
102302b553caSRafal Jaworowski 				break;
102402b553caSRafal Jaworowski 			}
102502b553caSRafal Jaworowski 		}
102602b553caSRafal Jaworowski 	}
102702b553caSRafal Jaworowski 
102802b553caSRafal Jaworowski 	/* There is no free channel */
102902b553caSRafal Jaworowski 	if (channel < 0)
103002b553caSRafal Jaworowski 		return (-1);
103102b553caSRafal Jaworowski 
103202b553caSRafal Jaworowski 	/* Check FIFO level on selected channel */
103302b553caSRafal Jaworowski 	reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
103402b553caSRafal Jaworowski 
103502b553caSRafal Jaworowski 	switch(sc->sc_version) {
103602b553caSRafal Jaworowski 	case 2:
103702b553caSRafal Jaworowski 		fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M;
103802b553caSRafal Jaworowski 		break;
103902b553caSRafal Jaworowski 	case 3:
104002b553caSRafal Jaworowski 		fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M;
104102b553caSRafal Jaworowski 		break;
104202b553caSRafal Jaworowski 	}
104302b553caSRafal Jaworowski 
104402b553caSRafal Jaworowski 	if (fflvl >= SEC_MAX_FIFO_LEVEL)
104502b553caSRafal Jaworowski 		return (-1);
104602b553caSRafal Jaworowski 
104702b553caSRafal Jaworowski 	/* Enqueue descriptor in channel */
104802b553caSRafal Jaworowski 	SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr);
104902b553caSRafal Jaworowski 
105002b553caSRafal Jaworowski 	return (channel);
105102b553caSRafal Jaworowski }
105202b553caSRafal Jaworowski 
105302b553caSRafal Jaworowski static void
105402b553caSRafal Jaworowski sec_enqueue(struct sec_softc *sc)
105502b553caSRafal Jaworowski {
105602b553caSRafal Jaworowski 	struct sec_desc *desc;
105702b553caSRafal Jaworowski 	int ch0, ch1;
105802b553caSRafal Jaworowski 
105902b553caSRafal Jaworowski 	SEC_LOCK(sc, descriptors);
106002b553caSRafal Jaworowski 	SEC_LOCK(sc, controller);
106102b553caSRafal Jaworowski 
106202b553caSRafal Jaworowski 	while (SEC_READY_DESC_CNT(sc) > 0) {
106302b553caSRafal Jaworowski 		desc = SEC_GET_READY_DESC(sc);
106402b553caSRafal Jaworowski 
106502b553caSRafal Jaworowski 		ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0);
106602b553caSRafal Jaworowski 		ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1);
106702b553caSRafal Jaworowski 
106802b553caSRafal Jaworowski 		/*
106902b553caSRafal Jaworowski 		 * Both EU are used by the same channel.
107002b553caSRafal Jaworowski 		 * Enqueue descriptor in channel used by busy EUs.
107102b553caSRafal Jaworowski 		 */
107202b553caSRafal Jaworowski 		if (ch0 >= 0 && ch0 == ch1) {
107302b553caSRafal Jaworowski 			if (sec_enqueue_desc(sc, desc, ch0) >= 0) {
107402b553caSRafal Jaworowski 				SEC_DESC_READY2QUEUED(sc);
107502b553caSRafal Jaworowski 				continue;
107602b553caSRafal Jaworowski 			}
107702b553caSRafal Jaworowski 		}
107802b553caSRafal Jaworowski 
107902b553caSRafal Jaworowski 		/*
108002b553caSRafal Jaworowski 		 * Only one EU is free.
108102b553caSRafal Jaworowski 		 * Enqueue descriptor in channel used by busy EU.
108202b553caSRafal Jaworowski 		 */
108302b553caSRafal Jaworowski 		if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) {
108402b553caSRafal Jaworowski 			if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1)
108502b553caSRafal Jaworowski 			    >= 0) {
108602b553caSRafal Jaworowski 				SEC_DESC_READY2QUEUED(sc);
108702b553caSRafal Jaworowski 				continue;
108802b553caSRafal Jaworowski 			}
108902b553caSRafal Jaworowski 		}
109002b553caSRafal Jaworowski 
109102b553caSRafal Jaworowski 		/*
109202b553caSRafal Jaworowski 		 * Both EU are free.
109302b553caSRafal Jaworowski 		 * Enqueue descriptor in first free channel.
109402b553caSRafal Jaworowski 		 */
109502b553caSRafal Jaworowski 		if (ch0 < 0 && ch1 < 0) {
109602b553caSRafal Jaworowski 			if (sec_enqueue_desc(sc, desc, -1) >= 0) {
109702b553caSRafal Jaworowski 				SEC_DESC_READY2QUEUED(sc);
109802b553caSRafal Jaworowski 				continue;
109902b553caSRafal Jaworowski 			}
110002b553caSRafal Jaworowski 		}
110102b553caSRafal Jaworowski 
110202b553caSRafal Jaworowski 		/* Current descriptor can not be queued at the moment */
110302b553caSRafal Jaworowski 		SEC_PUT_BACK_READY_DESC(sc);
110402b553caSRafal Jaworowski 		break;
110502b553caSRafal Jaworowski 	}
110602b553caSRafal Jaworowski 
110702b553caSRafal Jaworowski 	SEC_UNLOCK(sc, controller);
110802b553caSRafal Jaworowski 	SEC_UNLOCK(sc, descriptors);
110902b553caSRafal Jaworowski }
111002b553caSRafal Jaworowski 
111102b553caSRafal Jaworowski static struct sec_desc *
111202b553caSRafal Jaworowski sec_find_desc(struct sec_softc *sc, bus_addr_t paddr)
111302b553caSRafal Jaworowski {
111402b553caSRafal Jaworowski 	struct sec_desc *desc = NULL;
111502b553caSRafal Jaworowski 	int i;
111602b553caSRafal Jaworowski 
111702b553caSRafal Jaworowski 	SEC_LOCK_ASSERT(sc, descriptors);
111802b553caSRafal Jaworowski 
111902b553caSRafal Jaworowski 	for (i = 0; i < SEC_CHANNELS; i++) {
112002b553caSRafal Jaworowski 		if (sc->sc_desc[i].sd_desc_paddr == paddr) {
112102b553caSRafal Jaworowski 			desc = &(sc->sc_desc[i]);
112202b553caSRafal Jaworowski 			break;
112302b553caSRafal Jaworowski 		}
112402b553caSRafal Jaworowski 	}
112502b553caSRafal Jaworowski 
112602b553caSRafal Jaworowski 	return (desc);
112702b553caSRafal Jaworowski }
112802b553caSRafal Jaworowski 
112902b553caSRafal Jaworowski static int
113002b553caSRafal Jaworowski sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n,
113102b553caSRafal Jaworowski     bus_addr_t data, bus_size_t dsize)
113202b553caSRafal Jaworowski {
113302b553caSRafal Jaworowski 	struct sec_hw_desc_ptr *ptr;
113402b553caSRafal Jaworowski 
113502b553caSRafal Jaworowski 	SEC_LOCK_ASSERT(sc, descriptors);
113602b553caSRafal Jaworowski 
113702b553caSRafal Jaworowski 	ptr = &(desc->sd_desc->shd_pointer[n]);
113802b553caSRafal Jaworowski 	ptr->shdp_length = dsize;
113902b553caSRafal Jaworowski 	ptr->shdp_extent = 0;
114002b553caSRafal Jaworowski 	ptr->shdp_j = 0;
114102b553caSRafal Jaworowski 	ptr->shdp_ptr = data;
114202b553caSRafal Jaworowski 
114302b553caSRafal Jaworowski 	return (0);
114402b553caSRafal Jaworowski }
114502b553caSRafal Jaworowski 
114602b553caSRafal Jaworowski static int
114702b553caSRafal Jaworowski sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
114802b553caSRafal Jaworowski     u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype)
114902b553caSRafal Jaworowski {
115002b553caSRafal Jaworowski 	struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
115102b553caSRafal Jaworowski 	struct sec_hw_desc_ptr *ptr;
115202b553caSRafal Jaworowski 	int error;
115302b553caSRafal Jaworowski 
115402b553caSRafal Jaworowski 	SEC_LOCK_ASSERT(sc, descriptors);
115502b553caSRafal Jaworowski 
115602b553caSRafal Jaworowski 	/* For flat memory map only requested region */
115702b553caSRafal Jaworowski 	if (dtype == SEC_MEMORY) {
115802b553caSRafal Jaworowski 		 data = (uint8_t*)(data) + doffset;
115902b553caSRafal Jaworowski 		 sdmi.sdmi_offset = 0;
116002b553caSRafal Jaworowski 	}
116102b553caSRafal Jaworowski 
116202b553caSRafal Jaworowski 	error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize,
116302b553caSRafal Jaworowski 	    dtype, &sdmi);
116402b553caSRafal Jaworowski 
116502b553caSRafal Jaworowski 	if (error)
116602b553caSRafal Jaworowski 		return (error);
116702b553caSRafal Jaworowski 
116802b553caSRafal Jaworowski 	sdmi.sdmi_lt_last->sl_lt->shl_r = 1;
116902b553caSRafal Jaworowski 	desc->sd_lt_used += sdmi.sdmi_lt_used;
117002b553caSRafal Jaworowski 
117102b553caSRafal Jaworowski 	ptr = &(desc->sd_desc->shd_pointer[n]);
117202b553caSRafal Jaworowski 	ptr->shdp_length = dsize;
117302b553caSRafal Jaworowski 	ptr->shdp_extent = 0;
117402b553caSRafal Jaworowski 	ptr->shdp_j = 1;
117502b553caSRafal Jaworowski 	ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr;
117602b553caSRafal Jaworowski 
117702b553caSRafal Jaworowski 	return (0);
117802b553caSRafal Jaworowski }
117902b553caSRafal Jaworowski 
118002b553caSRafal Jaworowski static int
118102b553caSRafal Jaworowski sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
118202b553caSRafal Jaworowski     struct cryptoini **mac)
118302b553caSRafal Jaworowski {
118402b553caSRafal Jaworowski 	struct cryptoini *e, *m;
118502b553caSRafal Jaworowski 
118602b553caSRafal Jaworowski 	e = cri;
118702b553caSRafal Jaworowski 	m = cri->cri_next;
118802b553caSRafal Jaworowski 
118902b553caSRafal Jaworowski 	/* We can haldle only two operations */
119002b553caSRafal Jaworowski 	if (m && m->cri_next)
119102b553caSRafal Jaworowski 		return (EINVAL);
119202b553caSRafal Jaworowski 
119302b553caSRafal Jaworowski 	if (sec_mdeu_can_handle(e->cri_alg)) {
119402b553caSRafal Jaworowski 		cri = m;
119502b553caSRafal Jaworowski 		m = e;
119602b553caSRafal Jaworowski 		e = cri;
119702b553caSRafal Jaworowski 	}
119802b553caSRafal Jaworowski 
119902b553caSRafal Jaworowski 	if (m && !sec_mdeu_can_handle(m->cri_alg))
120002b553caSRafal Jaworowski 		return (EINVAL);
120102b553caSRafal Jaworowski 
120202b553caSRafal Jaworowski 	*enc = e;
120302b553caSRafal Jaworowski 	*mac = m;
120402b553caSRafal Jaworowski 
120502b553caSRafal Jaworowski 	return (0);
120602b553caSRafal Jaworowski }
120702b553caSRafal Jaworowski 
120802b553caSRafal Jaworowski static int
120902b553caSRafal Jaworowski sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
121002b553caSRafal Jaworowski     struct cryptodesc **mac)
121102b553caSRafal Jaworowski {
121202b553caSRafal Jaworowski 	struct cryptodesc *e, *m, *t;
121302b553caSRafal Jaworowski 
121402b553caSRafal Jaworowski 	e = crp->crp_desc;
121502b553caSRafal Jaworowski 	m = e->crd_next;
121602b553caSRafal Jaworowski 
121702b553caSRafal Jaworowski 	/* We can haldle only two operations */
121802b553caSRafal Jaworowski 	if (m && m->crd_next)
121902b553caSRafal Jaworowski 		return (EINVAL);
122002b553caSRafal Jaworowski 
122102b553caSRafal Jaworowski 	if (sec_mdeu_can_handle(e->crd_alg)) {
122202b553caSRafal Jaworowski 		t = m;
122302b553caSRafal Jaworowski 		m = e;
122402b553caSRafal Jaworowski 		e = t;
122502b553caSRafal Jaworowski 	}
122602b553caSRafal Jaworowski 
122702b553caSRafal Jaworowski 	if (m && !sec_mdeu_can_handle(m->crd_alg))
122802b553caSRafal Jaworowski 		return (EINVAL);
122902b553caSRafal Jaworowski 
123002b553caSRafal Jaworowski 	*enc = e;
123102b553caSRafal Jaworowski 	*mac = m;
123202b553caSRafal Jaworowski 
123302b553caSRafal Jaworowski 	return (0);
123402b553caSRafal Jaworowski }
123502b553caSRafal Jaworowski 
123602b553caSRafal Jaworowski static int
123702b553caSRafal Jaworowski sec_alloc_session(struct sec_softc *sc)
123802b553caSRafal Jaworowski {
123902b553caSRafal Jaworowski 	struct sec_session *ses = NULL;
124002b553caSRafal Jaworowski 	int sid = -1;
124102b553caSRafal Jaworowski 	u_int i;
124202b553caSRafal Jaworowski 
124302b553caSRafal Jaworowski 	SEC_LOCK(sc, sessions);
124402b553caSRafal Jaworowski 
124502b553caSRafal Jaworowski 	for (i = 0; i < SEC_MAX_SESSIONS; i++) {
124602b553caSRafal Jaworowski 		if (sc->sc_sessions[i].ss_used == 0) {
124702b553caSRafal Jaworowski 			ses = &(sc->sc_sessions[i]);
124802b553caSRafal Jaworowski 			ses->ss_used = 1;
124902b553caSRafal Jaworowski 			ses->ss_ivlen = 0;
125002b553caSRafal Jaworowski 			ses->ss_klen = 0;
125102b553caSRafal Jaworowski 			ses->ss_mklen = 0;
125202b553caSRafal Jaworowski 			sid = i;
125302b553caSRafal Jaworowski 			break;
125402b553caSRafal Jaworowski 		}
125502b553caSRafal Jaworowski 	}
125602b553caSRafal Jaworowski 
125702b553caSRafal Jaworowski 	SEC_UNLOCK(sc, sessions);
125802b553caSRafal Jaworowski 
125902b553caSRafal Jaworowski 	return (sid);
126002b553caSRafal Jaworowski }
126102b553caSRafal Jaworowski 
126202b553caSRafal Jaworowski static struct sec_session *
126302b553caSRafal Jaworowski sec_get_session(struct sec_softc *sc, u_int sid)
126402b553caSRafal Jaworowski {
126502b553caSRafal Jaworowski 	struct sec_session *ses;
126602b553caSRafal Jaworowski 
126702b553caSRafal Jaworowski 	if (sid >= SEC_MAX_SESSIONS)
126802b553caSRafal Jaworowski 		return (NULL);
126902b553caSRafal Jaworowski 
127002b553caSRafal Jaworowski 	SEC_LOCK(sc, sessions);
127102b553caSRafal Jaworowski 
127202b553caSRafal Jaworowski 	ses = &(sc->sc_sessions[sid]);
127302b553caSRafal Jaworowski 
127402b553caSRafal Jaworowski 	if (ses->ss_used == 0)
127502b553caSRafal Jaworowski 		ses = NULL;
127602b553caSRafal Jaworowski 
127702b553caSRafal Jaworowski 	SEC_UNLOCK(sc, sessions);
127802b553caSRafal Jaworowski 
127902b553caSRafal Jaworowski 	return (ses);
128002b553caSRafal Jaworowski }
128102b553caSRafal Jaworowski 
128202b553caSRafal Jaworowski static int
128302b553caSRafal Jaworowski sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
128402b553caSRafal Jaworowski {
128502b553caSRafal Jaworowski 	struct sec_softc *sc = device_get_softc(dev);
128602b553caSRafal Jaworowski 	struct sec_eu_methods *eu = sec_eus;
128702b553caSRafal Jaworowski 	struct cryptoini *enc = NULL;
128802b553caSRafal Jaworowski 	struct cryptoini *mac = NULL;
128902b553caSRafal Jaworowski 	struct sec_session *ses;
129002b553caSRafal Jaworowski 	int error = -1;
129102b553caSRafal Jaworowski 	int sid;
129202b553caSRafal Jaworowski 
129302b553caSRafal Jaworowski 	error = sec_split_cri(cri, &enc, &mac);
129402b553caSRafal Jaworowski 	if (error)
129502b553caSRafal Jaworowski 		return (error);
129602b553caSRafal Jaworowski 
129702b553caSRafal Jaworowski 	/* Check key lengths */
129802b553caSRafal Jaworowski 	if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN)
129902b553caSRafal Jaworowski 		return (E2BIG);
130002b553caSRafal Jaworowski 
130102b553caSRafal Jaworowski 	if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN)
130202b553caSRafal Jaworowski 		return (E2BIG);
130302b553caSRafal Jaworowski 
130402b553caSRafal Jaworowski 	/* Only SEC 3.0 supports digests larger than 256 bits */
130502b553caSRafal Jaworowski 	if (sc->sc_version < 3 && mac && mac->cri_klen > 256)
130602b553caSRafal Jaworowski 		return (E2BIG);
130702b553caSRafal Jaworowski 
130802b553caSRafal Jaworowski 	sid = sec_alloc_session(sc);
130902b553caSRafal Jaworowski 	if (sid < 0)
131002b553caSRafal Jaworowski 		return (ENOMEM);
131102b553caSRafal Jaworowski 
131202b553caSRafal Jaworowski 	ses = sec_get_session(sc, sid);
131302b553caSRafal Jaworowski 
131402b553caSRafal Jaworowski 	/* Find EU for this session */
131502b553caSRafal Jaworowski 	while (eu->sem_make_desc != NULL) {
131602b553caSRafal Jaworowski 		error = eu->sem_newsession(sc, ses, enc, mac);
131702b553caSRafal Jaworowski 		if (error >= 0)
131802b553caSRafal Jaworowski 			break;
131902b553caSRafal Jaworowski 
132002b553caSRafal Jaworowski 		eu++;
132102b553caSRafal Jaworowski 	}
132202b553caSRafal Jaworowski 
132302b553caSRafal Jaworowski 	/* If not found, return EINVAL */
132402b553caSRafal Jaworowski 	if (error < 0) {
132502b553caSRafal Jaworowski 		sec_free_session(sc, ses);
132602b553caSRafal Jaworowski 		return (EINVAL);
132702b553caSRafal Jaworowski 	}
132802b553caSRafal Jaworowski 
132902b553caSRafal Jaworowski 	/* Save cipher key */
133002b553caSRafal Jaworowski 	if (enc && enc->cri_key) {
133102b553caSRafal Jaworowski 		ses->ss_klen = enc->cri_klen / 8;
133202b553caSRafal Jaworowski 		memcpy(ses->ss_key, enc->cri_key, ses->ss_klen);
133302b553caSRafal Jaworowski 	}
133402b553caSRafal Jaworowski 
133502b553caSRafal Jaworowski 	/* Save digest key */
133602b553caSRafal Jaworowski 	if (mac && mac->cri_key) {
133702b553caSRafal Jaworowski 		ses->ss_mklen = mac->cri_klen / 8;
133802b553caSRafal Jaworowski 		memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen);
133902b553caSRafal Jaworowski 	}
134002b553caSRafal Jaworowski 
134102b553caSRafal Jaworowski 	ses->ss_eu = eu;
134202b553caSRafal Jaworowski 	*sidp = sid;
134302b553caSRafal Jaworowski 
134402b553caSRafal Jaworowski 	return (0);
134502b553caSRafal Jaworowski }
134602b553caSRafal Jaworowski 
134702b553caSRafal Jaworowski static int
134802b553caSRafal Jaworowski sec_freesession(device_t dev, uint64_t tid)
134902b553caSRafal Jaworowski {
135002b553caSRafal Jaworowski 	struct sec_softc *sc = device_get_softc(dev);
135102b553caSRafal Jaworowski 	struct sec_session *ses;
135202b553caSRafal Jaworowski 	int error = 0;
135302b553caSRafal Jaworowski 
135402b553caSRafal Jaworowski 	ses = sec_get_session(sc, CRYPTO_SESID2LID(tid));
135502b553caSRafal Jaworowski 	if (ses == NULL)
135602b553caSRafal Jaworowski 		return (EINVAL);
135702b553caSRafal Jaworowski 
135802b553caSRafal Jaworowski 	sec_free_session(sc, ses);
135902b553caSRafal Jaworowski 
136002b553caSRafal Jaworowski 	return (error);
136102b553caSRafal Jaworowski }
136202b553caSRafal Jaworowski 
136302b553caSRafal Jaworowski static int
136402b553caSRafal Jaworowski sec_process(device_t dev, struct cryptop *crp, int hint)
136502b553caSRafal Jaworowski {
136602b553caSRafal Jaworowski 	struct sec_softc *sc = device_get_softc(dev);
136702b553caSRafal Jaworowski 	struct sec_desc *desc = NULL;
136802b553caSRafal Jaworowski 	struct cryptodesc *mac, *enc;
136902b553caSRafal Jaworowski 	struct sec_session *ses;
137002b553caSRafal Jaworowski 	int buftype, error = 0;
137102b553caSRafal Jaworowski 
137202b553caSRafal Jaworowski 	/* Check Session ID */
137302b553caSRafal Jaworowski 	ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid));
137402b553caSRafal Jaworowski 	if (ses == NULL) {
137502b553caSRafal Jaworowski 		crp->crp_etype = EINVAL;
137602b553caSRafal Jaworowski 		crypto_done(crp);
137702b553caSRafal Jaworowski 		return (0);
137802b553caSRafal Jaworowski 	}
137902b553caSRafal Jaworowski 
138002b553caSRafal Jaworowski 	/* Check for input length */
138102b553caSRafal Jaworowski 	if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) {
138202b553caSRafal Jaworowski 		crp->crp_etype = E2BIG;
138302b553caSRafal Jaworowski 		crypto_done(crp);
138402b553caSRafal Jaworowski 		return (0);
138502b553caSRafal Jaworowski 	}
138602b553caSRafal Jaworowski 
138702b553caSRafal Jaworowski 	/* Get descriptors */
138802b553caSRafal Jaworowski 	if (sec_split_crp(crp, &enc, &mac)) {
138902b553caSRafal Jaworowski 		crp->crp_etype = EINVAL;
139002b553caSRafal Jaworowski 		crypto_done(crp);
139102b553caSRafal Jaworowski 		return (0);
139202b553caSRafal Jaworowski 	}
139302b553caSRafal Jaworowski 
139402b553caSRafal Jaworowski 	SEC_LOCK(sc, descriptors);
139502b553caSRafal Jaworowski 	SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
139602b553caSRafal Jaworowski 
139702b553caSRafal Jaworowski 	/* Block driver if there is no free descriptors or we are going down */
139802b553caSRafal Jaworowski 	if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) {
139902b553caSRafal Jaworowski 		sc->sc_blocked |= CRYPTO_SYMQ;
140002b553caSRafal Jaworowski 		SEC_UNLOCK(sc, descriptors);
140102b553caSRafal Jaworowski 		return (ERESTART);
140202b553caSRafal Jaworowski 	}
140302b553caSRafal Jaworowski 
140402b553caSRafal Jaworowski 	/* Prepare descriptor */
140502b553caSRafal Jaworowski 	desc = SEC_GET_FREE_DESC(sc);
140602b553caSRafal Jaworowski 	desc->sd_lt_used = 0;
140702b553caSRafal Jaworowski 	desc->sd_error = 0;
140802b553caSRafal Jaworowski 	desc->sd_crp = crp;
140902b553caSRafal Jaworowski 
141002b553caSRafal Jaworowski 	if (crp->crp_flags & CRYPTO_F_IOV)
141102b553caSRafal Jaworowski 		buftype = SEC_UIO;
141202b553caSRafal Jaworowski 	else if (crp->crp_flags & CRYPTO_F_IMBUF)
141302b553caSRafal Jaworowski 		buftype = SEC_MBUF;
141402b553caSRafal Jaworowski 	else
141502b553caSRafal Jaworowski 		buftype = SEC_MEMORY;
141602b553caSRafal Jaworowski 
141702b553caSRafal Jaworowski 	if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
141802b553caSRafal Jaworowski 		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
141902b553caSRafal Jaworowski 			memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
142002b553caSRafal Jaworowski 			    ses->ss_ivlen);
142102b553caSRafal Jaworowski 		else
142202b553caSRafal Jaworowski 			arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0);
142302b553caSRafal Jaworowski 
142402b553caSRafal Jaworowski 		if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
142502b553caSRafal Jaworowski 			crypto_copyback(crp->crp_flags, crp->crp_buf,
142602b553caSRafal Jaworowski 			    enc->crd_inject, ses->ss_ivlen,
142702b553caSRafal Jaworowski 			    desc->sd_desc->shd_iv);
142802b553caSRafal Jaworowski 	} else if (enc) {
142902b553caSRafal Jaworowski 		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
143002b553caSRafal Jaworowski 			memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
143102b553caSRafal Jaworowski 			    ses->ss_ivlen);
143202b553caSRafal Jaworowski 		else
143302b553caSRafal Jaworowski 			crypto_copydata(crp->crp_flags, crp->crp_buf,
143402b553caSRafal Jaworowski 			    enc->crd_inject, ses->ss_ivlen,
143502b553caSRafal Jaworowski 			    desc->sd_desc->shd_iv);
143602b553caSRafal Jaworowski 	}
143702b553caSRafal Jaworowski 
143802b553caSRafal Jaworowski 	if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
143902b553caSRafal Jaworowski 		if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
144002b553caSRafal Jaworowski 			ses->ss_klen = enc->crd_klen / 8;
144102b553caSRafal Jaworowski 			memcpy(ses->ss_key, enc->crd_key, ses->ss_klen);
144202b553caSRafal Jaworowski 		} else
144302b553caSRafal Jaworowski 			error = E2BIG;
144402b553caSRafal Jaworowski 	}
144502b553caSRafal Jaworowski 
144602b553caSRafal Jaworowski 	if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
144702b553caSRafal Jaworowski 		if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
144802b553caSRafal Jaworowski 			ses->ss_mklen = mac->crd_klen / 8;
144902b553caSRafal Jaworowski 			memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen);
145002b553caSRafal Jaworowski 		} else
145102b553caSRafal Jaworowski 			error = E2BIG;
145202b553caSRafal Jaworowski 	}
145302b553caSRafal Jaworowski 
145402b553caSRafal Jaworowski 	if (!error) {
145502b553caSRafal Jaworowski 		memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen);
145602b553caSRafal Jaworowski 		memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen);
145702b553caSRafal Jaworowski 
145802b553caSRafal Jaworowski 		error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype);
145902b553caSRafal Jaworowski 	}
146002b553caSRafal Jaworowski 
146102b553caSRafal Jaworowski 	if (error) {
146202b553caSRafal Jaworowski 		SEC_DESC_FREE_POINTERS(desc);
146302b553caSRafal Jaworowski 		SEC_DESC_PUT_BACK_LT(sc, desc);
146402b553caSRafal Jaworowski 		SEC_PUT_BACK_FREE_DESC(sc);
146502b553caSRafal Jaworowski 		SEC_UNLOCK(sc, descriptors);
146602b553caSRafal Jaworowski 		crp->crp_etype = error;
146702b553caSRafal Jaworowski 		crypto_done(crp);
146802b553caSRafal Jaworowski 		return (0);
146902b553caSRafal Jaworowski 	}
147002b553caSRafal Jaworowski 
147102b553caSRafal Jaworowski 	/*
147202b553caSRafal Jaworowski 	 * Skip DONE interrupt if this is not last request in burst, but only
147302b553caSRafal Jaworowski 	 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE
147402b553caSRafal Jaworowski 	 * signaling on each descriptor.
147502b553caSRafal Jaworowski 	 */
147602b553caSRafal Jaworowski 	if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3)
147702b553caSRafal Jaworowski 		desc->sd_desc->shd_dn = 0;
147802b553caSRafal Jaworowski 	else
147902b553caSRafal Jaworowski 		desc->sd_desc->shd_dn = 1;
148002b553caSRafal Jaworowski 
148102b553caSRafal Jaworowski 	SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
148202b553caSRafal Jaworowski 	SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD |
148302b553caSRafal Jaworowski 	    BUS_DMASYNC_POSTWRITE);
148402b553caSRafal Jaworowski 	SEC_DESC_FREE2READY(sc);
148502b553caSRafal Jaworowski 	SEC_UNLOCK(sc, descriptors);
148602b553caSRafal Jaworowski 
148702b553caSRafal Jaworowski 	/* Enqueue ready descriptors in hardware */
148802b553caSRafal Jaworowski 	sec_enqueue(sc);
148902b553caSRafal Jaworowski 
149002b553caSRafal Jaworowski 	return (0);
149102b553caSRafal Jaworowski }
149202b553caSRafal Jaworowski 
149302b553caSRafal Jaworowski static int
149402b553caSRafal Jaworowski sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
149502b553caSRafal Jaworowski     struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
149602b553caSRafal Jaworowski     int buftype)
149702b553caSRafal Jaworowski {
149802b553caSRafal Jaworowski 	struct sec_hw_desc *hd = desc->sd_desc;
149902b553caSRafal Jaworowski 	int error;
150002b553caSRafal Jaworowski 
150102b553caSRafal Jaworowski 	hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
150202b553caSRafal Jaworowski 	hd->shd_eu_sel1 = SEC_EU_NONE;
150302b553caSRafal Jaworowski 	hd->shd_mode1 = 0;
150402b553caSRafal Jaworowski 
150502b553caSRafal Jaworowski 	/* Pointer 0: NULL */
150602b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
150702b553caSRafal Jaworowski 	if (error)
150802b553caSRafal Jaworowski 		return (error);
150902b553caSRafal Jaworowski 
151002b553caSRafal Jaworowski 	/* Pointer 1: IV IN */
151102b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
151202b553caSRafal Jaworowski 	    offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
151302b553caSRafal Jaworowski 	if (error)
151402b553caSRafal Jaworowski 		return (error);
151502b553caSRafal Jaworowski 
151602b553caSRafal Jaworowski 	/* Pointer 2: Cipher Key */
151702b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
151802b553caSRafal Jaworowski 	    offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
151902b553caSRafal Jaworowski  	if (error)
152002b553caSRafal Jaworowski 		return (error);
152102b553caSRafal Jaworowski 
152202b553caSRafal Jaworowski 	/* Pointer 3: Data IN */
152302b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip,
152402b553caSRafal Jaworowski 	    enc->crd_len, buftype);
152502b553caSRafal Jaworowski 	if (error)
152602b553caSRafal Jaworowski 		return (error);
152702b553caSRafal Jaworowski 
152802b553caSRafal Jaworowski 	/* Pointer 4: Data OUT */
152902b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
153002b553caSRafal Jaworowski 	    enc->crd_len, buftype);
153102b553caSRafal Jaworowski 	if (error)
153202b553caSRafal Jaworowski 		return (error);
153302b553caSRafal Jaworowski 
153402b553caSRafal Jaworowski 	/* Pointer 5: IV OUT (Not used: NULL) */
153502b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 5, 0, 0);
153602b553caSRafal Jaworowski 	if (error)
153702b553caSRafal Jaworowski 		return (error);
153802b553caSRafal Jaworowski 
153902b553caSRafal Jaworowski 	/* Pointer 6: NULL */
154002b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
154102b553caSRafal Jaworowski 
154202b553caSRafal Jaworowski 	return (error);
154302b553caSRafal Jaworowski }
154402b553caSRafal Jaworowski 
154502b553caSRafal Jaworowski static int
154602b553caSRafal Jaworowski sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
154702b553caSRafal Jaworowski     struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
154802b553caSRafal Jaworowski     struct cryptodesc *mac, int buftype)
154902b553caSRafal Jaworowski {
155002b553caSRafal Jaworowski 	struct sec_hw_desc *hd = desc->sd_desc;
155102b553caSRafal Jaworowski 	u_int eu, mode, hashlen;
155202b553caSRafal Jaworowski 	int error;
155302b553caSRafal Jaworowski 
155402b553caSRafal Jaworowski 	if (mac->crd_len < enc->crd_len)
155502b553caSRafal Jaworowski 		return (EINVAL);
155602b553caSRafal Jaworowski 
155702b553caSRafal Jaworowski 	if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len)
155802b553caSRafal Jaworowski 		return (EINVAL);
155902b553caSRafal Jaworowski 
156002b553caSRafal Jaworowski 	error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
156102b553caSRafal Jaworowski 	if (error)
156202b553caSRafal Jaworowski 		return (error);
156302b553caSRafal Jaworowski 
156402b553caSRafal Jaworowski 	hd->shd_desc_type = SEC_DT_HMAC_SNOOP;
156502b553caSRafal Jaworowski 	hd->shd_eu_sel1 = eu;
156602b553caSRafal Jaworowski 	hd->shd_mode1 = mode;
156702b553caSRafal Jaworowski 
156802b553caSRafal Jaworowski 	/* Pointer 0: HMAC Key */
156902b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
157002b553caSRafal Jaworowski 	    offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen);
157102b553caSRafal Jaworowski 	if (error)
157202b553caSRafal Jaworowski 		return (error);
157302b553caSRafal Jaworowski 
157402b553caSRafal Jaworowski 	/* Pointer 1: HMAC-Only Data IN */
157502b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip,
157602b553caSRafal Jaworowski 	    mac->crd_len - enc->crd_len, buftype);
157702b553caSRafal Jaworowski 	if (error)
157802b553caSRafal Jaworowski 		return (error);
157902b553caSRafal Jaworowski 
158002b553caSRafal Jaworowski 	/* Pointer 2: Cipher Key */
158102b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
158202b553caSRafal Jaworowski 	    offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
158302b553caSRafal Jaworowski  	if (error)
158402b553caSRafal Jaworowski 		return (error);
158502b553caSRafal Jaworowski 
158602b553caSRafal Jaworowski 	/* Pointer 3: IV IN */
158702b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
158802b553caSRafal Jaworowski 	    offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
158902b553caSRafal Jaworowski 	if (error)
159002b553caSRafal Jaworowski 		return (error);
159102b553caSRafal Jaworowski 
159202b553caSRafal Jaworowski 	/* Pointer 4: Data IN */
159302b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
159402b553caSRafal Jaworowski 	    enc->crd_len, buftype);
159502b553caSRafal Jaworowski 	if (error)
159602b553caSRafal Jaworowski 		return (error);
159702b553caSRafal Jaworowski 
159802b553caSRafal Jaworowski 	/* Pointer 5: Data OUT */
159902b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip,
160002b553caSRafal Jaworowski 	    enc->crd_len, buftype);
160102b553caSRafal Jaworowski 	if (error)
160202b553caSRafal Jaworowski 		return (error);
160302b553caSRafal Jaworowski 
160402b553caSRafal Jaworowski 	/* Pointer 6: HMAC OUT */
160502b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject,
160602b553caSRafal Jaworowski 	    hashlen, buftype);
160702b553caSRafal Jaworowski 
160802b553caSRafal Jaworowski 	return (error);
160902b553caSRafal Jaworowski }
161002b553caSRafal Jaworowski 
161102b553caSRafal Jaworowski /* AESU */
161202b553caSRafal Jaworowski 
161302b553caSRafal Jaworowski static int
161402b553caSRafal Jaworowski sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses,
161502b553caSRafal Jaworowski     struct cryptoini *enc, struct cryptoini *mac)
161602b553caSRafal Jaworowski {
161702b553caSRafal Jaworowski 
161802b553caSRafal Jaworowski 	if (enc == NULL)
161902b553caSRafal Jaworowski 		return (-1);
162002b553caSRafal Jaworowski 
162102b553caSRafal Jaworowski 	if (enc->cri_alg != CRYPTO_AES_CBC)
162202b553caSRafal Jaworowski 		return (-1);
162302b553caSRafal Jaworowski 
162402b553caSRafal Jaworowski 	ses->ss_ivlen = AES_BLOCK_LEN;
162502b553caSRafal Jaworowski 
162602b553caSRafal Jaworowski 	return (0);
162702b553caSRafal Jaworowski }
162802b553caSRafal Jaworowski 
162902b553caSRafal Jaworowski static int
163002b553caSRafal Jaworowski sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses,
163102b553caSRafal Jaworowski     struct sec_desc *desc, struct cryptop *crp, int buftype)
163202b553caSRafal Jaworowski {
163302b553caSRafal Jaworowski 	struct sec_hw_desc *hd = desc->sd_desc;
163402b553caSRafal Jaworowski 	struct cryptodesc *enc, *mac;
163502b553caSRafal Jaworowski 	int error;
163602b553caSRafal Jaworowski 
163702b553caSRafal Jaworowski 	error = sec_split_crp(crp, &enc, &mac);
163802b553caSRafal Jaworowski 	if (error)
163902b553caSRafal Jaworowski 		return (error);
164002b553caSRafal Jaworowski 
164102b553caSRafal Jaworowski 	if (!enc)
164202b553caSRafal Jaworowski 		return (EINVAL);
164302b553caSRafal Jaworowski 
164402b553caSRafal Jaworowski 	hd->shd_eu_sel0 = SEC_EU_AESU;
164502b553caSRafal Jaworowski 	hd->shd_mode0 = SEC_AESU_MODE_CBC;
164602b553caSRafal Jaworowski 
164702b553caSRafal Jaworowski 	if (enc->crd_alg != CRYPTO_AES_CBC)
164802b553caSRafal Jaworowski 		return (EINVAL);
164902b553caSRafal Jaworowski 
165002b553caSRafal Jaworowski 	if (enc->crd_flags & CRD_F_ENCRYPT) {
165102b553caSRafal Jaworowski 		hd->shd_mode0 |= SEC_AESU_MODE_ED;
165202b553caSRafal Jaworowski 		hd->shd_dir = 0;
165302b553caSRafal Jaworowski 	} else
165402b553caSRafal Jaworowski 		hd->shd_dir = 1;
165502b553caSRafal Jaworowski 
165602b553caSRafal Jaworowski 	if (mac)
165702b553caSRafal Jaworowski 		error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
165802b553caSRafal Jaworowski 		    buftype);
165902b553caSRafal Jaworowski 	else
166002b553caSRafal Jaworowski 		error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
166102b553caSRafal Jaworowski 		    buftype);
166202b553caSRafal Jaworowski 
166302b553caSRafal Jaworowski 	return (error);
166402b553caSRafal Jaworowski }
166502b553caSRafal Jaworowski 
166602b553caSRafal Jaworowski /* DEU */
166702b553caSRafal Jaworowski 
166802b553caSRafal Jaworowski static int
166902b553caSRafal Jaworowski sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses,
167002b553caSRafal Jaworowski     struct cryptoini *enc, struct cryptoini *mac)
167102b553caSRafal Jaworowski {
167202b553caSRafal Jaworowski 
167302b553caSRafal Jaworowski 	if (enc == NULL)
167402b553caSRafal Jaworowski 		return (-1);
167502b553caSRafal Jaworowski 
167602b553caSRafal Jaworowski 	switch (enc->cri_alg) {
167702b553caSRafal Jaworowski 	case CRYPTO_DES_CBC:
167802b553caSRafal Jaworowski 	case CRYPTO_3DES_CBC:
167902b553caSRafal Jaworowski 		break;
168002b553caSRafal Jaworowski 	default:
168102b553caSRafal Jaworowski 		return (-1);
168202b553caSRafal Jaworowski 	}
168302b553caSRafal Jaworowski 
168402b553caSRafal Jaworowski 	ses->ss_ivlen = DES_BLOCK_LEN;
168502b553caSRafal Jaworowski 
168602b553caSRafal Jaworowski 	return (0);
168702b553caSRafal Jaworowski }
168802b553caSRafal Jaworowski 
168902b553caSRafal Jaworowski static int
169002b553caSRafal Jaworowski sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses,
169102b553caSRafal Jaworowski     struct sec_desc *desc, struct cryptop *crp, int buftype)
169202b553caSRafal Jaworowski {
169302b553caSRafal Jaworowski 	struct sec_hw_desc *hd = desc->sd_desc;
169402b553caSRafal Jaworowski 	struct cryptodesc *enc, *mac;
169502b553caSRafal Jaworowski 	int error;
169602b553caSRafal Jaworowski 
169702b553caSRafal Jaworowski 	error = sec_split_crp(crp, &enc, &mac);
169802b553caSRafal Jaworowski 	if (error)
169902b553caSRafal Jaworowski 		return (error);
170002b553caSRafal Jaworowski 
170102b553caSRafal Jaworowski 	if (!enc)
170202b553caSRafal Jaworowski 		return (EINVAL);
170302b553caSRafal Jaworowski 
170402b553caSRafal Jaworowski 	hd->shd_eu_sel0 = SEC_EU_DEU;
170502b553caSRafal Jaworowski 	hd->shd_mode0 = SEC_DEU_MODE_CBC;
170602b553caSRafal Jaworowski 
170702b553caSRafal Jaworowski 	switch (enc->crd_alg) {
170802b553caSRafal Jaworowski 	case CRYPTO_3DES_CBC:
170902b553caSRafal Jaworowski 		hd->shd_mode0 |= SEC_DEU_MODE_TS;
171002b553caSRafal Jaworowski 		break;
171102b553caSRafal Jaworowski 	case CRYPTO_DES_CBC:
171202b553caSRafal Jaworowski 		break;
171302b553caSRafal Jaworowski 	default:
171402b553caSRafal Jaworowski 		return (EINVAL);
171502b553caSRafal Jaworowski 	}
171602b553caSRafal Jaworowski 
171702b553caSRafal Jaworowski 	if (enc->crd_flags & CRD_F_ENCRYPT) {
171802b553caSRafal Jaworowski 		hd->shd_mode0 |= SEC_DEU_MODE_ED;
171902b553caSRafal Jaworowski 		hd->shd_dir = 0;
172002b553caSRafal Jaworowski 	} else
172102b553caSRafal Jaworowski 		hd->shd_dir = 1;
172202b553caSRafal Jaworowski 
172302b553caSRafal Jaworowski 	if (mac)
172402b553caSRafal Jaworowski 		error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
172502b553caSRafal Jaworowski 		    buftype);
172602b553caSRafal Jaworowski 	else
172702b553caSRafal Jaworowski 		error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
172802b553caSRafal Jaworowski 		    buftype);
172902b553caSRafal Jaworowski 
173002b553caSRafal Jaworowski 	return (error);
173102b553caSRafal Jaworowski }
173202b553caSRafal Jaworowski 
173302b553caSRafal Jaworowski /* MDEU */
173402b553caSRafal Jaworowski 
173502b553caSRafal Jaworowski static int
173602b553caSRafal Jaworowski sec_mdeu_can_handle(u_int alg)
173702b553caSRafal Jaworowski {
173802b553caSRafal Jaworowski 	switch (alg) {
173902b553caSRafal Jaworowski 	case CRYPTO_MD5:
174002b553caSRafal Jaworowski 	case CRYPTO_SHA1:
174102b553caSRafal Jaworowski 	case CRYPTO_MD5_HMAC:
174202b553caSRafal Jaworowski 	case CRYPTO_SHA1_HMAC:
174302b553caSRafal Jaworowski 	case CRYPTO_SHA2_256_HMAC:
174402b553caSRafal Jaworowski 	case CRYPTO_SHA2_384_HMAC:
174502b553caSRafal Jaworowski 	case CRYPTO_SHA2_512_HMAC:
174602b553caSRafal Jaworowski 		return (1);
174702b553caSRafal Jaworowski 	default:
174802b553caSRafal Jaworowski 		return (0);
174902b553caSRafal Jaworowski 	}
175002b553caSRafal Jaworowski }
175102b553caSRafal Jaworowski 
175202b553caSRafal Jaworowski static int
175302b553caSRafal Jaworowski sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen)
175402b553caSRafal Jaworowski {
175502b553caSRafal Jaworowski 
175602b553caSRafal Jaworowski 	*mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
175702b553caSRafal Jaworowski 	*eu = SEC_EU_NONE;
175802b553caSRafal Jaworowski 
175902b553caSRafal Jaworowski 	switch (crd->crd_alg) {
176002b553caSRafal Jaworowski 	case CRYPTO_MD5_HMAC:
176102b553caSRafal Jaworowski 		*mode |= SEC_MDEU_MODE_HMAC;
176202b553caSRafal Jaworowski 		/* FALLTHROUGH */
176302b553caSRafal Jaworowski 	case CRYPTO_MD5:
176402b553caSRafal Jaworowski 		*eu = SEC_EU_MDEU_A;
176502b553caSRafal Jaworowski 		*mode |= SEC_MDEU_MODE_MD5;
176602b553caSRafal Jaworowski 		*hashlen = MD5_HASH_LEN;
176702b553caSRafal Jaworowski 		break;
176802b553caSRafal Jaworowski 	case CRYPTO_SHA1_HMAC:
176902b553caSRafal Jaworowski 		*mode |= SEC_MDEU_MODE_HMAC;
177002b553caSRafal Jaworowski 		/* FALLTHROUGH */
177102b553caSRafal Jaworowski 	case CRYPTO_SHA1:
177202b553caSRafal Jaworowski 		*eu = SEC_EU_MDEU_A;
177302b553caSRafal Jaworowski 		*mode |= SEC_MDEU_MODE_SHA1;
177402b553caSRafal Jaworowski 		*hashlen = SHA1_HASH_LEN;
177502b553caSRafal Jaworowski 		break;
177602b553caSRafal Jaworowski 	case CRYPTO_SHA2_256_HMAC:
177702b553caSRafal Jaworowski 		*mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256;
177802b553caSRafal Jaworowski 		*eu = SEC_EU_MDEU_A;
177902b553caSRafal Jaworowski 		break;
178002b553caSRafal Jaworowski 	case CRYPTO_SHA2_384_HMAC:
178102b553caSRafal Jaworowski 		*mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384;
178202b553caSRafal Jaworowski 		*eu = SEC_EU_MDEU_B;
178302b553caSRafal Jaworowski 		break;
178402b553caSRafal Jaworowski 	case CRYPTO_SHA2_512_HMAC:
178502b553caSRafal Jaworowski 		*mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512;
178602b553caSRafal Jaworowski 		*eu = SEC_EU_MDEU_B;
178702b553caSRafal Jaworowski 		break;
178802b553caSRafal Jaworowski 	default:
178902b553caSRafal Jaworowski 		return (EINVAL);
179002b553caSRafal Jaworowski 	}
179102b553caSRafal Jaworowski 
179202b553caSRafal Jaworowski 	if (*mode & SEC_MDEU_MODE_HMAC)
179302b553caSRafal Jaworowski 		*hashlen = SEC_HMAC_HASH_LEN;
179402b553caSRafal Jaworowski 
179502b553caSRafal Jaworowski 	return (0);
179602b553caSRafal Jaworowski }
179702b553caSRafal Jaworowski 
179802b553caSRafal Jaworowski static int
179902b553caSRafal Jaworowski sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses,
180002b553caSRafal Jaworowski     struct cryptoini *enc, struct cryptoini *mac)
180102b553caSRafal Jaworowski {
180202b553caSRafal Jaworowski 
180302b553caSRafal Jaworowski 	if (mac && sec_mdeu_can_handle(mac->cri_alg))
180402b553caSRafal Jaworowski 		return (0);
180502b553caSRafal Jaworowski 
180602b553caSRafal Jaworowski 	return (-1);
180702b553caSRafal Jaworowski }
180802b553caSRafal Jaworowski 
180902b553caSRafal Jaworowski static int
181002b553caSRafal Jaworowski sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
181102b553caSRafal Jaworowski     struct sec_desc *desc, struct cryptop *crp, int buftype)
181202b553caSRafal Jaworowski {
181302b553caSRafal Jaworowski 	struct cryptodesc *enc, *mac;
181402b553caSRafal Jaworowski 	struct sec_hw_desc *hd = desc->sd_desc;
181502b553caSRafal Jaworowski 	u_int eu, mode, hashlen;
181602b553caSRafal Jaworowski 	int error;
181702b553caSRafal Jaworowski 
181802b553caSRafal Jaworowski 	error = sec_split_crp(crp, &enc, &mac);
181902b553caSRafal Jaworowski 	if (error)
182002b553caSRafal Jaworowski 		return (error);
182102b553caSRafal Jaworowski 
182202b553caSRafal Jaworowski 	if (enc)
182302b553caSRafal Jaworowski 		return (EINVAL);
182402b553caSRafal Jaworowski 
182502b553caSRafal Jaworowski 	error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
182602b553caSRafal Jaworowski 	if (error)
182702b553caSRafal Jaworowski 		return (error);
182802b553caSRafal Jaworowski 
182902b553caSRafal Jaworowski 	hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
183002b553caSRafal Jaworowski 	hd->shd_eu_sel0 = eu;
183102b553caSRafal Jaworowski 	hd->shd_mode0 = mode;
183202b553caSRafal Jaworowski 	hd->shd_eu_sel1 = SEC_EU_NONE;
183302b553caSRafal Jaworowski 	hd->shd_mode1 = 0;
183402b553caSRafal Jaworowski 
183502b553caSRafal Jaworowski 	/* Pointer 0: NULL */
183602b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
183702b553caSRafal Jaworowski 	if (error)
183802b553caSRafal Jaworowski 		return (error);
183902b553caSRafal Jaworowski 
184002b553caSRafal Jaworowski 	/* Pointer 1: Context In (Not used: NULL) */
184102b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 1, 0, 0);
184202b553caSRafal Jaworowski 	if (error)
184302b553caSRafal Jaworowski 		return (error);
184402b553caSRafal Jaworowski 
184502b553caSRafal Jaworowski 	/* Pointer 2: HMAC Key (or NULL, depending on digest type) */
184602b553caSRafal Jaworowski 	if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
184702b553caSRafal Jaworowski 		error = sec_make_pointer_direct(sc, desc, 2,
184802b553caSRafal Jaworowski 		    desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
184902b553caSRafal Jaworowski 		    shd_mkey), ses->ss_mklen);
185002b553caSRafal Jaworowski 	else
185102b553caSRafal Jaworowski 		error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
185202b553caSRafal Jaworowski 
185302b553caSRafal Jaworowski 	if (error)
185402b553caSRafal Jaworowski 		return (error);
185502b553caSRafal Jaworowski 
185602b553caSRafal Jaworowski 	/* Pointer 3: Input Data */
185702b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip,
185802b553caSRafal Jaworowski 	    mac->crd_len, buftype);
185902b553caSRafal Jaworowski 	if (error)
186002b553caSRafal Jaworowski 		return (error);
186102b553caSRafal Jaworowski 
186202b553caSRafal Jaworowski 	/* Pointer 4: NULL */
186302b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 4, 0, 0);
186402b553caSRafal Jaworowski 	if (error)
186502b553caSRafal Jaworowski 		return (error);
186602b553caSRafal Jaworowski 
186702b553caSRafal Jaworowski 	/* Pointer 5: Hash out */
186802b553caSRafal Jaworowski 	error = sec_make_pointer(sc, desc, 5, crp->crp_buf,
186902b553caSRafal Jaworowski 	    mac->crd_inject, hashlen, buftype);
187002b553caSRafal Jaworowski 	if (error)
187102b553caSRafal Jaworowski 		return (error);
187202b553caSRafal Jaworowski 
187302b553caSRafal Jaworowski 	/* Pointer 6: NULL */
187402b553caSRafal Jaworowski 	error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
187502b553caSRafal Jaworowski 
187602b553caSRafal Jaworowski 	return (0);
187702b553caSRafal Jaworowski }
1878