102b553caSRafal Jaworowski /*- 202b553caSRafal Jaworowski * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 302b553caSRafal Jaworowski * All rights reserved. 402b553caSRafal Jaworowski * 502b553caSRafal Jaworowski * Redistribution and use in source and binary forms, with or without 602b553caSRafal Jaworowski * modification, are permitted provided that the following conditions 702b553caSRafal Jaworowski * are met: 802b553caSRafal Jaworowski * 1. Redistributions of source code must retain the above copyright 902b553caSRafal Jaworowski * notice, this list of conditions and the following disclaimer. 1002b553caSRafal Jaworowski * 2. Redistributions in binary form must reproduce the above copyright 1102b553caSRafal Jaworowski * notice, this list of conditions and the following disclaimer in the 1202b553caSRafal Jaworowski * documentation and/or other materials provided with the distribution. 1302b553caSRafal Jaworowski * 1402b553caSRafal Jaworowski * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1502b553caSRafal Jaworowski * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1602b553caSRafal Jaworowski * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 1702b553caSRafal Jaworowski * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 1802b553caSRafal Jaworowski * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 1902b553caSRafal Jaworowski * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 2002b553caSRafal Jaworowski * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 2102b553caSRafal Jaworowski * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 2202b553caSRafal Jaworowski * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 2302b553caSRafal Jaworowski * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2402b553caSRafal Jaworowski */ 2502b553caSRafal Jaworowski 2602b553caSRafal Jaworowski /* 2702b553caSRafal Jaworowski * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 2802b553caSRafal Jaworowski * 3.0 are supported. 2902b553caSRafal Jaworowski */ 3002b553caSRafal Jaworowski 3102b553caSRafal Jaworowski #include <sys/cdefs.h> 3202b553caSRafal Jaworowski __FBSDID("$FreeBSD$"); 3302b553caSRafal Jaworowski 3402b553caSRafal Jaworowski #include <sys/param.h> 3502b553caSRafal Jaworowski #include <sys/systm.h> 3602b553caSRafal Jaworowski #include <sys/bus.h> 3702b553caSRafal Jaworowski #include <sys/endian.h> 3802b553caSRafal Jaworowski #include <sys/kernel.h> 3902b553caSRafal Jaworowski #include <sys/lock.h> 4002b553caSRafal Jaworowski #include <sys/malloc.h> 4102b553caSRafal Jaworowski #include <sys/mbuf.h> 4202b553caSRafal Jaworowski #include <sys/module.h> 4302b553caSRafal Jaworowski #include <sys/mutex.h> 4402b553caSRafal Jaworowski #include <sys/random.h> 4502b553caSRafal Jaworowski #include <sys/rman.h> 4602b553caSRafal Jaworowski 4702b553caSRafal Jaworowski #include <machine/bus.h> 4802b553caSRafal Jaworowski #include <machine/ocpbus.h> 4902b553caSRafal Jaworowski #include <machine/resource.h> 5002b553caSRafal Jaworowski 5102b553caSRafal Jaworowski #include <opencrypto/cryptodev.h> 5202b553caSRafal Jaworowski #include "cryptodev_if.h" 5302b553caSRafal Jaworowski 5402b553caSRafal Jaworowski #include <dev/sec/sec.h> 5502b553caSRafal Jaworowski 5602b553caSRafal Jaworowski static int sec_probe(device_t dev); 5702b553caSRafal Jaworowski static int sec_attach(device_t dev); 5802b553caSRafal Jaworowski static int sec_detach(device_t dev); 5902b553caSRafal Jaworowski static int sec_suspend(device_t dev); 6002b553caSRafal Jaworowski static int sec_resume(device_t dev); 61661ee6eeSRafal Jaworowski static int sec_shutdown(device_t dev); 6202b553caSRafal Jaworowski static void sec_primary_intr(void *arg); 6302b553caSRafal Jaworowski static void sec_secondary_intr(void *arg); 6402b553caSRafal Jaworowski static int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 6502b553caSRafal Jaworowski void **ihand, int *irid, driver_intr_t handler, const char *iname); 6602b553caSRafal Jaworowski static void sec_release_intr(struct sec_softc *sc, struct resource *ires, 6702b553caSRafal Jaworowski void *ihand, int irid, const char *iname); 6802b553caSRafal Jaworowski static int sec_controller_reset(struct sec_softc *sc); 6902b553caSRafal Jaworowski static int sec_channel_reset(struct sec_softc *sc, int channel, int full); 7002b553caSRafal Jaworowski static int sec_init(struct sec_softc *sc); 7102b553caSRafal Jaworowski static int sec_alloc_dma_mem(struct sec_softc *sc, 7202b553caSRafal Jaworowski struct sec_dma_mem *dma_mem, bus_size_t size); 7302b553caSRafal Jaworowski static int sec_desc_map_dma(struct sec_softc *sc, 7402b553caSRafal Jaworowski struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type, 7502b553caSRafal Jaworowski struct sec_desc_map_info *sdmi); 7602b553caSRafal Jaworowski static void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 7702b553caSRafal Jaworowski static void sec_enqueue(struct sec_softc *sc); 7802b553caSRafal Jaworowski static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 7902b553caSRafal Jaworowski int channel); 8002b553caSRafal Jaworowski static int sec_eu_channel(struct sec_softc *sc, int eu); 8102b553caSRafal Jaworowski static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 8202b553caSRafal Jaworowski u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype); 8302b553caSRafal Jaworowski static int sec_make_pointer_direct(struct sec_softc *sc, 8402b553caSRafal Jaworowski struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 8502b553caSRafal Jaworowski static int sec_alloc_session(struct sec_softc *sc); 8602b553caSRafal Jaworowski static int sec_newsession(device_t dev, u_int32_t *sidp, 8702b553caSRafal Jaworowski struct cryptoini *cri); 8802b553caSRafal Jaworowski static int sec_freesession(device_t dev, uint64_t tid); 8902b553caSRafal Jaworowski static int sec_process(device_t dev, struct cryptop *crp, int hint); 9002b553caSRafal Jaworowski static int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 9102b553caSRafal Jaworowski struct cryptoini **mac); 9202b553caSRafal Jaworowski static int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 9302b553caSRafal Jaworowski struct cryptodesc **mac); 9402b553caSRafal Jaworowski static int sec_build_common_ns_desc(struct sec_softc *sc, 9502b553caSRafal Jaworowski struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 9602b553caSRafal Jaworowski struct cryptodesc *enc, int buftype); 9702b553caSRafal Jaworowski static int sec_build_common_s_desc(struct sec_softc *sc, 9802b553caSRafal Jaworowski struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp, 9902b553caSRafal Jaworowski struct cryptodesc *enc, struct cryptodesc *mac, int buftype); 10002b553caSRafal Jaworowski 10102b553caSRafal Jaworowski static struct sec_session *sec_get_session(struct sec_softc *sc, u_int sid); 10202b553caSRafal Jaworowski static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 10302b553caSRafal Jaworowski 10402b553caSRafal Jaworowski /* AESU */ 10502b553caSRafal Jaworowski static int sec_aesu_newsession(struct sec_softc *sc, 10602b553caSRafal Jaworowski struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 10702b553caSRafal Jaworowski static int sec_aesu_make_desc(struct sec_softc *sc, 10802b553caSRafal Jaworowski struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 10902b553caSRafal Jaworowski int buftype); 11002b553caSRafal Jaworowski 11102b553caSRafal Jaworowski /* DEU */ 11202b553caSRafal Jaworowski static int sec_deu_newsession(struct sec_softc *sc, 11302b553caSRafal Jaworowski struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 11402b553caSRafal Jaworowski static int sec_deu_make_desc(struct sec_softc *sc, 11502b553caSRafal Jaworowski struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 11602b553caSRafal Jaworowski int buftype); 11702b553caSRafal Jaworowski 11802b553caSRafal Jaworowski /* MDEU */ 11902b553caSRafal Jaworowski static int sec_mdeu_can_handle(u_int alg); 12002b553caSRafal Jaworowski static int sec_mdeu_config(struct cryptodesc *crd, 12102b553caSRafal Jaworowski u_int *eu, u_int *mode, u_int *hashlen); 12202b553caSRafal Jaworowski static int sec_mdeu_newsession(struct sec_softc *sc, 12302b553caSRafal Jaworowski struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac); 12402b553caSRafal Jaworowski static int sec_mdeu_make_desc(struct sec_softc *sc, 12502b553caSRafal Jaworowski struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp, 12602b553caSRafal Jaworowski int buftype); 12702b553caSRafal Jaworowski 12802b553caSRafal Jaworowski static device_method_t sec_methods[] = { 12902b553caSRafal Jaworowski /* Device interface */ 13002b553caSRafal Jaworowski DEVMETHOD(device_probe, sec_probe), 13102b553caSRafal Jaworowski DEVMETHOD(device_attach, sec_attach), 13202b553caSRafal Jaworowski DEVMETHOD(device_detach, sec_detach), 13302b553caSRafal Jaworowski 13402b553caSRafal Jaworowski DEVMETHOD(device_suspend, sec_suspend), 13502b553caSRafal Jaworowski DEVMETHOD(device_resume, sec_resume), 13602b553caSRafal Jaworowski DEVMETHOD(device_shutdown, sec_shutdown), 13702b553caSRafal Jaworowski 13802b553caSRafal Jaworowski /* Bus interface */ 13902b553caSRafal Jaworowski DEVMETHOD(bus_print_child, bus_generic_print_child), 14002b553caSRafal Jaworowski DEVMETHOD(bus_driver_added, bus_generic_driver_added), 14102b553caSRafal Jaworowski 14202b553caSRafal Jaworowski /* Crypto methods */ 14302b553caSRafal Jaworowski DEVMETHOD(cryptodev_newsession, sec_newsession), 14402b553caSRafal Jaworowski DEVMETHOD(cryptodev_freesession,sec_freesession), 14502b553caSRafal Jaworowski DEVMETHOD(cryptodev_process, sec_process), 14602b553caSRafal Jaworowski 14702b553caSRafal Jaworowski { 0, 0 } 14802b553caSRafal Jaworowski }; 14902b553caSRafal Jaworowski static driver_t sec_driver = { 15002b553caSRafal Jaworowski "sec", 15102b553caSRafal Jaworowski sec_methods, 15202b553caSRafal Jaworowski sizeof(struct sec_softc), 15302b553caSRafal Jaworowski }; 15402b553caSRafal Jaworowski 15502b553caSRafal Jaworowski static devclass_t sec_devclass; 15602b553caSRafal Jaworowski DRIVER_MODULE(sec, ocpbus, sec_driver, sec_devclass, 0, 0); 15702b553caSRafal Jaworowski MODULE_DEPEND(sec, crypto, 1, 1, 1); 15802b553caSRafal Jaworowski 15902b553caSRafal Jaworowski static struct sec_eu_methods sec_eus[] = { 16002b553caSRafal Jaworowski { 16102b553caSRafal Jaworowski sec_aesu_newsession, 16202b553caSRafal Jaworowski sec_aesu_make_desc, 16302b553caSRafal Jaworowski }, 16402b553caSRafal Jaworowski { 16502b553caSRafal Jaworowski sec_deu_newsession, 16602b553caSRafal Jaworowski sec_deu_make_desc, 16702b553caSRafal Jaworowski }, 16802b553caSRafal Jaworowski { 16902b553caSRafal Jaworowski sec_mdeu_newsession, 17002b553caSRafal Jaworowski sec_mdeu_make_desc, 17102b553caSRafal Jaworowski }, 17202b553caSRafal Jaworowski { NULL, NULL } 17302b553caSRafal Jaworowski }; 17402b553caSRafal Jaworowski 17502b553caSRafal Jaworowski static inline void 17602b553caSRafal Jaworowski sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 17702b553caSRafal Jaworowski { 17802b553caSRafal Jaworowski 17902b553caSRafal Jaworowski /* Sync only if dma memory is valid */ 18002b553caSRafal Jaworowski if (dma_mem->dma_vaddr != NULL) 18102b553caSRafal Jaworowski bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 18202b553caSRafal Jaworowski } 18302b553caSRafal Jaworowski 18402b553caSRafal Jaworowski static inline void 18502b553caSRafal Jaworowski sec_free_session(struct sec_softc *sc, struct sec_session *ses) 18602b553caSRafal Jaworowski { 18702b553caSRafal Jaworowski 18802b553caSRafal Jaworowski SEC_LOCK(sc, sessions); 18902b553caSRafal Jaworowski ses->ss_used = 0; 19002b553caSRafal Jaworowski SEC_UNLOCK(sc, sessions); 19102b553caSRafal Jaworowski } 19202b553caSRafal Jaworowski 19302b553caSRafal Jaworowski static inline void * 19402b553caSRafal Jaworowski sec_get_pointer_data(struct sec_desc *desc, u_int n) 19502b553caSRafal Jaworowski { 19602b553caSRafal Jaworowski 19702b553caSRafal Jaworowski return (desc->sd_ptr_dmem[n].dma_vaddr); 19802b553caSRafal Jaworowski } 19902b553caSRafal Jaworowski 20002b553caSRafal Jaworowski static int 20102b553caSRafal Jaworowski sec_probe(device_t dev) 20202b553caSRafal Jaworowski { 20302b553caSRafal Jaworowski struct sec_softc *sc; 20402b553caSRafal Jaworowski device_t parent; 20502b553caSRafal Jaworowski uintptr_t devtype; 20602b553caSRafal Jaworowski uint64_t id; 20702b553caSRafal Jaworowski int error; 20802b553caSRafal Jaworowski 20902b553caSRafal Jaworowski parent = device_get_parent(dev); 21002b553caSRafal Jaworowski error = BUS_READ_IVAR(parent, dev, OCPBUS_IVAR_DEVTYPE, &devtype); 21102b553caSRafal Jaworowski if (error) 21202b553caSRafal Jaworowski return (error); 21302b553caSRafal Jaworowski 21402b553caSRafal Jaworowski if (devtype != OCPBUS_DEVTYPE_SEC) 21502b553caSRafal Jaworowski return (ENXIO); 21602b553caSRafal Jaworowski 21702b553caSRafal Jaworowski sc = device_get_softc(dev); 21802b553caSRafal Jaworowski 21902b553caSRafal Jaworowski sc->sc_rrid = 0; 22002b553caSRafal Jaworowski sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid, 22102b553caSRafal Jaworowski 0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE); 22202b553caSRafal Jaworowski 22302b553caSRafal Jaworowski if (sc->sc_rres == NULL) 22402b553caSRafal Jaworowski return (ENXIO); 22502b553caSRafal Jaworowski 22602b553caSRafal Jaworowski sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 22702b553caSRafal Jaworowski sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 22802b553caSRafal Jaworowski 22902b553caSRafal Jaworowski id = SEC_READ(sc, SEC_ID); 23002b553caSRafal Jaworowski 23102b553caSRafal Jaworowski bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 23202b553caSRafal Jaworowski 23302b553caSRafal Jaworowski switch (id) { 23402b553caSRafal Jaworowski case SEC_20_ID: 23502b553caSRafal Jaworowski device_set_desc(dev, "Freescale Security Engine 2.0"); 23602b553caSRafal Jaworowski sc->sc_version = 2; 23702b553caSRafal Jaworowski break; 23802b553caSRafal Jaworowski case SEC_30_ID: 23902b553caSRafal Jaworowski device_set_desc(dev, "Freescale Security Engine 3.0"); 24002b553caSRafal Jaworowski sc->sc_version = 3; 24102b553caSRafal Jaworowski break; 24202b553caSRafal Jaworowski default: 24302b553caSRafal Jaworowski device_printf(dev, "unknown SEC ID 0x%016llx!\n", id); 24402b553caSRafal Jaworowski return (ENXIO); 24502b553caSRafal Jaworowski } 24602b553caSRafal Jaworowski 24702b553caSRafal Jaworowski return (0); 24802b553caSRafal Jaworowski } 24902b553caSRafal Jaworowski 25002b553caSRafal Jaworowski static int 25102b553caSRafal Jaworowski sec_attach(device_t dev) 25202b553caSRafal Jaworowski { 25302b553caSRafal Jaworowski struct sec_softc *sc; 25402b553caSRafal Jaworowski struct sec_hw_lt *lt; 25502b553caSRafal Jaworowski int error = 0; 25602b553caSRafal Jaworowski int i; 25702b553caSRafal Jaworowski 25802b553caSRafal Jaworowski sc = device_get_softc(dev); 25902b553caSRafal Jaworowski sc->sc_dev = dev; 26002b553caSRafal Jaworowski sc->sc_blocked = 0; 26102b553caSRafal Jaworowski sc->sc_shutdown = 0; 26202b553caSRafal Jaworowski 26302b553caSRafal Jaworowski sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE); 26402b553caSRafal Jaworowski if (sc->sc_cid < 0) { 26502b553caSRafal Jaworowski device_printf(dev, "could not get crypto driver ID!\n"); 26602b553caSRafal Jaworowski return (ENXIO); 26702b553caSRafal Jaworowski } 26802b553caSRafal Jaworowski 26902b553caSRafal Jaworowski /* Init locks */ 27002b553caSRafal Jaworowski mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 27102b553caSRafal Jaworowski "SEC Controller lock", MTX_DEF); 27202b553caSRafal Jaworowski mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 27302b553caSRafal Jaworowski "SEC Descriptors lock", MTX_DEF); 27402b553caSRafal Jaworowski mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 27502b553caSRafal Jaworowski "SEC Sessions lock", MTX_DEF); 27602b553caSRafal Jaworowski 27702b553caSRafal Jaworowski /* Allocate I/O memory for SEC registers */ 27802b553caSRafal Jaworowski sc->sc_rrid = 0; 27902b553caSRafal Jaworowski sc->sc_rres = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid, 28002b553caSRafal Jaworowski 0ul, ~0ul, SEC_IO_SIZE, RF_ACTIVE); 28102b553caSRafal Jaworowski 28202b553caSRafal Jaworowski if (sc->sc_rres == NULL) { 28302b553caSRafal Jaworowski device_printf(dev, "could not allocate I/O memory!\n"); 28402b553caSRafal Jaworowski goto fail1; 28502b553caSRafal Jaworowski } 28602b553caSRafal Jaworowski 28702b553caSRafal Jaworowski sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 28802b553caSRafal Jaworowski sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 28902b553caSRafal Jaworowski 29002b553caSRafal Jaworowski /* Setup interrupts */ 29102b553caSRafal Jaworowski sc->sc_pri_irid = 0; 29202b553caSRafal Jaworowski error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 29302b553caSRafal Jaworowski &sc->sc_pri_irid, sec_primary_intr, "primary"); 29402b553caSRafal Jaworowski 29502b553caSRafal Jaworowski if (error) 29602b553caSRafal Jaworowski goto fail2; 29702b553caSRafal Jaworowski 29802b553caSRafal Jaworowski sc->sc_sec_irid = 1; 29902b553caSRafal Jaworowski error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 30002b553caSRafal Jaworowski &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 30102b553caSRafal Jaworowski 30202b553caSRafal Jaworowski if (error) 30302b553caSRafal Jaworowski goto fail3; 30402b553caSRafal Jaworowski 30502b553caSRafal Jaworowski /* Alloc DMA memory for descriptors and link tables */ 30602b553caSRafal Jaworowski error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 30702b553caSRafal Jaworowski SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 30802b553caSRafal Jaworowski 30902b553caSRafal Jaworowski if (error) 31002b553caSRafal Jaworowski goto fail4; 31102b553caSRafal Jaworowski 31202b553caSRafal Jaworowski error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 31302b553caSRafal Jaworowski (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 31402b553caSRafal Jaworowski 31502b553caSRafal Jaworowski if (error) 31602b553caSRafal Jaworowski goto fail5; 31702b553caSRafal Jaworowski 31802b553caSRafal Jaworowski /* Fill in descriptors and link tables */ 31902b553caSRafal Jaworowski for (i = 0; i < SEC_DESCRIPTORS; i++) { 32002b553caSRafal Jaworowski sc->sc_desc[i].sd_desc = 32102b553caSRafal Jaworowski (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 32202b553caSRafal Jaworowski sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 32302b553caSRafal Jaworowski (i * sizeof(struct sec_hw_desc)); 32402b553caSRafal Jaworowski } 32502b553caSRafal Jaworowski 32602b553caSRafal Jaworowski for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 32702b553caSRafal Jaworowski sc->sc_lt[i].sl_lt = 32802b553caSRafal Jaworowski (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 32902b553caSRafal Jaworowski sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 33002b553caSRafal Jaworowski (i * sizeof(struct sec_hw_lt)); 33102b553caSRafal Jaworowski } 33202b553caSRafal Jaworowski 33302b553caSRafal Jaworowski /* Last entry in link table is used to create a circle */ 33402b553caSRafal Jaworowski lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 33502b553caSRafal Jaworowski lt->shl_length = 0; 33602b553caSRafal Jaworowski lt->shl_r = 0; 33702b553caSRafal Jaworowski lt->shl_n = 1; 33802b553caSRafal Jaworowski lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 33902b553caSRafal Jaworowski 34002b553caSRafal Jaworowski /* Init descriptor and link table queues pointers */ 34102b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 34202b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 34302b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 34402b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 34502b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 34602b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 34702b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 34802b553caSRafal Jaworowski SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 34902b553caSRafal Jaworowski 35002b553caSRafal Jaworowski /* Create masks for fast checks */ 35102b553caSRafal Jaworowski sc->sc_int_error_mask = 0; 35202b553caSRafal Jaworowski for (i = 0; i < SEC_CHANNELS; i++) 35302b553caSRafal Jaworowski sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 35402b553caSRafal Jaworowski 35502b553caSRafal Jaworowski switch (sc->sc_version) { 35602b553caSRafal Jaworowski case 2: 35702b553caSRafal Jaworowski sc->sc_channel_idle_mask = 35802b553caSRafal Jaworowski (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 35902b553caSRafal Jaworowski (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 36002b553caSRafal Jaworowski (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 36102b553caSRafal Jaworowski (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 36202b553caSRafal Jaworowski break; 36302b553caSRafal Jaworowski case 3: 36402b553caSRafal Jaworowski sc->sc_channel_idle_mask = 36502b553caSRafal Jaworowski (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 36602b553caSRafal Jaworowski (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 36702b553caSRafal Jaworowski (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 36802b553caSRafal Jaworowski (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 36902b553caSRafal Jaworowski break; 37002b553caSRafal Jaworowski } 37102b553caSRafal Jaworowski 37202b553caSRafal Jaworowski /* Init hardware */ 37302b553caSRafal Jaworowski error = sec_init(sc); 37402b553caSRafal Jaworowski 37502b553caSRafal Jaworowski if (error) 37602b553caSRafal Jaworowski goto fail6; 37702b553caSRafal Jaworowski 37802b553caSRafal Jaworowski /* Register in OCF (AESU) */ 37902b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 38002b553caSRafal Jaworowski 38102b553caSRafal Jaworowski /* Register in OCF (DEU) */ 38202b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 38302b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 38402b553caSRafal Jaworowski 38502b553caSRafal Jaworowski /* Register in OCF (MDEU) */ 38602b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 38702b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 38802b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 38902b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 39002b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 39102b553caSRafal Jaworowski if (sc->sc_version >= 3) { 39202b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0); 39302b553caSRafal Jaworowski crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0); 39402b553caSRafal Jaworowski } 39502b553caSRafal Jaworowski 39602b553caSRafal Jaworowski return (0); 39702b553caSRafal Jaworowski 39802b553caSRafal Jaworowski fail6: 39902b553caSRafal Jaworowski sec_free_dma_mem(&(sc->sc_lt_dmem)); 40002b553caSRafal Jaworowski fail5: 40102b553caSRafal Jaworowski sec_free_dma_mem(&(sc->sc_desc_dmem)); 40202b553caSRafal Jaworowski fail4: 40302b553caSRafal Jaworowski sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 40402b553caSRafal Jaworowski sc->sc_sec_irid, "secondary"); 40502b553caSRafal Jaworowski fail3: 40602b553caSRafal Jaworowski sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 40702b553caSRafal Jaworowski sc->sc_pri_irid, "primary"); 40802b553caSRafal Jaworowski fail2: 40902b553caSRafal Jaworowski bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 41002b553caSRafal Jaworowski fail1: 41102b553caSRafal Jaworowski mtx_destroy(&sc->sc_controller_lock); 41202b553caSRafal Jaworowski mtx_destroy(&sc->sc_descriptors_lock); 41302b553caSRafal Jaworowski mtx_destroy(&sc->sc_sessions_lock); 41402b553caSRafal Jaworowski 41502b553caSRafal Jaworowski return (ENXIO); 41602b553caSRafal Jaworowski } 41702b553caSRafal Jaworowski 41802b553caSRafal Jaworowski static int 41902b553caSRafal Jaworowski sec_detach(device_t dev) 42002b553caSRafal Jaworowski { 42102b553caSRafal Jaworowski struct sec_softc *sc = device_get_softc(dev); 42202b553caSRafal Jaworowski int i, error, timeout = SEC_TIMEOUT; 42302b553caSRafal Jaworowski 42402b553caSRafal Jaworowski /* Prepare driver to shutdown */ 42502b553caSRafal Jaworowski SEC_LOCK(sc, descriptors); 42602b553caSRafal Jaworowski sc->sc_shutdown = 1; 42702b553caSRafal Jaworowski SEC_UNLOCK(sc, descriptors); 42802b553caSRafal Jaworowski 42902b553caSRafal Jaworowski /* Wait until all queued processing finishes */ 43002b553caSRafal Jaworowski while (1) { 43102b553caSRafal Jaworowski SEC_LOCK(sc, descriptors); 43202b553caSRafal Jaworowski i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 43302b553caSRafal Jaworowski SEC_UNLOCK(sc, descriptors); 43402b553caSRafal Jaworowski 43502b553caSRafal Jaworowski if (i == 0) 43602b553caSRafal Jaworowski break; 43702b553caSRafal Jaworowski 43802b553caSRafal Jaworowski if (timeout < 0) { 43902b553caSRafal Jaworowski device_printf(dev, "queue flush timeout!\n"); 44002b553caSRafal Jaworowski 44102b553caSRafal Jaworowski /* DMA can be still active - stop it */ 44202b553caSRafal Jaworowski for (i = 0; i < SEC_CHANNELS; i++) 44302b553caSRafal Jaworowski sec_channel_reset(sc, i, 1); 44402b553caSRafal Jaworowski 44502b553caSRafal Jaworowski break; 44602b553caSRafal Jaworowski } 44702b553caSRafal Jaworowski 44802b553caSRafal Jaworowski timeout -= 1000; 44902b553caSRafal Jaworowski DELAY(1000); 45002b553caSRafal Jaworowski } 45102b553caSRafal Jaworowski 45202b553caSRafal Jaworowski /* Disable interrupts */ 45302b553caSRafal Jaworowski SEC_WRITE(sc, SEC_IER, 0); 45402b553caSRafal Jaworowski 45502b553caSRafal Jaworowski /* Unregister from OCF */ 45602b553caSRafal Jaworowski crypto_unregister_all(sc->sc_cid); 45702b553caSRafal Jaworowski 45802b553caSRafal Jaworowski /* Free DMA memory */ 45902b553caSRafal Jaworowski for (i = 0; i < SEC_DESCRIPTORS; i++) 46002b553caSRafal Jaworowski SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 46102b553caSRafal Jaworowski 46202b553caSRafal Jaworowski sec_free_dma_mem(&(sc->sc_lt_dmem)); 46302b553caSRafal Jaworowski sec_free_dma_mem(&(sc->sc_desc_dmem)); 46402b553caSRafal Jaworowski 46502b553caSRafal Jaworowski /* Release interrupts */ 46602b553caSRafal Jaworowski sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 46702b553caSRafal Jaworowski sc->sc_pri_irid, "primary"); 46802b553caSRafal Jaworowski sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 46902b553caSRafal Jaworowski sc->sc_sec_irid, "secondary"); 47002b553caSRafal Jaworowski 47102b553caSRafal Jaworowski /* Release memory */ 47202b553caSRafal Jaworowski if (sc->sc_rres) { 47302b553caSRafal Jaworowski error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 47402b553caSRafal Jaworowski sc->sc_rres); 47502b553caSRafal Jaworowski if (error) 47602b553caSRafal Jaworowski device_printf(dev, "bus_release_resource() failed for" 47702b553caSRafal Jaworowski " I/O memory, error %d\n", error); 47802b553caSRafal Jaworowski 47902b553caSRafal Jaworowski sc->sc_rres = NULL; 48002b553caSRafal Jaworowski } 48102b553caSRafal Jaworowski 48202b553caSRafal Jaworowski mtx_destroy(&sc->sc_controller_lock); 48302b553caSRafal Jaworowski mtx_destroy(&sc->sc_descriptors_lock); 48402b553caSRafal Jaworowski mtx_destroy(&sc->sc_sessions_lock); 48502b553caSRafal Jaworowski 48602b553caSRafal Jaworowski return (0); 48702b553caSRafal Jaworowski } 48802b553caSRafal Jaworowski 48902b553caSRafal Jaworowski static int 49002b553caSRafal Jaworowski sec_suspend(device_t dev) 49102b553caSRafal Jaworowski { 49202b553caSRafal Jaworowski 49302b553caSRafal Jaworowski return (0); 49402b553caSRafal Jaworowski } 49502b553caSRafal Jaworowski 49602b553caSRafal Jaworowski static int 49702b553caSRafal Jaworowski sec_resume(device_t dev) 49802b553caSRafal Jaworowski { 49902b553caSRafal Jaworowski 50002b553caSRafal Jaworowski return (0); 50102b553caSRafal Jaworowski } 50202b553caSRafal Jaworowski 503661ee6eeSRafal Jaworowski static int 50402b553caSRafal Jaworowski sec_shutdown(device_t dev) 50502b553caSRafal Jaworowski { 506661ee6eeSRafal Jaworowski 507661ee6eeSRafal Jaworowski return (0); 50802b553caSRafal Jaworowski } 50902b553caSRafal Jaworowski 51002b553caSRafal Jaworowski static int 51102b553caSRafal Jaworowski sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 51202b553caSRafal Jaworowski int *irid, driver_intr_t handler, const char *iname) 51302b553caSRafal Jaworowski { 51402b553caSRafal Jaworowski int error; 51502b553caSRafal Jaworowski 51602b553caSRafal Jaworowski (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 51702b553caSRafal Jaworowski RF_ACTIVE); 51802b553caSRafal Jaworowski 51902b553caSRafal Jaworowski if ((*ires) == NULL) { 52002b553caSRafal Jaworowski device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 52102b553caSRafal Jaworowski return (ENXIO); 52202b553caSRafal Jaworowski } 52302b553caSRafal Jaworowski 52402b553caSRafal Jaworowski error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 52502b553caSRafal Jaworowski NULL, handler, sc, ihand); 52602b553caSRafal Jaworowski 52702b553caSRafal Jaworowski if (error) { 52802b553caSRafal Jaworowski device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 52902b553caSRafal Jaworowski if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 53002b553caSRafal Jaworowski device_printf(sc->sc_dev, "could not release %s IRQ\n", 53102b553caSRafal Jaworowski iname); 53202b553caSRafal Jaworowski 53302b553caSRafal Jaworowski (*ires) = NULL; 53402b553caSRafal Jaworowski return (error); 53502b553caSRafal Jaworowski } 53602b553caSRafal Jaworowski 53702b553caSRafal Jaworowski return (0); 53802b553caSRafal Jaworowski } 53902b553caSRafal Jaworowski 54002b553caSRafal Jaworowski static void 54102b553caSRafal Jaworowski sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 54202b553caSRafal Jaworowski int irid, const char *iname) 54302b553caSRafal Jaworowski { 54402b553caSRafal Jaworowski int error; 54502b553caSRafal Jaworowski 54602b553caSRafal Jaworowski if (ires == NULL) 54702b553caSRafal Jaworowski return; 54802b553caSRafal Jaworowski 54902b553caSRafal Jaworowski error = bus_teardown_intr(sc->sc_dev, ires, ihand); 55002b553caSRafal Jaworowski if (error) 55102b553caSRafal Jaworowski device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 55202b553caSRafal Jaworowski " IRQ, error %d\n", iname, error); 55302b553caSRafal Jaworowski 55402b553caSRafal Jaworowski error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 55502b553caSRafal Jaworowski if (error) 55602b553caSRafal Jaworowski device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 55702b553caSRafal Jaworowski " IRQ, error %d\n", iname, error); 55802b553caSRafal Jaworowski } 55902b553caSRafal Jaworowski 56002b553caSRafal Jaworowski static void 56102b553caSRafal Jaworowski sec_primary_intr(void *arg) 56202b553caSRafal Jaworowski { 56302b553caSRafal Jaworowski struct sec_softc *sc = arg; 56402b553caSRafal Jaworowski struct sec_desc *desc; 56502b553caSRafal Jaworowski uint64_t isr; 56602b553caSRafal Jaworowski int i, wakeup = 0; 56702b553caSRafal Jaworowski 56802b553caSRafal Jaworowski SEC_LOCK(sc, controller); 56902b553caSRafal Jaworowski 57002b553caSRafal Jaworowski /* Check for errors */ 57102b553caSRafal Jaworowski isr = SEC_READ(sc, SEC_ISR); 57202b553caSRafal Jaworowski if (isr & sc->sc_int_error_mask) { 57302b553caSRafal Jaworowski /* Check each channel for error */ 57402b553caSRafal Jaworowski for (i = 0; i < SEC_CHANNELS; i++) { 57502b553caSRafal Jaworowski if ((isr & SEC_INT_CH_ERR(i)) == 0) 57602b553caSRafal Jaworowski continue; 57702b553caSRafal Jaworowski 57802b553caSRafal Jaworowski device_printf(sc->sc_dev, 57902b553caSRafal Jaworowski "I/O error on channel %i!\n", i); 58002b553caSRafal Jaworowski 58102b553caSRafal Jaworowski /* Find and mark problematic descriptor */ 58202b553caSRafal Jaworowski desc = sec_find_desc(sc, SEC_READ(sc, 58302b553caSRafal Jaworowski SEC_CHAN_CDPR(i))); 58402b553caSRafal Jaworowski 58502b553caSRafal Jaworowski if (desc != NULL) 58602b553caSRafal Jaworowski desc->sd_error = EIO; 58702b553caSRafal Jaworowski 58802b553caSRafal Jaworowski /* Do partial channel reset */ 58902b553caSRafal Jaworowski sec_channel_reset(sc, i, 0); 59002b553caSRafal Jaworowski } 59102b553caSRafal Jaworowski } 59202b553caSRafal Jaworowski 59302b553caSRafal Jaworowski /* ACK interrupt */ 59402b553caSRafal Jaworowski SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 59502b553caSRafal Jaworowski 59602b553caSRafal Jaworowski SEC_UNLOCK(sc, controller); 59702b553caSRafal Jaworowski SEC_LOCK(sc, descriptors); 59802b553caSRafal Jaworowski 59902b553caSRafal Jaworowski /* Handle processed descriptors */ 60002b553caSRafal Jaworowski SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 60102b553caSRafal Jaworowski 60202b553caSRafal Jaworowski while (SEC_QUEUED_DESC_CNT(sc) > 0) { 60302b553caSRafal Jaworowski desc = SEC_GET_QUEUED_DESC(sc); 60402b553caSRafal Jaworowski 60502b553caSRafal Jaworowski if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 60602b553caSRafal Jaworowski SEC_PUT_BACK_QUEUED_DESC(sc); 60702b553caSRafal Jaworowski break; 60802b553caSRafal Jaworowski } 60902b553caSRafal Jaworowski 61002b553caSRafal Jaworowski SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 61102b553caSRafal Jaworowski BUS_DMASYNC_PREWRITE); 61202b553caSRafal Jaworowski 61302b553caSRafal Jaworowski desc->sd_crp->crp_etype = desc->sd_error; 61402b553caSRafal Jaworowski crypto_done(desc->sd_crp); 61502b553caSRafal Jaworowski 61602b553caSRafal Jaworowski SEC_DESC_FREE_POINTERS(desc); 61702b553caSRafal Jaworowski SEC_DESC_FREE_LT(sc, desc); 61802b553caSRafal Jaworowski SEC_DESC_QUEUED2FREE(sc); 61902b553caSRafal Jaworowski } 62002b553caSRafal Jaworowski 62102b553caSRafal Jaworowski SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 62202b553caSRafal Jaworowski 62302b553caSRafal Jaworowski if (!sc->sc_shutdown) { 62402b553caSRafal Jaworowski wakeup = sc->sc_blocked; 62502b553caSRafal Jaworowski sc->sc_blocked = 0; 62602b553caSRafal Jaworowski } 62702b553caSRafal Jaworowski 62802b553caSRafal Jaworowski SEC_UNLOCK(sc, descriptors); 62902b553caSRafal Jaworowski 63002b553caSRafal Jaworowski /* Enqueue ready descriptors in hardware */ 63102b553caSRafal Jaworowski sec_enqueue(sc); 63202b553caSRafal Jaworowski 63302b553caSRafal Jaworowski if (wakeup) 63402b553caSRafal Jaworowski crypto_unblock(sc->sc_cid, wakeup); 63502b553caSRafal Jaworowski } 63602b553caSRafal Jaworowski 63702b553caSRafal Jaworowski static void 63802b553caSRafal Jaworowski sec_secondary_intr(void *arg) 63902b553caSRafal Jaworowski { 64002b553caSRafal Jaworowski struct sec_softc *sc = arg; 64102b553caSRafal Jaworowski 64202b553caSRafal Jaworowski device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 64302b553caSRafal Jaworowski sec_primary_intr(arg); 64402b553caSRafal Jaworowski } 64502b553caSRafal Jaworowski 64602b553caSRafal Jaworowski static int 64702b553caSRafal Jaworowski sec_controller_reset(struct sec_softc *sc) 64802b553caSRafal Jaworowski { 64902b553caSRafal Jaworowski int timeout = SEC_TIMEOUT; 65002b553caSRafal Jaworowski 65102b553caSRafal Jaworowski /* Reset Controller */ 65202b553caSRafal Jaworowski SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 65302b553caSRafal Jaworowski 65402b553caSRafal Jaworowski while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 65502b553caSRafal Jaworowski DELAY(1000); 65602b553caSRafal Jaworowski timeout -= 1000; 65702b553caSRafal Jaworowski 65802b553caSRafal Jaworowski if (timeout < 0) { 65902b553caSRafal Jaworowski device_printf(sc->sc_dev, "timeout while waiting for " 66002b553caSRafal Jaworowski "device reset!\n"); 66102b553caSRafal Jaworowski return (ETIMEDOUT); 66202b553caSRafal Jaworowski } 66302b553caSRafal Jaworowski } 66402b553caSRafal Jaworowski 66502b553caSRafal Jaworowski return (0); 66602b553caSRafal Jaworowski } 66702b553caSRafal Jaworowski 66802b553caSRafal Jaworowski static int 66902b553caSRafal Jaworowski sec_channel_reset(struct sec_softc *sc, int channel, int full) 67002b553caSRafal Jaworowski { 67102b553caSRafal Jaworowski int timeout = SEC_TIMEOUT; 67202b553caSRafal Jaworowski uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 67302b553caSRafal Jaworowski uint64_t reg; 67402b553caSRafal Jaworowski 67502b553caSRafal Jaworowski /* Reset Channel */ 67602b553caSRafal Jaworowski reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 67702b553caSRafal Jaworowski SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 67802b553caSRafal Jaworowski 67902b553caSRafal Jaworowski while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 68002b553caSRafal Jaworowski DELAY(1000); 68102b553caSRafal Jaworowski timeout -= 1000; 68202b553caSRafal Jaworowski 68302b553caSRafal Jaworowski if (timeout < 0) { 68402b553caSRafal Jaworowski device_printf(sc->sc_dev, "timeout while waiting for " 68502b553caSRafal Jaworowski "channel reset!\n"); 68602b553caSRafal Jaworowski return (ETIMEDOUT); 68702b553caSRafal Jaworowski } 68802b553caSRafal Jaworowski } 68902b553caSRafal Jaworowski 69002b553caSRafal Jaworowski if (full) { 69102b553caSRafal Jaworowski reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 69202b553caSRafal Jaworowski 69302b553caSRafal Jaworowski switch(sc->sc_version) { 69402b553caSRafal Jaworowski case 2: 69502b553caSRafal Jaworowski reg |= SEC_CHAN_CCR_CDWE; 69602b553caSRafal Jaworowski break; 69702b553caSRafal Jaworowski case 3: 69802b553caSRafal Jaworowski reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 69902b553caSRafal Jaworowski break; 70002b553caSRafal Jaworowski } 70102b553caSRafal Jaworowski 70202b553caSRafal Jaworowski SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 70302b553caSRafal Jaworowski } 70402b553caSRafal Jaworowski 70502b553caSRafal Jaworowski return (0); 70602b553caSRafal Jaworowski } 70702b553caSRafal Jaworowski 70802b553caSRafal Jaworowski static int 70902b553caSRafal Jaworowski sec_init(struct sec_softc *sc) 71002b553caSRafal Jaworowski { 71102b553caSRafal Jaworowski uint64_t reg; 71202b553caSRafal Jaworowski int error, i; 71302b553caSRafal Jaworowski 71402b553caSRafal Jaworowski /* Reset controller twice to clear all pending interrupts */ 71502b553caSRafal Jaworowski error = sec_controller_reset(sc); 71602b553caSRafal Jaworowski if (error) 71702b553caSRafal Jaworowski return (error); 71802b553caSRafal Jaworowski 71902b553caSRafal Jaworowski error = sec_controller_reset(sc); 72002b553caSRafal Jaworowski if (error) 72102b553caSRafal Jaworowski return (error); 72202b553caSRafal Jaworowski 72302b553caSRafal Jaworowski /* Reset channels */ 72402b553caSRafal Jaworowski for (i = 0; i < SEC_CHANNELS; i++) { 72502b553caSRafal Jaworowski error = sec_channel_reset(sc, i, 1); 72602b553caSRafal Jaworowski if (error) 72702b553caSRafal Jaworowski return (error); 72802b553caSRafal Jaworowski } 72902b553caSRafal Jaworowski 73002b553caSRafal Jaworowski /* Enable Interrupts */ 73102b553caSRafal Jaworowski reg = SEC_INT_ITO; 73202b553caSRafal Jaworowski for (i = 0; i < SEC_CHANNELS; i++) 73302b553caSRafal Jaworowski reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 73402b553caSRafal Jaworowski 73502b553caSRafal Jaworowski SEC_WRITE(sc, SEC_IER, reg); 73602b553caSRafal Jaworowski 73702b553caSRafal Jaworowski return (error); 73802b553caSRafal Jaworowski } 73902b553caSRafal Jaworowski 74002b553caSRafal Jaworowski static void 74102b553caSRafal Jaworowski sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 74202b553caSRafal Jaworowski { 74302b553caSRafal Jaworowski struct sec_dma_mem *dma_mem = arg; 74402b553caSRafal Jaworowski 74502b553caSRafal Jaworowski if (error) 74602b553caSRafal Jaworowski return; 74702b553caSRafal Jaworowski 74802b553caSRafal Jaworowski KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 74902b553caSRafal Jaworowski dma_mem->dma_paddr = segs->ds_addr; 75002b553caSRafal Jaworowski } 75102b553caSRafal Jaworowski 75202b553caSRafal Jaworowski static void 75302b553caSRafal Jaworowski sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 75402b553caSRafal Jaworowski int error) 75502b553caSRafal Jaworowski { 75602b553caSRafal Jaworowski struct sec_desc_map_info *sdmi = arg; 75702b553caSRafal Jaworowski struct sec_softc *sc = sdmi->sdmi_sc; 75802b553caSRafal Jaworowski struct sec_lt *lt = NULL; 75902b553caSRafal Jaworowski bus_addr_t addr; 76002b553caSRafal Jaworowski bus_size_t size; 76102b553caSRafal Jaworowski int i; 76202b553caSRafal Jaworowski 76302b553caSRafal Jaworowski SEC_LOCK_ASSERT(sc, descriptors); 76402b553caSRafal Jaworowski 76502b553caSRafal Jaworowski if (error) 76602b553caSRafal Jaworowski return; 76702b553caSRafal Jaworowski 76802b553caSRafal Jaworowski for (i = 0; i < nseg; i++) { 76902b553caSRafal Jaworowski addr = segs[i].ds_addr; 77002b553caSRafal Jaworowski size = segs[i].ds_len; 77102b553caSRafal Jaworowski 77202b553caSRafal Jaworowski /* Skip requested offset */ 77302b553caSRafal Jaworowski if (sdmi->sdmi_offset >= size) { 77402b553caSRafal Jaworowski sdmi->sdmi_offset -= size; 77502b553caSRafal Jaworowski continue; 77602b553caSRafal Jaworowski } 77702b553caSRafal Jaworowski 77802b553caSRafal Jaworowski addr += sdmi->sdmi_offset; 77902b553caSRafal Jaworowski size -= sdmi->sdmi_offset; 78002b553caSRafal Jaworowski sdmi->sdmi_offset = 0; 78102b553caSRafal Jaworowski 78202b553caSRafal Jaworowski /* Do not link more than requested */ 78302b553caSRafal Jaworowski if (sdmi->sdmi_size < size) 78402b553caSRafal Jaworowski size = sdmi->sdmi_size; 78502b553caSRafal Jaworowski 78602b553caSRafal Jaworowski lt = SEC_ALLOC_LT_ENTRY(sc); 78702b553caSRafal Jaworowski lt->sl_lt->shl_length = size; 78802b553caSRafal Jaworowski lt->sl_lt->shl_r = 0; 78902b553caSRafal Jaworowski lt->sl_lt->shl_n = 0; 79002b553caSRafal Jaworowski lt->sl_lt->shl_ptr = addr; 79102b553caSRafal Jaworowski 79202b553caSRafal Jaworowski if (sdmi->sdmi_lt_first == NULL) 79302b553caSRafal Jaworowski sdmi->sdmi_lt_first = lt; 79402b553caSRafal Jaworowski 79502b553caSRafal Jaworowski sdmi->sdmi_lt_used += 1; 79602b553caSRafal Jaworowski 79702b553caSRafal Jaworowski if ((sdmi->sdmi_size -= size) == 0) 79802b553caSRafal Jaworowski break; 79902b553caSRafal Jaworowski } 80002b553caSRafal Jaworowski 80102b553caSRafal Jaworowski sdmi->sdmi_lt_last = lt; 80202b553caSRafal Jaworowski } 80302b553caSRafal Jaworowski 80402b553caSRafal Jaworowski static void 80502b553caSRafal Jaworowski sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 80602b553caSRafal Jaworowski bus_size_t size, int error) 80702b553caSRafal Jaworowski { 80802b553caSRafal Jaworowski 80902b553caSRafal Jaworowski sec_dma_map_desc_cb(arg, segs, nseg, error); 81002b553caSRafal Jaworowski } 81102b553caSRafal Jaworowski 81202b553caSRafal Jaworowski static int 81302b553caSRafal Jaworowski sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 81402b553caSRafal Jaworowski bus_size_t size) 81502b553caSRafal Jaworowski { 81602b553caSRafal Jaworowski int error; 81702b553caSRafal Jaworowski 81802b553caSRafal Jaworowski if (dma_mem->dma_vaddr != NULL) 81902b553caSRafal Jaworowski return (EBUSY); 82002b553caSRafal Jaworowski 82102b553caSRafal Jaworowski error = bus_dma_tag_create(NULL, /* parent */ 82202b553caSRafal Jaworowski SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 82302b553caSRafal Jaworowski BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 82402b553caSRafal Jaworowski BUS_SPACE_MAXADDR, /* highaddr */ 82502b553caSRafal Jaworowski NULL, NULL, /* filtfunc, filtfuncarg */ 82602b553caSRafal Jaworowski size, 1, /* maxsize, nsegments */ 82702b553caSRafal Jaworowski size, 0, /* maxsegsz, flags */ 82802b553caSRafal Jaworowski NULL, NULL, /* lockfunc, lockfuncarg */ 82902b553caSRafal Jaworowski &(dma_mem->dma_tag)); /* dmat */ 83002b553caSRafal Jaworowski 83102b553caSRafal Jaworowski if (error) { 83202b553caSRafal Jaworowski device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 83302b553caSRafal Jaworowski " %i!\n", error); 83402b553caSRafal Jaworowski goto err1; 83502b553caSRafal Jaworowski } 83602b553caSRafal Jaworowski 83702b553caSRafal Jaworowski error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 83802b553caSRafal Jaworowski BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 83902b553caSRafal Jaworowski 84002b553caSRafal Jaworowski if (error) { 84102b553caSRafal Jaworowski device_printf(sc->sc_dev, "failed to allocate DMA safe" 84202b553caSRafal Jaworowski " memory, error %i!\n", error); 84302b553caSRafal Jaworowski goto err2; 84402b553caSRafal Jaworowski } 84502b553caSRafal Jaworowski 84602b553caSRafal Jaworowski error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 84702b553caSRafal Jaworowski dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 84802b553caSRafal Jaworowski BUS_DMA_NOWAIT); 84902b553caSRafal Jaworowski 85002b553caSRafal Jaworowski if (error) { 85102b553caSRafal Jaworowski device_printf(sc->sc_dev, "cannot get address of the DMA" 85202b553caSRafal Jaworowski " memory, error %i\n", error); 85302b553caSRafal Jaworowski goto err3; 85402b553caSRafal Jaworowski } 85502b553caSRafal Jaworowski 85602b553caSRafal Jaworowski dma_mem->dma_is_map = 0; 85702b553caSRafal Jaworowski return (0); 85802b553caSRafal Jaworowski 85902b553caSRafal Jaworowski err3: 86002b553caSRafal Jaworowski bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 86102b553caSRafal Jaworowski err2: 86202b553caSRafal Jaworowski bus_dma_tag_destroy(dma_mem->dma_tag); 86302b553caSRafal Jaworowski err1: 86402b553caSRafal Jaworowski dma_mem->dma_vaddr = NULL; 86502b553caSRafal Jaworowski return(error); 86602b553caSRafal Jaworowski } 86702b553caSRafal Jaworowski 86802b553caSRafal Jaworowski static int 86902b553caSRafal Jaworowski sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem, 87002b553caSRafal Jaworowski bus_size_t size, int type, struct sec_desc_map_info *sdmi) 87102b553caSRafal Jaworowski { 87202b553caSRafal Jaworowski int error; 87302b553caSRafal Jaworowski 87402b553caSRafal Jaworowski if (dma_mem->dma_vaddr != NULL) 87502b553caSRafal Jaworowski return (EBUSY); 87602b553caSRafal Jaworowski 87702b553caSRafal Jaworowski switch (type) { 87802b553caSRafal Jaworowski case SEC_MEMORY: 87902b553caSRafal Jaworowski break; 88002b553caSRafal Jaworowski case SEC_UIO: 88102b553caSRafal Jaworowski size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 88202b553caSRafal Jaworowski break; 88302b553caSRafal Jaworowski case SEC_MBUF: 88402b553caSRafal Jaworowski size = m_length((struct mbuf*)mem, NULL); 88502b553caSRafal Jaworowski break; 88602b553caSRafal Jaworowski default: 88702b553caSRafal Jaworowski return (EINVAL); 88802b553caSRafal Jaworowski } 88902b553caSRafal Jaworowski 89002b553caSRafal Jaworowski error = bus_dma_tag_create(NULL, /* parent */ 89102b553caSRafal Jaworowski SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 89202b553caSRafal Jaworowski BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 89302b553caSRafal Jaworowski BUS_SPACE_MAXADDR, /* highaddr */ 89402b553caSRafal Jaworowski NULL, NULL, /* filtfunc, filtfuncarg */ 89502b553caSRafal Jaworowski size, /* maxsize */ 89602b553caSRafal Jaworowski SEC_FREE_LT_CNT(sc), /* nsegments */ 89702b553caSRafal Jaworowski SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 89802b553caSRafal Jaworowski NULL, NULL, /* lockfunc, lockfuncarg */ 89902b553caSRafal Jaworowski &(dma_mem->dma_tag)); /* dmat */ 90002b553caSRafal Jaworowski 90102b553caSRafal Jaworowski if (error) { 90202b553caSRafal Jaworowski device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 90302b553caSRafal Jaworowski " %i!\n", error); 90402b553caSRafal Jaworowski dma_mem->dma_vaddr = NULL; 90502b553caSRafal Jaworowski return (error); 90602b553caSRafal Jaworowski } 90702b553caSRafal Jaworowski 90802b553caSRafal Jaworowski error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 90902b553caSRafal Jaworowski 91002b553caSRafal Jaworowski if (error) { 91102b553caSRafal Jaworowski device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 91202b553caSRafal Jaworowski "\n", error); 91302b553caSRafal Jaworowski bus_dma_tag_destroy(dma_mem->dma_tag); 91402b553caSRafal Jaworowski return (error); 91502b553caSRafal Jaworowski } 91602b553caSRafal Jaworowski 91702b553caSRafal Jaworowski switch (type) { 91802b553caSRafal Jaworowski case SEC_MEMORY: 91902b553caSRafal Jaworowski error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 92002b553caSRafal Jaworowski mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 92102b553caSRafal Jaworowski break; 92202b553caSRafal Jaworowski case SEC_UIO: 92302b553caSRafal Jaworowski error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map, 92402b553caSRafal Jaworowski mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 92502b553caSRafal Jaworowski break; 92602b553caSRafal Jaworowski case SEC_MBUF: 92702b553caSRafal Jaworowski error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map, 92802b553caSRafal Jaworowski mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT); 92902b553caSRafal Jaworowski break; 93002b553caSRafal Jaworowski } 93102b553caSRafal Jaworowski 93202b553caSRafal Jaworowski if (error) { 93302b553caSRafal Jaworowski device_printf(sc->sc_dev, "cannot get address of the DMA" 93402b553caSRafal Jaworowski " memory, error %i!\n", error); 93502b553caSRafal Jaworowski bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 93602b553caSRafal Jaworowski bus_dma_tag_destroy(dma_mem->dma_tag); 93702b553caSRafal Jaworowski return (error); 93802b553caSRafal Jaworowski } 93902b553caSRafal Jaworowski 94002b553caSRafal Jaworowski dma_mem->dma_is_map = 1; 94102b553caSRafal Jaworowski dma_mem->dma_vaddr = mem; 94202b553caSRafal Jaworowski 94302b553caSRafal Jaworowski return (0); 94402b553caSRafal Jaworowski } 94502b553caSRafal Jaworowski 94602b553caSRafal Jaworowski static void 94702b553caSRafal Jaworowski sec_free_dma_mem(struct sec_dma_mem *dma_mem) 94802b553caSRafal Jaworowski { 94902b553caSRafal Jaworowski 95002b553caSRafal Jaworowski /* Check for double free */ 95102b553caSRafal Jaworowski if (dma_mem->dma_vaddr == NULL) 95202b553caSRafal Jaworowski return; 95302b553caSRafal Jaworowski 95402b553caSRafal Jaworowski bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 95502b553caSRafal Jaworowski 95602b553caSRafal Jaworowski if (dma_mem->dma_is_map) 95702b553caSRafal Jaworowski bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 95802b553caSRafal Jaworowski else 95902b553caSRafal Jaworowski bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 96002b553caSRafal Jaworowski dma_mem->dma_map); 96102b553caSRafal Jaworowski 96202b553caSRafal Jaworowski bus_dma_tag_destroy(dma_mem->dma_tag); 96302b553caSRafal Jaworowski dma_mem->dma_vaddr = NULL; 96402b553caSRafal Jaworowski } 96502b553caSRafal Jaworowski 96602b553caSRafal Jaworowski static int 96702b553caSRafal Jaworowski sec_eu_channel(struct sec_softc *sc, int eu) 96802b553caSRafal Jaworowski { 96902b553caSRafal Jaworowski uint64_t reg; 97002b553caSRafal Jaworowski int channel = 0; 97102b553caSRafal Jaworowski 97202b553caSRafal Jaworowski SEC_LOCK_ASSERT(sc, controller); 97302b553caSRafal Jaworowski 97402b553caSRafal Jaworowski reg = SEC_READ(sc, SEC_EUASR); 97502b553caSRafal Jaworowski 97602b553caSRafal Jaworowski switch (eu) { 97702b553caSRafal Jaworowski case SEC_EU_AFEU: 97802b553caSRafal Jaworowski channel = SEC_EUASR_AFEU(reg); 97902b553caSRafal Jaworowski break; 98002b553caSRafal Jaworowski case SEC_EU_DEU: 98102b553caSRafal Jaworowski channel = SEC_EUASR_DEU(reg); 98202b553caSRafal Jaworowski break; 98302b553caSRafal Jaworowski case SEC_EU_MDEU_A: 98402b553caSRafal Jaworowski case SEC_EU_MDEU_B: 98502b553caSRafal Jaworowski channel = SEC_EUASR_MDEU(reg); 98602b553caSRafal Jaworowski break; 98702b553caSRafal Jaworowski case SEC_EU_RNGU: 98802b553caSRafal Jaworowski channel = SEC_EUASR_RNGU(reg); 98902b553caSRafal Jaworowski break; 99002b553caSRafal Jaworowski case SEC_EU_PKEU: 99102b553caSRafal Jaworowski channel = SEC_EUASR_PKEU(reg); 99202b553caSRafal Jaworowski break; 99302b553caSRafal Jaworowski case SEC_EU_AESU: 99402b553caSRafal Jaworowski channel = SEC_EUASR_AESU(reg); 99502b553caSRafal Jaworowski break; 99602b553caSRafal Jaworowski case SEC_EU_KEU: 99702b553caSRafal Jaworowski channel = SEC_EUASR_KEU(reg); 99802b553caSRafal Jaworowski break; 99902b553caSRafal Jaworowski case SEC_EU_CRCU: 100002b553caSRafal Jaworowski channel = SEC_EUASR_CRCU(reg); 100102b553caSRafal Jaworowski break; 100202b553caSRafal Jaworowski } 100302b553caSRafal Jaworowski 100402b553caSRafal Jaworowski return (channel - 1); 100502b553caSRafal Jaworowski } 100602b553caSRafal Jaworowski 100702b553caSRafal Jaworowski static int 100802b553caSRafal Jaworowski sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 100902b553caSRafal Jaworowski { 101002b553caSRafal Jaworowski u_int fflvl = SEC_MAX_FIFO_LEVEL; 101102b553caSRafal Jaworowski uint64_t reg; 101202b553caSRafal Jaworowski int i; 101302b553caSRafal Jaworowski 101402b553caSRafal Jaworowski SEC_LOCK_ASSERT(sc, controller); 101502b553caSRafal Jaworowski 101602b553caSRafal Jaworowski /* Find free channel if have not got one */ 101702b553caSRafal Jaworowski if (channel < 0) { 101802b553caSRafal Jaworowski for (i = 0; i < SEC_CHANNELS; i++) { 101902b553caSRafal Jaworowski reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 102002b553caSRafal Jaworowski 102102b553caSRafal Jaworowski if ((reg & sc->sc_channel_idle_mask) == 0) { 102202b553caSRafal Jaworowski channel = i; 102302b553caSRafal Jaworowski break; 102402b553caSRafal Jaworowski } 102502b553caSRafal Jaworowski } 102602b553caSRafal Jaworowski } 102702b553caSRafal Jaworowski 102802b553caSRafal Jaworowski /* There is no free channel */ 102902b553caSRafal Jaworowski if (channel < 0) 103002b553caSRafal Jaworowski return (-1); 103102b553caSRafal Jaworowski 103202b553caSRafal Jaworowski /* Check FIFO level on selected channel */ 103302b553caSRafal Jaworowski reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 103402b553caSRafal Jaworowski 103502b553caSRafal Jaworowski switch(sc->sc_version) { 103602b553caSRafal Jaworowski case 2: 103702b553caSRafal Jaworowski fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 103802b553caSRafal Jaworowski break; 103902b553caSRafal Jaworowski case 3: 104002b553caSRafal Jaworowski fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 104102b553caSRafal Jaworowski break; 104202b553caSRafal Jaworowski } 104302b553caSRafal Jaworowski 104402b553caSRafal Jaworowski if (fflvl >= SEC_MAX_FIFO_LEVEL) 104502b553caSRafal Jaworowski return (-1); 104602b553caSRafal Jaworowski 104702b553caSRafal Jaworowski /* Enqueue descriptor in channel */ 104802b553caSRafal Jaworowski SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 104902b553caSRafal Jaworowski 105002b553caSRafal Jaworowski return (channel); 105102b553caSRafal Jaworowski } 105202b553caSRafal Jaworowski 105302b553caSRafal Jaworowski static void 105402b553caSRafal Jaworowski sec_enqueue(struct sec_softc *sc) 105502b553caSRafal Jaworowski { 105602b553caSRafal Jaworowski struct sec_desc *desc; 105702b553caSRafal Jaworowski int ch0, ch1; 105802b553caSRafal Jaworowski 105902b553caSRafal Jaworowski SEC_LOCK(sc, descriptors); 106002b553caSRafal Jaworowski SEC_LOCK(sc, controller); 106102b553caSRafal Jaworowski 106202b553caSRafal Jaworowski while (SEC_READY_DESC_CNT(sc) > 0) { 106302b553caSRafal Jaworowski desc = SEC_GET_READY_DESC(sc); 106402b553caSRafal Jaworowski 106502b553caSRafal Jaworowski ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 106602b553caSRafal Jaworowski ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 106702b553caSRafal Jaworowski 106802b553caSRafal Jaworowski /* 106902b553caSRafal Jaworowski * Both EU are used by the same channel. 107002b553caSRafal Jaworowski * Enqueue descriptor in channel used by busy EUs. 107102b553caSRafal Jaworowski */ 107202b553caSRafal Jaworowski if (ch0 >= 0 && ch0 == ch1) { 107302b553caSRafal Jaworowski if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 107402b553caSRafal Jaworowski SEC_DESC_READY2QUEUED(sc); 107502b553caSRafal Jaworowski continue; 107602b553caSRafal Jaworowski } 107702b553caSRafal Jaworowski } 107802b553caSRafal Jaworowski 107902b553caSRafal Jaworowski /* 108002b553caSRafal Jaworowski * Only one EU is free. 108102b553caSRafal Jaworowski * Enqueue descriptor in channel used by busy EU. 108202b553caSRafal Jaworowski */ 108302b553caSRafal Jaworowski if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 108402b553caSRafal Jaworowski if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 108502b553caSRafal Jaworowski >= 0) { 108602b553caSRafal Jaworowski SEC_DESC_READY2QUEUED(sc); 108702b553caSRafal Jaworowski continue; 108802b553caSRafal Jaworowski } 108902b553caSRafal Jaworowski } 109002b553caSRafal Jaworowski 109102b553caSRafal Jaworowski /* 109202b553caSRafal Jaworowski * Both EU are free. 109302b553caSRafal Jaworowski * Enqueue descriptor in first free channel. 109402b553caSRafal Jaworowski */ 109502b553caSRafal Jaworowski if (ch0 < 0 && ch1 < 0) { 109602b553caSRafal Jaworowski if (sec_enqueue_desc(sc, desc, -1) >= 0) { 109702b553caSRafal Jaworowski SEC_DESC_READY2QUEUED(sc); 109802b553caSRafal Jaworowski continue; 109902b553caSRafal Jaworowski } 110002b553caSRafal Jaworowski } 110102b553caSRafal Jaworowski 110202b553caSRafal Jaworowski /* Current descriptor can not be queued at the moment */ 110302b553caSRafal Jaworowski SEC_PUT_BACK_READY_DESC(sc); 110402b553caSRafal Jaworowski break; 110502b553caSRafal Jaworowski } 110602b553caSRafal Jaworowski 110702b553caSRafal Jaworowski SEC_UNLOCK(sc, controller); 110802b553caSRafal Jaworowski SEC_UNLOCK(sc, descriptors); 110902b553caSRafal Jaworowski } 111002b553caSRafal Jaworowski 111102b553caSRafal Jaworowski static struct sec_desc * 111202b553caSRafal Jaworowski sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 111302b553caSRafal Jaworowski { 111402b553caSRafal Jaworowski struct sec_desc *desc = NULL; 111502b553caSRafal Jaworowski int i; 111602b553caSRafal Jaworowski 111702b553caSRafal Jaworowski SEC_LOCK_ASSERT(sc, descriptors); 111802b553caSRafal Jaworowski 111902b553caSRafal Jaworowski for (i = 0; i < SEC_CHANNELS; i++) { 112002b553caSRafal Jaworowski if (sc->sc_desc[i].sd_desc_paddr == paddr) { 112102b553caSRafal Jaworowski desc = &(sc->sc_desc[i]); 112202b553caSRafal Jaworowski break; 112302b553caSRafal Jaworowski } 112402b553caSRafal Jaworowski } 112502b553caSRafal Jaworowski 112602b553caSRafal Jaworowski return (desc); 112702b553caSRafal Jaworowski } 112802b553caSRafal Jaworowski 112902b553caSRafal Jaworowski static int 113002b553caSRafal Jaworowski sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 113102b553caSRafal Jaworowski bus_addr_t data, bus_size_t dsize) 113202b553caSRafal Jaworowski { 113302b553caSRafal Jaworowski struct sec_hw_desc_ptr *ptr; 113402b553caSRafal Jaworowski 113502b553caSRafal Jaworowski SEC_LOCK_ASSERT(sc, descriptors); 113602b553caSRafal Jaworowski 113702b553caSRafal Jaworowski ptr = &(desc->sd_desc->shd_pointer[n]); 113802b553caSRafal Jaworowski ptr->shdp_length = dsize; 113902b553caSRafal Jaworowski ptr->shdp_extent = 0; 114002b553caSRafal Jaworowski ptr->shdp_j = 0; 114102b553caSRafal Jaworowski ptr->shdp_ptr = data; 114202b553caSRafal Jaworowski 114302b553caSRafal Jaworowski return (0); 114402b553caSRafal Jaworowski } 114502b553caSRafal Jaworowski 114602b553caSRafal Jaworowski static int 114702b553caSRafal Jaworowski sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 114802b553caSRafal Jaworowski u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype) 114902b553caSRafal Jaworowski { 115002b553caSRafal Jaworowski struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 115102b553caSRafal Jaworowski struct sec_hw_desc_ptr *ptr; 115202b553caSRafal Jaworowski int error; 115302b553caSRafal Jaworowski 115402b553caSRafal Jaworowski SEC_LOCK_ASSERT(sc, descriptors); 115502b553caSRafal Jaworowski 115602b553caSRafal Jaworowski /* For flat memory map only requested region */ 115702b553caSRafal Jaworowski if (dtype == SEC_MEMORY) { 115802b553caSRafal Jaworowski data = (uint8_t*)(data) + doffset; 115902b553caSRafal Jaworowski sdmi.sdmi_offset = 0; 116002b553caSRafal Jaworowski } 116102b553caSRafal Jaworowski 116202b553caSRafal Jaworowski error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize, 116302b553caSRafal Jaworowski dtype, &sdmi); 116402b553caSRafal Jaworowski 116502b553caSRafal Jaworowski if (error) 116602b553caSRafal Jaworowski return (error); 116702b553caSRafal Jaworowski 116802b553caSRafal Jaworowski sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 116902b553caSRafal Jaworowski desc->sd_lt_used += sdmi.sdmi_lt_used; 117002b553caSRafal Jaworowski 117102b553caSRafal Jaworowski ptr = &(desc->sd_desc->shd_pointer[n]); 117202b553caSRafal Jaworowski ptr->shdp_length = dsize; 117302b553caSRafal Jaworowski ptr->shdp_extent = 0; 117402b553caSRafal Jaworowski ptr->shdp_j = 1; 117502b553caSRafal Jaworowski ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 117602b553caSRafal Jaworowski 117702b553caSRafal Jaworowski return (0); 117802b553caSRafal Jaworowski } 117902b553caSRafal Jaworowski 118002b553caSRafal Jaworowski static int 118102b553caSRafal Jaworowski sec_split_cri(struct cryptoini *cri, struct cryptoini **enc, 118202b553caSRafal Jaworowski struct cryptoini **mac) 118302b553caSRafal Jaworowski { 118402b553caSRafal Jaworowski struct cryptoini *e, *m; 118502b553caSRafal Jaworowski 118602b553caSRafal Jaworowski e = cri; 118702b553caSRafal Jaworowski m = cri->cri_next; 118802b553caSRafal Jaworowski 118902b553caSRafal Jaworowski /* We can haldle only two operations */ 119002b553caSRafal Jaworowski if (m && m->cri_next) 119102b553caSRafal Jaworowski return (EINVAL); 119202b553caSRafal Jaworowski 119302b553caSRafal Jaworowski if (sec_mdeu_can_handle(e->cri_alg)) { 119402b553caSRafal Jaworowski cri = m; 119502b553caSRafal Jaworowski m = e; 119602b553caSRafal Jaworowski e = cri; 119702b553caSRafal Jaworowski } 119802b553caSRafal Jaworowski 119902b553caSRafal Jaworowski if (m && !sec_mdeu_can_handle(m->cri_alg)) 120002b553caSRafal Jaworowski return (EINVAL); 120102b553caSRafal Jaworowski 120202b553caSRafal Jaworowski *enc = e; 120302b553caSRafal Jaworowski *mac = m; 120402b553caSRafal Jaworowski 120502b553caSRafal Jaworowski return (0); 120602b553caSRafal Jaworowski } 120702b553caSRafal Jaworowski 120802b553caSRafal Jaworowski static int 120902b553caSRafal Jaworowski sec_split_crp(struct cryptop *crp, struct cryptodesc **enc, 121002b553caSRafal Jaworowski struct cryptodesc **mac) 121102b553caSRafal Jaworowski { 121202b553caSRafal Jaworowski struct cryptodesc *e, *m, *t; 121302b553caSRafal Jaworowski 121402b553caSRafal Jaworowski e = crp->crp_desc; 121502b553caSRafal Jaworowski m = e->crd_next; 121602b553caSRafal Jaworowski 121702b553caSRafal Jaworowski /* We can haldle only two operations */ 121802b553caSRafal Jaworowski if (m && m->crd_next) 121902b553caSRafal Jaworowski return (EINVAL); 122002b553caSRafal Jaworowski 122102b553caSRafal Jaworowski if (sec_mdeu_can_handle(e->crd_alg)) { 122202b553caSRafal Jaworowski t = m; 122302b553caSRafal Jaworowski m = e; 122402b553caSRafal Jaworowski e = t; 122502b553caSRafal Jaworowski } 122602b553caSRafal Jaworowski 122702b553caSRafal Jaworowski if (m && !sec_mdeu_can_handle(m->crd_alg)) 122802b553caSRafal Jaworowski return (EINVAL); 122902b553caSRafal Jaworowski 123002b553caSRafal Jaworowski *enc = e; 123102b553caSRafal Jaworowski *mac = m; 123202b553caSRafal Jaworowski 123302b553caSRafal Jaworowski return (0); 123402b553caSRafal Jaworowski } 123502b553caSRafal Jaworowski 123602b553caSRafal Jaworowski static int 123702b553caSRafal Jaworowski sec_alloc_session(struct sec_softc *sc) 123802b553caSRafal Jaworowski { 123902b553caSRafal Jaworowski struct sec_session *ses = NULL; 124002b553caSRafal Jaworowski int sid = -1; 124102b553caSRafal Jaworowski u_int i; 124202b553caSRafal Jaworowski 124302b553caSRafal Jaworowski SEC_LOCK(sc, sessions); 124402b553caSRafal Jaworowski 124502b553caSRafal Jaworowski for (i = 0; i < SEC_MAX_SESSIONS; i++) { 124602b553caSRafal Jaworowski if (sc->sc_sessions[i].ss_used == 0) { 124702b553caSRafal Jaworowski ses = &(sc->sc_sessions[i]); 124802b553caSRafal Jaworowski ses->ss_used = 1; 124902b553caSRafal Jaworowski ses->ss_ivlen = 0; 125002b553caSRafal Jaworowski ses->ss_klen = 0; 125102b553caSRafal Jaworowski ses->ss_mklen = 0; 125202b553caSRafal Jaworowski sid = i; 125302b553caSRafal Jaworowski break; 125402b553caSRafal Jaworowski } 125502b553caSRafal Jaworowski } 125602b553caSRafal Jaworowski 125702b553caSRafal Jaworowski SEC_UNLOCK(sc, sessions); 125802b553caSRafal Jaworowski 125902b553caSRafal Jaworowski return (sid); 126002b553caSRafal Jaworowski } 126102b553caSRafal Jaworowski 126202b553caSRafal Jaworowski static struct sec_session * 126302b553caSRafal Jaworowski sec_get_session(struct sec_softc *sc, u_int sid) 126402b553caSRafal Jaworowski { 126502b553caSRafal Jaworowski struct sec_session *ses; 126602b553caSRafal Jaworowski 126702b553caSRafal Jaworowski if (sid >= SEC_MAX_SESSIONS) 126802b553caSRafal Jaworowski return (NULL); 126902b553caSRafal Jaworowski 127002b553caSRafal Jaworowski SEC_LOCK(sc, sessions); 127102b553caSRafal Jaworowski 127202b553caSRafal Jaworowski ses = &(sc->sc_sessions[sid]); 127302b553caSRafal Jaworowski 127402b553caSRafal Jaworowski if (ses->ss_used == 0) 127502b553caSRafal Jaworowski ses = NULL; 127602b553caSRafal Jaworowski 127702b553caSRafal Jaworowski SEC_UNLOCK(sc, sessions); 127802b553caSRafal Jaworowski 127902b553caSRafal Jaworowski return (ses); 128002b553caSRafal Jaworowski } 128102b553caSRafal Jaworowski 128202b553caSRafal Jaworowski static int 128302b553caSRafal Jaworowski sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri) 128402b553caSRafal Jaworowski { 128502b553caSRafal Jaworowski struct sec_softc *sc = device_get_softc(dev); 128602b553caSRafal Jaworowski struct sec_eu_methods *eu = sec_eus; 128702b553caSRafal Jaworowski struct cryptoini *enc = NULL; 128802b553caSRafal Jaworowski struct cryptoini *mac = NULL; 128902b553caSRafal Jaworowski struct sec_session *ses; 129002b553caSRafal Jaworowski int error = -1; 129102b553caSRafal Jaworowski int sid; 129202b553caSRafal Jaworowski 129302b553caSRafal Jaworowski error = sec_split_cri(cri, &enc, &mac); 129402b553caSRafal Jaworowski if (error) 129502b553caSRafal Jaworowski return (error); 129602b553caSRafal Jaworowski 129702b553caSRafal Jaworowski /* Check key lengths */ 129802b553caSRafal Jaworowski if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN) 129902b553caSRafal Jaworowski return (E2BIG); 130002b553caSRafal Jaworowski 130102b553caSRafal Jaworowski if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN) 130202b553caSRafal Jaworowski return (E2BIG); 130302b553caSRafal Jaworowski 130402b553caSRafal Jaworowski /* Only SEC 3.0 supports digests larger than 256 bits */ 130502b553caSRafal Jaworowski if (sc->sc_version < 3 && mac && mac->cri_klen > 256) 130602b553caSRafal Jaworowski return (E2BIG); 130702b553caSRafal Jaworowski 130802b553caSRafal Jaworowski sid = sec_alloc_session(sc); 130902b553caSRafal Jaworowski if (sid < 0) 131002b553caSRafal Jaworowski return (ENOMEM); 131102b553caSRafal Jaworowski 131202b553caSRafal Jaworowski ses = sec_get_session(sc, sid); 131302b553caSRafal Jaworowski 131402b553caSRafal Jaworowski /* Find EU for this session */ 131502b553caSRafal Jaworowski while (eu->sem_make_desc != NULL) { 131602b553caSRafal Jaworowski error = eu->sem_newsession(sc, ses, enc, mac); 131702b553caSRafal Jaworowski if (error >= 0) 131802b553caSRafal Jaworowski break; 131902b553caSRafal Jaworowski 132002b553caSRafal Jaworowski eu++; 132102b553caSRafal Jaworowski } 132202b553caSRafal Jaworowski 132302b553caSRafal Jaworowski /* If not found, return EINVAL */ 132402b553caSRafal Jaworowski if (error < 0) { 132502b553caSRafal Jaworowski sec_free_session(sc, ses); 132602b553caSRafal Jaworowski return (EINVAL); 132702b553caSRafal Jaworowski } 132802b553caSRafal Jaworowski 132902b553caSRafal Jaworowski /* Save cipher key */ 133002b553caSRafal Jaworowski if (enc && enc->cri_key) { 133102b553caSRafal Jaworowski ses->ss_klen = enc->cri_klen / 8; 133202b553caSRafal Jaworowski memcpy(ses->ss_key, enc->cri_key, ses->ss_klen); 133302b553caSRafal Jaworowski } 133402b553caSRafal Jaworowski 133502b553caSRafal Jaworowski /* Save digest key */ 133602b553caSRafal Jaworowski if (mac && mac->cri_key) { 133702b553caSRafal Jaworowski ses->ss_mklen = mac->cri_klen / 8; 133802b553caSRafal Jaworowski memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen); 133902b553caSRafal Jaworowski } 134002b553caSRafal Jaworowski 134102b553caSRafal Jaworowski ses->ss_eu = eu; 134202b553caSRafal Jaworowski *sidp = sid; 134302b553caSRafal Jaworowski 134402b553caSRafal Jaworowski return (0); 134502b553caSRafal Jaworowski } 134602b553caSRafal Jaworowski 134702b553caSRafal Jaworowski static int 134802b553caSRafal Jaworowski sec_freesession(device_t dev, uint64_t tid) 134902b553caSRafal Jaworowski { 135002b553caSRafal Jaworowski struct sec_softc *sc = device_get_softc(dev); 135102b553caSRafal Jaworowski struct sec_session *ses; 135202b553caSRafal Jaworowski int error = 0; 135302b553caSRafal Jaworowski 135402b553caSRafal Jaworowski ses = sec_get_session(sc, CRYPTO_SESID2LID(tid)); 135502b553caSRafal Jaworowski if (ses == NULL) 135602b553caSRafal Jaworowski return (EINVAL); 135702b553caSRafal Jaworowski 135802b553caSRafal Jaworowski sec_free_session(sc, ses); 135902b553caSRafal Jaworowski 136002b553caSRafal Jaworowski return (error); 136102b553caSRafal Jaworowski } 136202b553caSRafal Jaworowski 136302b553caSRafal Jaworowski static int 136402b553caSRafal Jaworowski sec_process(device_t dev, struct cryptop *crp, int hint) 136502b553caSRafal Jaworowski { 136602b553caSRafal Jaworowski struct sec_softc *sc = device_get_softc(dev); 136702b553caSRafal Jaworowski struct sec_desc *desc = NULL; 136802b553caSRafal Jaworowski struct cryptodesc *mac, *enc; 136902b553caSRafal Jaworowski struct sec_session *ses; 137002b553caSRafal Jaworowski int buftype, error = 0; 137102b553caSRafal Jaworowski 137202b553caSRafal Jaworowski /* Check Session ID */ 137302b553caSRafal Jaworowski ses = sec_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid)); 137402b553caSRafal Jaworowski if (ses == NULL) { 137502b553caSRafal Jaworowski crp->crp_etype = EINVAL; 137602b553caSRafal Jaworowski crypto_done(crp); 137702b553caSRafal Jaworowski return (0); 137802b553caSRafal Jaworowski } 137902b553caSRafal Jaworowski 138002b553caSRafal Jaworowski /* Check for input length */ 138102b553caSRafal Jaworowski if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) { 138202b553caSRafal Jaworowski crp->crp_etype = E2BIG; 138302b553caSRafal Jaworowski crypto_done(crp); 138402b553caSRafal Jaworowski return (0); 138502b553caSRafal Jaworowski } 138602b553caSRafal Jaworowski 138702b553caSRafal Jaworowski /* Get descriptors */ 138802b553caSRafal Jaworowski if (sec_split_crp(crp, &enc, &mac)) { 138902b553caSRafal Jaworowski crp->crp_etype = EINVAL; 139002b553caSRafal Jaworowski crypto_done(crp); 139102b553caSRafal Jaworowski return (0); 139202b553caSRafal Jaworowski } 139302b553caSRafal Jaworowski 139402b553caSRafal Jaworowski SEC_LOCK(sc, descriptors); 139502b553caSRafal Jaworowski SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 139602b553caSRafal Jaworowski 139702b553caSRafal Jaworowski /* Block driver if there is no free descriptors or we are going down */ 139802b553caSRafal Jaworowski if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 139902b553caSRafal Jaworowski sc->sc_blocked |= CRYPTO_SYMQ; 140002b553caSRafal Jaworowski SEC_UNLOCK(sc, descriptors); 140102b553caSRafal Jaworowski return (ERESTART); 140202b553caSRafal Jaworowski } 140302b553caSRafal Jaworowski 140402b553caSRafal Jaworowski /* Prepare descriptor */ 140502b553caSRafal Jaworowski desc = SEC_GET_FREE_DESC(sc); 140602b553caSRafal Jaworowski desc->sd_lt_used = 0; 140702b553caSRafal Jaworowski desc->sd_error = 0; 140802b553caSRafal Jaworowski desc->sd_crp = crp; 140902b553caSRafal Jaworowski 141002b553caSRafal Jaworowski if (crp->crp_flags & CRYPTO_F_IOV) 141102b553caSRafal Jaworowski buftype = SEC_UIO; 141202b553caSRafal Jaworowski else if (crp->crp_flags & CRYPTO_F_IMBUF) 141302b553caSRafal Jaworowski buftype = SEC_MBUF; 141402b553caSRafal Jaworowski else 141502b553caSRafal Jaworowski buftype = SEC_MEMORY; 141602b553caSRafal Jaworowski 141702b553caSRafal Jaworowski if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 141802b553caSRafal Jaworowski if (enc->crd_flags & CRD_F_IV_EXPLICIT) 141902b553caSRafal Jaworowski memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 142002b553caSRafal Jaworowski ses->ss_ivlen); 142102b553caSRafal Jaworowski else 142202b553caSRafal Jaworowski arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0); 142302b553caSRafal Jaworowski 142402b553caSRafal Jaworowski if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 142502b553caSRafal Jaworowski crypto_copyback(crp->crp_flags, crp->crp_buf, 142602b553caSRafal Jaworowski enc->crd_inject, ses->ss_ivlen, 142702b553caSRafal Jaworowski desc->sd_desc->shd_iv); 142802b553caSRafal Jaworowski } else if (enc) { 142902b553caSRafal Jaworowski if (enc->crd_flags & CRD_F_IV_EXPLICIT) 143002b553caSRafal Jaworowski memcpy(desc->sd_desc->shd_iv, enc->crd_iv, 143102b553caSRafal Jaworowski ses->ss_ivlen); 143202b553caSRafal Jaworowski else 143302b553caSRafal Jaworowski crypto_copydata(crp->crp_flags, crp->crp_buf, 143402b553caSRafal Jaworowski enc->crd_inject, ses->ss_ivlen, 143502b553caSRafal Jaworowski desc->sd_desc->shd_iv); 143602b553caSRafal Jaworowski } 143702b553caSRafal Jaworowski 143802b553caSRafal Jaworowski if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 143902b553caSRafal Jaworowski if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 144002b553caSRafal Jaworowski ses->ss_klen = enc->crd_klen / 8; 144102b553caSRafal Jaworowski memcpy(ses->ss_key, enc->crd_key, ses->ss_klen); 144202b553caSRafal Jaworowski } else 144302b553caSRafal Jaworowski error = E2BIG; 144402b553caSRafal Jaworowski } 144502b553caSRafal Jaworowski 144602b553caSRafal Jaworowski if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 144702b553caSRafal Jaworowski if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) { 144802b553caSRafal Jaworowski ses->ss_mklen = mac->crd_klen / 8; 144902b553caSRafal Jaworowski memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen); 145002b553caSRafal Jaworowski } else 145102b553caSRafal Jaworowski error = E2BIG; 145202b553caSRafal Jaworowski } 145302b553caSRafal Jaworowski 145402b553caSRafal Jaworowski if (!error) { 145502b553caSRafal Jaworowski memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen); 145602b553caSRafal Jaworowski memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen); 145702b553caSRafal Jaworowski 145802b553caSRafal Jaworowski error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype); 145902b553caSRafal Jaworowski } 146002b553caSRafal Jaworowski 146102b553caSRafal Jaworowski if (error) { 146202b553caSRafal Jaworowski SEC_DESC_FREE_POINTERS(desc); 146302b553caSRafal Jaworowski SEC_DESC_PUT_BACK_LT(sc, desc); 146402b553caSRafal Jaworowski SEC_PUT_BACK_FREE_DESC(sc); 146502b553caSRafal Jaworowski SEC_UNLOCK(sc, descriptors); 146602b553caSRafal Jaworowski crp->crp_etype = error; 146702b553caSRafal Jaworowski crypto_done(crp); 146802b553caSRafal Jaworowski return (0); 146902b553caSRafal Jaworowski } 147002b553caSRafal Jaworowski 147102b553caSRafal Jaworowski /* 147202b553caSRafal Jaworowski * Skip DONE interrupt if this is not last request in burst, but only 147302b553caSRafal Jaworowski * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 147402b553caSRafal Jaworowski * signaling on each descriptor. 147502b553caSRafal Jaworowski */ 147602b553caSRafal Jaworowski if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 147702b553caSRafal Jaworowski desc->sd_desc->shd_dn = 0; 147802b553caSRafal Jaworowski else 147902b553caSRafal Jaworowski desc->sd_desc->shd_dn = 1; 148002b553caSRafal Jaworowski 148102b553caSRafal Jaworowski SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 148202b553caSRafal Jaworowski SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 148302b553caSRafal Jaworowski BUS_DMASYNC_POSTWRITE); 148402b553caSRafal Jaworowski SEC_DESC_FREE2READY(sc); 148502b553caSRafal Jaworowski SEC_UNLOCK(sc, descriptors); 148602b553caSRafal Jaworowski 148702b553caSRafal Jaworowski /* Enqueue ready descriptors in hardware */ 148802b553caSRafal Jaworowski sec_enqueue(sc); 148902b553caSRafal Jaworowski 149002b553caSRafal Jaworowski return (0); 149102b553caSRafal Jaworowski } 149202b553caSRafal Jaworowski 149302b553caSRafal Jaworowski static int 149402b553caSRafal Jaworowski sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 149502b553caSRafal Jaworowski struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 149602b553caSRafal Jaworowski int buftype) 149702b553caSRafal Jaworowski { 149802b553caSRafal Jaworowski struct sec_hw_desc *hd = desc->sd_desc; 149902b553caSRafal Jaworowski int error; 150002b553caSRafal Jaworowski 150102b553caSRafal Jaworowski hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 150202b553caSRafal Jaworowski hd->shd_eu_sel1 = SEC_EU_NONE; 150302b553caSRafal Jaworowski hd->shd_mode1 = 0; 150402b553caSRafal Jaworowski 150502b553caSRafal Jaworowski /* Pointer 0: NULL */ 150602b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 150702b553caSRafal Jaworowski if (error) 150802b553caSRafal Jaworowski return (error); 150902b553caSRafal Jaworowski 151002b553caSRafal Jaworowski /* Pointer 1: IV IN */ 151102b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 151202b553caSRafal Jaworowski offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 151302b553caSRafal Jaworowski if (error) 151402b553caSRafal Jaworowski return (error); 151502b553caSRafal Jaworowski 151602b553caSRafal Jaworowski /* Pointer 2: Cipher Key */ 151702b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 151802b553caSRafal Jaworowski offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 151902b553caSRafal Jaworowski if (error) 152002b553caSRafal Jaworowski return (error); 152102b553caSRafal Jaworowski 152202b553caSRafal Jaworowski /* Pointer 3: Data IN */ 152302b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip, 152402b553caSRafal Jaworowski enc->crd_len, buftype); 152502b553caSRafal Jaworowski if (error) 152602b553caSRafal Jaworowski return (error); 152702b553caSRafal Jaworowski 152802b553caSRafal Jaworowski /* Pointer 4: Data OUT */ 152902b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 153002b553caSRafal Jaworowski enc->crd_len, buftype); 153102b553caSRafal Jaworowski if (error) 153202b553caSRafal Jaworowski return (error); 153302b553caSRafal Jaworowski 153402b553caSRafal Jaworowski /* Pointer 5: IV OUT (Not used: NULL) */ 153502b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 153602b553caSRafal Jaworowski if (error) 153702b553caSRafal Jaworowski return (error); 153802b553caSRafal Jaworowski 153902b553caSRafal Jaworowski /* Pointer 6: NULL */ 154002b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 154102b553caSRafal Jaworowski 154202b553caSRafal Jaworowski return (error); 154302b553caSRafal Jaworowski } 154402b553caSRafal Jaworowski 154502b553caSRafal Jaworowski static int 154602b553caSRafal Jaworowski sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 154702b553caSRafal Jaworowski struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc, 154802b553caSRafal Jaworowski struct cryptodesc *mac, int buftype) 154902b553caSRafal Jaworowski { 155002b553caSRafal Jaworowski struct sec_hw_desc *hd = desc->sd_desc; 155102b553caSRafal Jaworowski u_int eu, mode, hashlen; 155202b553caSRafal Jaworowski int error; 155302b553caSRafal Jaworowski 155402b553caSRafal Jaworowski if (mac->crd_len < enc->crd_len) 155502b553caSRafal Jaworowski return (EINVAL); 155602b553caSRafal Jaworowski 155702b553caSRafal Jaworowski if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len) 155802b553caSRafal Jaworowski return (EINVAL); 155902b553caSRafal Jaworowski 156002b553caSRafal Jaworowski error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 156102b553caSRafal Jaworowski if (error) 156202b553caSRafal Jaworowski return (error); 156302b553caSRafal Jaworowski 156402b553caSRafal Jaworowski hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 156502b553caSRafal Jaworowski hd->shd_eu_sel1 = eu; 156602b553caSRafal Jaworowski hd->shd_mode1 = mode; 156702b553caSRafal Jaworowski 156802b553caSRafal Jaworowski /* Pointer 0: HMAC Key */ 156902b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 157002b553caSRafal Jaworowski offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen); 157102b553caSRafal Jaworowski if (error) 157202b553caSRafal Jaworowski return (error); 157302b553caSRafal Jaworowski 157402b553caSRafal Jaworowski /* Pointer 1: HMAC-Only Data IN */ 157502b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip, 157602b553caSRafal Jaworowski mac->crd_len - enc->crd_len, buftype); 157702b553caSRafal Jaworowski if (error) 157802b553caSRafal Jaworowski return (error); 157902b553caSRafal Jaworowski 158002b553caSRafal Jaworowski /* Pointer 2: Cipher Key */ 158102b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 158202b553caSRafal Jaworowski offsetof(struct sec_hw_desc, shd_key), ses->ss_klen); 158302b553caSRafal Jaworowski if (error) 158402b553caSRafal Jaworowski return (error); 158502b553caSRafal Jaworowski 158602b553caSRafal Jaworowski /* Pointer 3: IV IN */ 158702b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 158802b553caSRafal Jaworowski offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen); 158902b553caSRafal Jaworowski if (error) 159002b553caSRafal Jaworowski return (error); 159102b553caSRafal Jaworowski 159202b553caSRafal Jaworowski /* Pointer 4: Data IN */ 159302b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip, 159402b553caSRafal Jaworowski enc->crd_len, buftype); 159502b553caSRafal Jaworowski if (error) 159602b553caSRafal Jaworowski return (error); 159702b553caSRafal Jaworowski 159802b553caSRafal Jaworowski /* Pointer 5: Data OUT */ 159902b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip, 160002b553caSRafal Jaworowski enc->crd_len, buftype); 160102b553caSRafal Jaworowski if (error) 160202b553caSRafal Jaworowski return (error); 160302b553caSRafal Jaworowski 160402b553caSRafal Jaworowski /* Pointer 6: HMAC OUT */ 160502b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject, 160602b553caSRafal Jaworowski hashlen, buftype); 160702b553caSRafal Jaworowski 160802b553caSRafal Jaworowski return (error); 160902b553caSRafal Jaworowski } 161002b553caSRafal Jaworowski 161102b553caSRafal Jaworowski /* AESU */ 161202b553caSRafal Jaworowski 161302b553caSRafal Jaworowski static int 161402b553caSRafal Jaworowski sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses, 161502b553caSRafal Jaworowski struct cryptoini *enc, struct cryptoini *mac) 161602b553caSRafal Jaworowski { 161702b553caSRafal Jaworowski 161802b553caSRafal Jaworowski if (enc == NULL) 161902b553caSRafal Jaworowski return (-1); 162002b553caSRafal Jaworowski 162102b553caSRafal Jaworowski if (enc->cri_alg != CRYPTO_AES_CBC) 162202b553caSRafal Jaworowski return (-1); 162302b553caSRafal Jaworowski 162402b553caSRafal Jaworowski ses->ss_ivlen = AES_BLOCK_LEN; 162502b553caSRafal Jaworowski 162602b553caSRafal Jaworowski return (0); 162702b553caSRafal Jaworowski } 162802b553caSRafal Jaworowski 162902b553caSRafal Jaworowski static int 163002b553caSRafal Jaworowski sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses, 163102b553caSRafal Jaworowski struct sec_desc *desc, struct cryptop *crp, int buftype) 163202b553caSRafal Jaworowski { 163302b553caSRafal Jaworowski struct sec_hw_desc *hd = desc->sd_desc; 163402b553caSRafal Jaworowski struct cryptodesc *enc, *mac; 163502b553caSRafal Jaworowski int error; 163602b553caSRafal Jaworowski 163702b553caSRafal Jaworowski error = sec_split_crp(crp, &enc, &mac); 163802b553caSRafal Jaworowski if (error) 163902b553caSRafal Jaworowski return (error); 164002b553caSRafal Jaworowski 164102b553caSRafal Jaworowski if (!enc) 164202b553caSRafal Jaworowski return (EINVAL); 164302b553caSRafal Jaworowski 164402b553caSRafal Jaworowski hd->shd_eu_sel0 = SEC_EU_AESU; 164502b553caSRafal Jaworowski hd->shd_mode0 = SEC_AESU_MODE_CBC; 164602b553caSRafal Jaworowski 164702b553caSRafal Jaworowski if (enc->crd_alg != CRYPTO_AES_CBC) 164802b553caSRafal Jaworowski return (EINVAL); 164902b553caSRafal Jaworowski 165002b553caSRafal Jaworowski if (enc->crd_flags & CRD_F_ENCRYPT) { 165102b553caSRafal Jaworowski hd->shd_mode0 |= SEC_AESU_MODE_ED; 165202b553caSRafal Jaworowski hd->shd_dir = 0; 165302b553caSRafal Jaworowski } else 165402b553caSRafal Jaworowski hd->shd_dir = 1; 165502b553caSRafal Jaworowski 165602b553caSRafal Jaworowski if (mac) 165702b553caSRafal Jaworowski error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 165802b553caSRafal Jaworowski buftype); 165902b553caSRafal Jaworowski else 166002b553caSRafal Jaworowski error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 166102b553caSRafal Jaworowski buftype); 166202b553caSRafal Jaworowski 166302b553caSRafal Jaworowski return (error); 166402b553caSRafal Jaworowski } 166502b553caSRafal Jaworowski 166602b553caSRafal Jaworowski /* DEU */ 166702b553caSRafal Jaworowski 166802b553caSRafal Jaworowski static int 166902b553caSRafal Jaworowski sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses, 167002b553caSRafal Jaworowski struct cryptoini *enc, struct cryptoini *mac) 167102b553caSRafal Jaworowski { 167202b553caSRafal Jaworowski 167302b553caSRafal Jaworowski if (enc == NULL) 167402b553caSRafal Jaworowski return (-1); 167502b553caSRafal Jaworowski 167602b553caSRafal Jaworowski switch (enc->cri_alg) { 167702b553caSRafal Jaworowski case CRYPTO_DES_CBC: 167802b553caSRafal Jaworowski case CRYPTO_3DES_CBC: 167902b553caSRafal Jaworowski break; 168002b553caSRafal Jaworowski default: 168102b553caSRafal Jaworowski return (-1); 168202b553caSRafal Jaworowski } 168302b553caSRafal Jaworowski 168402b553caSRafal Jaworowski ses->ss_ivlen = DES_BLOCK_LEN; 168502b553caSRafal Jaworowski 168602b553caSRafal Jaworowski return (0); 168702b553caSRafal Jaworowski } 168802b553caSRafal Jaworowski 168902b553caSRafal Jaworowski static int 169002b553caSRafal Jaworowski sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses, 169102b553caSRafal Jaworowski struct sec_desc *desc, struct cryptop *crp, int buftype) 169202b553caSRafal Jaworowski { 169302b553caSRafal Jaworowski struct sec_hw_desc *hd = desc->sd_desc; 169402b553caSRafal Jaworowski struct cryptodesc *enc, *mac; 169502b553caSRafal Jaworowski int error; 169602b553caSRafal Jaworowski 169702b553caSRafal Jaworowski error = sec_split_crp(crp, &enc, &mac); 169802b553caSRafal Jaworowski if (error) 169902b553caSRafal Jaworowski return (error); 170002b553caSRafal Jaworowski 170102b553caSRafal Jaworowski if (!enc) 170202b553caSRafal Jaworowski return (EINVAL); 170302b553caSRafal Jaworowski 170402b553caSRafal Jaworowski hd->shd_eu_sel0 = SEC_EU_DEU; 170502b553caSRafal Jaworowski hd->shd_mode0 = SEC_DEU_MODE_CBC; 170602b553caSRafal Jaworowski 170702b553caSRafal Jaworowski switch (enc->crd_alg) { 170802b553caSRafal Jaworowski case CRYPTO_3DES_CBC: 170902b553caSRafal Jaworowski hd->shd_mode0 |= SEC_DEU_MODE_TS; 171002b553caSRafal Jaworowski break; 171102b553caSRafal Jaworowski case CRYPTO_DES_CBC: 171202b553caSRafal Jaworowski break; 171302b553caSRafal Jaworowski default: 171402b553caSRafal Jaworowski return (EINVAL); 171502b553caSRafal Jaworowski } 171602b553caSRafal Jaworowski 171702b553caSRafal Jaworowski if (enc->crd_flags & CRD_F_ENCRYPT) { 171802b553caSRafal Jaworowski hd->shd_mode0 |= SEC_DEU_MODE_ED; 171902b553caSRafal Jaworowski hd->shd_dir = 0; 172002b553caSRafal Jaworowski } else 172102b553caSRafal Jaworowski hd->shd_dir = 1; 172202b553caSRafal Jaworowski 172302b553caSRafal Jaworowski if (mac) 172402b553caSRafal Jaworowski error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac, 172502b553caSRafal Jaworowski buftype); 172602b553caSRafal Jaworowski else 172702b553caSRafal Jaworowski error = sec_build_common_ns_desc(sc, desc, ses, crp, enc, 172802b553caSRafal Jaworowski buftype); 172902b553caSRafal Jaworowski 173002b553caSRafal Jaworowski return (error); 173102b553caSRafal Jaworowski } 173202b553caSRafal Jaworowski 173302b553caSRafal Jaworowski /* MDEU */ 173402b553caSRafal Jaworowski 173502b553caSRafal Jaworowski static int 173602b553caSRafal Jaworowski sec_mdeu_can_handle(u_int alg) 173702b553caSRafal Jaworowski { 173802b553caSRafal Jaworowski switch (alg) { 173902b553caSRafal Jaworowski case CRYPTO_MD5: 174002b553caSRafal Jaworowski case CRYPTO_SHA1: 174102b553caSRafal Jaworowski case CRYPTO_MD5_HMAC: 174202b553caSRafal Jaworowski case CRYPTO_SHA1_HMAC: 174302b553caSRafal Jaworowski case CRYPTO_SHA2_256_HMAC: 174402b553caSRafal Jaworowski case CRYPTO_SHA2_384_HMAC: 174502b553caSRafal Jaworowski case CRYPTO_SHA2_512_HMAC: 174602b553caSRafal Jaworowski return (1); 174702b553caSRafal Jaworowski default: 174802b553caSRafal Jaworowski return (0); 174902b553caSRafal Jaworowski } 175002b553caSRafal Jaworowski } 175102b553caSRafal Jaworowski 175202b553caSRafal Jaworowski static int 175302b553caSRafal Jaworowski sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen) 175402b553caSRafal Jaworowski { 175502b553caSRafal Jaworowski 175602b553caSRafal Jaworowski *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 175702b553caSRafal Jaworowski *eu = SEC_EU_NONE; 175802b553caSRafal Jaworowski 175902b553caSRafal Jaworowski switch (crd->crd_alg) { 176002b553caSRafal Jaworowski case CRYPTO_MD5_HMAC: 176102b553caSRafal Jaworowski *mode |= SEC_MDEU_MODE_HMAC; 176202b553caSRafal Jaworowski /* FALLTHROUGH */ 176302b553caSRafal Jaworowski case CRYPTO_MD5: 176402b553caSRafal Jaworowski *eu = SEC_EU_MDEU_A; 176502b553caSRafal Jaworowski *mode |= SEC_MDEU_MODE_MD5; 176602b553caSRafal Jaworowski *hashlen = MD5_HASH_LEN; 176702b553caSRafal Jaworowski break; 176802b553caSRafal Jaworowski case CRYPTO_SHA1_HMAC: 176902b553caSRafal Jaworowski *mode |= SEC_MDEU_MODE_HMAC; 177002b553caSRafal Jaworowski /* FALLTHROUGH */ 177102b553caSRafal Jaworowski case CRYPTO_SHA1: 177202b553caSRafal Jaworowski *eu = SEC_EU_MDEU_A; 177302b553caSRafal Jaworowski *mode |= SEC_MDEU_MODE_SHA1; 177402b553caSRafal Jaworowski *hashlen = SHA1_HASH_LEN; 177502b553caSRafal Jaworowski break; 177602b553caSRafal Jaworowski case CRYPTO_SHA2_256_HMAC: 177702b553caSRafal Jaworowski *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 177802b553caSRafal Jaworowski *eu = SEC_EU_MDEU_A; 177902b553caSRafal Jaworowski break; 178002b553caSRafal Jaworowski case CRYPTO_SHA2_384_HMAC: 178102b553caSRafal Jaworowski *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 178202b553caSRafal Jaworowski *eu = SEC_EU_MDEU_B; 178302b553caSRafal Jaworowski break; 178402b553caSRafal Jaworowski case CRYPTO_SHA2_512_HMAC: 178502b553caSRafal Jaworowski *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 178602b553caSRafal Jaworowski *eu = SEC_EU_MDEU_B; 178702b553caSRafal Jaworowski break; 178802b553caSRafal Jaworowski default: 178902b553caSRafal Jaworowski return (EINVAL); 179002b553caSRafal Jaworowski } 179102b553caSRafal Jaworowski 179202b553caSRafal Jaworowski if (*mode & SEC_MDEU_MODE_HMAC) 179302b553caSRafal Jaworowski *hashlen = SEC_HMAC_HASH_LEN; 179402b553caSRafal Jaworowski 179502b553caSRafal Jaworowski return (0); 179602b553caSRafal Jaworowski } 179702b553caSRafal Jaworowski 179802b553caSRafal Jaworowski static int 179902b553caSRafal Jaworowski sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses, 180002b553caSRafal Jaworowski struct cryptoini *enc, struct cryptoini *mac) 180102b553caSRafal Jaworowski { 180202b553caSRafal Jaworowski 180302b553caSRafal Jaworowski if (mac && sec_mdeu_can_handle(mac->cri_alg)) 180402b553caSRafal Jaworowski return (0); 180502b553caSRafal Jaworowski 180602b553caSRafal Jaworowski return (-1); 180702b553caSRafal Jaworowski } 180802b553caSRafal Jaworowski 180902b553caSRafal Jaworowski static int 181002b553caSRafal Jaworowski sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses, 181102b553caSRafal Jaworowski struct sec_desc *desc, struct cryptop *crp, int buftype) 181202b553caSRafal Jaworowski { 181302b553caSRafal Jaworowski struct cryptodesc *enc, *mac; 181402b553caSRafal Jaworowski struct sec_hw_desc *hd = desc->sd_desc; 181502b553caSRafal Jaworowski u_int eu, mode, hashlen; 181602b553caSRafal Jaworowski int error; 181702b553caSRafal Jaworowski 181802b553caSRafal Jaworowski error = sec_split_crp(crp, &enc, &mac); 181902b553caSRafal Jaworowski if (error) 182002b553caSRafal Jaworowski return (error); 182102b553caSRafal Jaworowski 182202b553caSRafal Jaworowski if (enc) 182302b553caSRafal Jaworowski return (EINVAL); 182402b553caSRafal Jaworowski 182502b553caSRafal Jaworowski error = sec_mdeu_config(mac, &eu, &mode, &hashlen); 182602b553caSRafal Jaworowski if (error) 182702b553caSRafal Jaworowski return (error); 182802b553caSRafal Jaworowski 182902b553caSRafal Jaworowski hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 183002b553caSRafal Jaworowski hd->shd_eu_sel0 = eu; 183102b553caSRafal Jaworowski hd->shd_mode0 = mode; 183202b553caSRafal Jaworowski hd->shd_eu_sel1 = SEC_EU_NONE; 183302b553caSRafal Jaworowski hd->shd_mode1 = 0; 183402b553caSRafal Jaworowski 183502b553caSRafal Jaworowski /* Pointer 0: NULL */ 183602b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 183702b553caSRafal Jaworowski if (error) 183802b553caSRafal Jaworowski return (error); 183902b553caSRafal Jaworowski 184002b553caSRafal Jaworowski /* Pointer 1: Context In (Not used: NULL) */ 184102b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 184202b553caSRafal Jaworowski if (error) 184302b553caSRafal Jaworowski return (error); 184402b553caSRafal Jaworowski 184502b553caSRafal Jaworowski /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 184602b553caSRafal Jaworowski if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 184702b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 2, 184802b553caSRafal Jaworowski desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 184902b553caSRafal Jaworowski shd_mkey), ses->ss_mklen); 185002b553caSRafal Jaworowski else 185102b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 185202b553caSRafal Jaworowski 185302b553caSRafal Jaworowski if (error) 185402b553caSRafal Jaworowski return (error); 185502b553caSRafal Jaworowski 185602b553caSRafal Jaworowski /* Pointer 3: Input Data */ 185702b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip, 185802b553caSRafal Jaworowski mac->crd_len, buftype); 185902b553caSRafal Jaworowski if (error) 186002b553caSRafal Jaworowski return (error); 186102b553caSRafal Jaworowski 186202b553caSRafal Jaworowski /* Pointer 4: NULL */ 186302b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 186402b553caSRafal Jaworowski if (error) 186502b553caSRafal Jaworowski return (error); 186602b553caSRafal Jaworowski 186702b553caSRafal Jaworowski /* Pointer 5: Hash out */ 186802b553caSRafal Jaworowski error = sec_make_pointer(sc, desc, 5, crp->crp_buf, 186902b553caSRafal Jaworowski mac->crd_inject, hashlen, buftype); 187002b553caSRafal Jaworowski if (error) 187102b553caSRafal Jaworowski return (error); 187202b553caSRafal Jaworowski 187302b553caSRafal Jaworowski /* Pointer 6: NULL */ 187402b553caSRafal Jaworowski error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 187502b553caSRafal Jaworowski 187602b553caSRafal Jaworowski return (0); 187702b553caSRafal Jaworowski } 1878