Lines Matching +full:cdr +full:- +full:mode
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
64 { "inside-secure,safexcel-eip97ies", (uintptr_t)97 },
65 { "inside-secure,safexcel-eip97", (uintptr_t)97 },
100 i = ring->cdr.read; in safexcel_next_request()
103 return (&ring->requests[i]); in safexcel_next_request()
111 if (ring->write == ring->read) in safexcel_cmd_descr_next()
113 cdesc = &ring->desc[ring->read]; in safexcel_cmd_descr_next()
114 ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; in safexcel_cmd_descr_next()
123 if (ring->write == ring->read) in safexcel_res_descr_next()
125 rdesc = &ring->desc[ring->read]; in safexcel_res_descr_next()
126 ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; in safexcel_res_descr_next()
135 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_alloc_request()
137 i = ring->cdr.write; in safexcel_alloc_request()
138 if ((i + 1) % SAFEXCEL_RING_SIZE == ring->cdr.read) in safexcel_alloc_request()
140 return (&ring->requests[i]); in safexcel_alloc_request()
148 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_free_request()
150 if (req->dmap_loaded) { in safexcel_free_request()
151 bus_dmamap_unload(ring->data_dtag, req->dmap); in safexcel_free_request()
152 req->dmap_loaded = false; in safexcel_free_request()
154 ctx = (struct safexcel_context_record *)req->ctx.vaddr; in safexcel_free_request()
155 explicit_bzero(ctx->data, sizeof(ctx->data)); in safexcel_free_request()
156 explicit_bzero(req->iv, sizeof(req->iv)); in safexcel_free_request()
171 ring = &sc->sc_ring[ringidx]; in safexcel_rdr_intr()
180 mtx_lock(&ring->mtx); in safexcel_rdr_intr()
186 ring = &sc->sc_ring[ringidx]; in safexcel_rdr_intr()
187 bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, in safexcel_rdr_intr()
189 bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, in safexcel_rdr_intr()
191 bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, in safexcel_rdr_intr()
198 bus_dmamap_sync(req->ctx.tag, req->ctx.map, in safexcel_rdr_intr()
200 bus_dmamap_sync(ring->data_dtag, req->dmap, in safexcel_rdr_intr()
203 while (req->cdescs-- > 0) { in safexcel_rdr_intr()
204 cdesc = safexcel_cmd_descr_next(&ring->cdr); in safexcel_rdr_intr()
207 if (req->cdescs == 0) in safexcel_rdr_intr()
208 KASSERT(cdesc->last_seg, in safexcel_rdr_intr()
211 nrdescs += req->rdescs; in safexcel_rdr_intr()
212 while (req->rdescs-- > 0) { in safexcel_rdr_intr()
213 rdesc = safexcel_res_descr_next(&ring->rdr); in safexcel_rdr_intr()
214 error = rdesc->result_data.error_code; in safexcel_rdr_intr()
217 req->crp->crp_etype == 0) { in safexcel_rdr_intr()
218 req->crp->crp_etype = EBADMSG; in safexcel_rdr_intr()
222 req->crp->crp_etype = EIO; in safexcel_rdr_intr()
227 TAILQ_INSERT_TAIL(&cq, req->crp, crp_next); in safexcel_rdr_intr()
230 mtx_lock(&ring->mtx); in safexcel_rdr_intr()
232 KASSERT(ring->queued >= nreqs, in safexcel_rdr_intr()
234 __func__, ring->queued, nreqs)); in safexcel_rdr_intr()
235 ring->queued -= nreqs; in safexcel_rdr_intr()
240 (sc->sc_config.rd_offset * nrdescs * sizeof(uint32_t))); in safexcel_rdr_intr()
241 blocked = ring->blocked; in safexcel_rdr_intr()
242 ring->blocked = 0; in safexcel_rdr_intr()
245 if (ring->queued != 0) { in safexcel_rdr_intr()
248 SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | imin(ring->queued, 16)); in safexcel_rdr_intr()
250 mtx_unlock(&ring->mtx); in safexcel_rdr_intr()
253 crypto_unblock(sc->sc_cid, blocked); in safexcel_rdr_intr()
269 sc = ih->sc; in safexcel_ring_intr()
270 ring = ih->ring; in safexcel_ring_intr()
274 /* CDR interrupts */ in safexcel_ring_intr()
306 if (sc->sc_type == 197) { in safexcel_configure()
307 sc->sc_offsets = eip197_regs_offset; in safexcel_configure()
310 sc->sc_offsets = eip97_regs_offset; in safexcel_configure()
321 sc->sc_config.aic_rings = i; in safexcel_configure()
322 if (sc->sc_config.aic_rings == 0) in safexcel_configure()
323 return (-1); in safexcel_configure()
328 return (-1); in safexcel_configure()
332 return (-1); in safexcel_configure()
334 sc->sc_config.hdw = in safexcel_configure()
336 mask = (1 << sc->sc_config.hdw) - 1; in safexcel_configure()
338 sc->sc_config.rings = reg & SAFEXCEL_N_RINGS_MASK; in safexcel_configure()
340 sc->sc_config.rings = MIN(sc->sc_config.rings, sc->sc_config.aic_rings); in safexcel_configure()
342 sc->sc_config.pes = (reg & pemask) >> SAFEXCEL_N_PES_OFFSET; in safexcel_configure()
344 sc->sc_config.cd_size = in safexcel_configure()
346 sc->sc_config.cd_offset = (sc->sc_config.cd_size + mask) & ~mask; in safexcel_configure()
348 sc->sc_config.rd_size = in safexcel_configure()
350 sc->sc_config.rd_offset = (sc->sc_config.rd_size + mask) & ~mask; in safexcel_configure()
352 sc->sc_config.atok_offset = in safexcel_configure()
453 device_printf(sc->sc_dev, "DSE reset timeout\n"); in safexcel_configure_dse()
454 return (-1); in safexcel_configure_dse()
485 for (i = 0; i < sc->sc_config.rings; i++) { in safexcel_hw_prepare_rings()
516 SAFEXCEL_RING_SIZE * sc->sc_config.cd_offset * in safexcel_hw_prepare_rings()
545 SAFEXCEL_RING_SIZE * sc->sc_config.rd_offset * in safexcel_hw_prepare_rings()
557 mask = (1 << sc->sc_config.hdw) - 1; in safexcel_hw_setup_rings()
558 cd_size_rnd = (sc->sc_config.cd_size + mask) >> sc->sc_config.hdw; in safexcel_hw_setup_rings()
559 val = (sizeof(struct safexcel_res_descr) - in safexcel_hw_setup_rings()
561 rd_size_rnd = (val + mask) >> sc->sc_config.hdw; in safexcel_hw_setup_rings()
563 for (i = 0; i < sc->sc_config.rings; i++) { in safexcel_hw_setup_rings()
564 ring = &sc->sc_ring[i]; in safexcel_hw_setup_rings()
573 SAFEXCEL_ADDR_LO(ring->cdr.dma.paddr)); in safexcel_hw_setup_rings()
576 SAFEXCEL_ADDR_HI(ring->cdr.dma.paddr)); in safexcel_hw_setup_rings()
581 (sc->sc_config.cd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | in safexcel_hw_setup_rings()
582 sc->sc_config.cd_size); in safexcel_hw_setup_rings()
586 ((SAFEXCEL_FETCH_COUNT * (cd_size_rnd << sc->sc_config.hdw)) << in safexcel_hw_setup_rings()
588 (SAFEXCEL_FETCH_COUNT * sc->sc_config.cd_offset)); in safexcel_hw_setup_rings()
608 SAFEXCEL_ADDR_LO(ring->rdr.dma.paddr)); in safexcel_hw_setup_rings()
611 SAFEXCEL_ADDR_HI(ring->rdr.dma.paddr)); in safexcel_hw_setup_rings()
616 (sc->sc_config.rd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | in safexcel_hw_setup_rings()
617 sc->sc_config.rd_size); in safexcel_hw_setup_rings()
621 ((SAFEXCEL_FETCH_COUNT * (rd_size_rnd << sc->sc_config.hdw)) << in safexcel_hw_setup_rings()
623 (SAFEXCEL_FETCH_COUNT * sc->sc_config.rd_offset)); in safexcel_hw_setup_rings()
650 for (i = 0; i < sc->sc_config.rings; i++) { in safexcel_hw_reset_rings()
729 for (ring_mask = 0, i = 0; i < sc->sc_config.rings; i++) { in safexcel_enable_pe_engine()
754 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_execute()
757 ring->pending++; in safexcel_execute()
758 ring->pending_cdesc += req->cdescs; in safexcel_execute()
759 ring->pending_rdesc += req->rdescs; in safexcel_execute()
763 ringidx = req->ringidx; in safexcel_execute()
765 busy = ring->queued != 0; in safexcel_execute()
766 ncdesc = ring->pending_cdesc + req->cdescs; in safexcel_execute()
767 nrdesc = ring->pending_rdesc + req->rdescs; in safexcel_execute()
768 ring->queued += ring->pending + 1; in safexcel_execute()
773 SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | ring->queued); in safexcel_execute()
777 nrdesc * sc->sc_config.rd_offset * sizeof(uint32_t)); in safexcel_execute()
780 ncdesc * sc->sc_config.cd_offset * sizeof(uint32_t)); in safexcel_execute()
782 ring->pending = ring->pending_cdesc = ring->pending_rdesc = 0; in safexcel_execute()
793 for (i = 0; i < sc->sc_config.rings; i++) { in safexcel_init_rings()
794 ring = &sc->sc_ring[i]; in safexcel_init_rings()
796 snprintf(ring->lockname, sizeof(ring->lockname), in safexcel_init_rings()
798 mtx_init(&ring->mtx, ring->lockname, NULL, MTX_DEF); in safexcel_init_rings()
800 ring->pending = ring->pending_cdesc = ring->pending_rdesc = 0; in safexcel_init_rings()
801 ring->queued = 0; in safexcel_init_rings()
802 ring->cdr.read = ring->cdr.write = 0; in safexcel_init_rings()
803 ring->rdr.read = ring->rdr.write = 0; in safexcel_init_rings()
805 cdesc = &ring->cdr.desc[j]; in safexcel_init_rings()
806 atok = ring->dma_atok.paddr + in safexcel_init_rings()
807 sc->sc_config.atok_offset * j; in safexcel_init_rings()
808 cdesc->atok_lo = SAFEXCEL_ADDR_LO(atok); in safexcel_init_rings()
809 cdesc->atok_hi = SAFEXCEL_ADDR_HI(atok); in safexcel_init_rings()
825 sdm->paddr = segs->ds_addr; in safexcel_dma_alloc_mem_cb()
834 KASSERT(sdm->vaddr == NULL, in safexcel_dma_alloc_mem()
837 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ in safexcel_dma_alloc_mem()
845 &sdm->tag); /* dmat */ in safexcel_dma_alloc_mem()
847 device_printf(sc->sc_dev, in safexcel_dma_alloc_mem()
852 error = bus_dmamem_alloc(sdm->tag, (void **)&sdm->vaddr, in safexcel_dma_alloc_mem()
853 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sdm->map); in safexcel_dma_alloc_mem()
855 device_printf(sc->sc_dev, in safexcel_dma_alloc_mem()
860 error = bus_dmamap_load(sdm->tag, sdm->map, sdm->vaddr, size, in safexcel_dma_alloc_mem()
863 device_printf(sc->sc_dev, in safexcel_dma_alloc_mem()
870 bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); in safexcel_dma_alloc_mem()
872 bus_dma_tag_destroy(sdm->tag); in safexcel_dma_alloc_mem()
874 sdm->vaddr = NULL; in safexcel_dma_alloc_mem()
882 bus_dmamap_unload(sdm->tag, sdm->map); in safexcel_dma_free_mem()
883 bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); in safexcel_dma_free_mem()
884 bus_dma_tag_destroy(sdm->tag); in safexcel_dma_free_mem()
893 for (i = 0; i < sc->sc_config.rings; i++) { in safexcel_dma_free_rings()
894 ring = &sc->sc_ring[i]; in safexcel_dma_free_rings()
895 safexcel_dma_free_mem(&ring->cdr.dma); in safexcel_dma_free_rings()
896 safexcel_dma_free_mem(&ring->dma_atok); in safexcel_dma_free_rings()
897 safexcel_dma_free_mem(&ring->rdr.dma); in safexcel_dma_free_rings()
898 bus_dma_tag_destroy(ring->data_dtag); in safexcel_dma_free_rings()
899 mtx_destroy(&ring->mtx); in safexcel_dma_free_rings()
910 for (i = 0; i < sc->sc_config.rings; i++) { in safexcel_dma_init()
911 ring = &sc->sc_ring[i]; in safexcel_dma_init()
914 bus_get_dma_tag(sc->sc_dev),/* parent */ in safexcel_dma_init()
924 &ring->data_dtag); /* dmat */ in safexcel_dma_init()
926 device_printf(sc->sc_dev, in safexcel_dma_init()
931 size = sizeof(uint32_t) * sc->sc_config.cd_offset * in safexcel_dma_init()
933 error = safexcel_dma_alloc_mem(sc, &ring->cdr.dma, size); in safexcel_dma_init()
935 device_printf(sc->sc_dev, in safexcel_dma_init()
936 "failed to allocate CDR DMA memory, error %d\n", in safexcel_dma_init()
940 ring->cdr.desc = in safexcel_dma_init()
941 (struct safexcel_cmd_descr *)ring->cdr.dma.vaddr; in safexcel_dma_init()
943 /* Allocate additional CDR token memory. */ in safexcel_dma_init()
944 size = (bus_size_t)sc->sc_config.atok_offset * in safexcel_dma_init()
946 error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, size); in safexcel_dma_init()
948 device_printf(sc->sc_dev, in safexcel_dma_init()
954 size = sizeof(uint32_t) * sc->sc_config.rd_offset * in safexcel_dma_init()
956 error = safexcel_dma_alloc_mem(sc, &ring->rdr.dma, size); in safexcel_dma_init()
958 device_printf(sc->sc_dev, in safexcel_dma_init()
963 ring->rdr.desc = in safexcel_dma_init()
964 (struct safexcel_res_descr *)ring->rdr.dma.vaddr; in safexcel_dma_init()
996 /* 23.3.7.2 Disable EIP-97 global Interrupts */ in safexcel_init_hw()
999 for (pe = 0; pe < sc->sc_config.pes; pe++) { in safexcel_init_hw()
1006 return (-1); in safexcel_init_hw()
1021 for (pe = 0; pe < sc->sc_config.pes; pe++) in safexcel_init_hw()
1034 for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) { in safexcel_setup_dev_interrupts()
1035 sc->sc_ih[i].sc = sc; in safexcel_setup_dev_interrupts()
1036 sc->sc_ih[i].ring = i; in safexcel_setup_dev_interrupts()
1038 if (bus_setup_intr(sc->sc_dev, sc->sc_intr[i], in safexcel_setup_dev_interrupts()
1040 &sc->sc_ih[i], &sc->sc_ih[i].handle)) { in safexcel_setup_dev_interrupts()
1041 device_printf(sc->sc_dev, in safexcel_setup_dev_interrupts()
1046 error = bus_bind_intr(sc->sc_dev, sc->sc_intr[i], i % mp_ncpus); in safexcel_setup_dev_interrupts()
1048 device_printf(sc->sc_dev, in safexcel_setup_dev_interrupts()
1056 bus_teardown_intr(sc->sc_dev, sc->sc_intr[j], in safexcel_setup_dev_interrupts()
1057 sc->sc_ih[j].handle); in safexcel_setup_dev_interrupts()
1068 bus_teardown_intr(sc->sc_dev, sc->sc_intr[i], in safexcel_teardown_dev_interrupts()
1069 sc->sc_ih[i].handle); in safexcel_teardown_dev_interrupts()
1080 dev = sc->sc_dev; in safexcel_alloc_dev_resources()
1084 sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, in safexcel_alloc_dev_resources()
1086 if (sc->sc_res == NULL) { in safexcel_alloc_dev_resources()
1093 error = ofw_bus_find_string_index(node, "interrupt-names", name, in safexcel_alloc_dev_resources()
1098 sc->sc_intr[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, in safexcel_alloc_dev_resources()
1100 if (sc->sc_intr[i] == NULL) { in safexcel_alloc_dev_resources()
1114 for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) in safexcel_alloc_dev_resources()
1116 rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); in safexcel_alloc_dev_resources()
1117 bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), in safexcel_alloc_dev_resources()
1118 sc->sc_res); in safexcel_alloc_dev_resources()
1127 for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) in safexcel_free_dev_resources()
1128 bus_release_resource(sc->sc_dev, SYS_RES_IRQ, in safexcel_free_dev_resources()
1129 rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); in safexcel_free_dev_resources()
1130 if (sc->sc_res != NULL) in safexcel_free_dev_resources()
1131 bus_release_resource(sc->sc_dev, SYS_RES_MEMORY, in safexcel_free_dev_resources()
1132 rman_get_rid(sc->sc_res), sc->sc_res); in safexcel_free_dev_resources()
1144 sc->sc_type = ofw_bus_search_compatible(dev, safexcel_compat)->ocd_data; in safexcel_probe()
1145 if (sc->sc_type == 0) in safexcel_probe()
1148 device_set_desc(dev, "SafeXcel EIP-97 crypto accelerator"); in safexcel_probe()
1165 sc->sc_dev = dev; in safexcel_attach()
1166 sc->sc_cid = -1; in safexcel_attach()
1177 for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { in safexcel_attach()
1178 ring = &sc->sc_ring[ringidx]; in safexcel_attach()
1180 ring->cmd_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); in safexcel_attach()
1181 ring->res_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); in safexcel_attach()
1184 req = &ring->requests[i]; in safexcel_attach()
1185 req->sc = sc; in safexcel_attach()
1186 req->ringidx = ringidx; in safexcel_attach()
1187 if (bus_dmamap_create(ring->data_dtag, in safexcel_attach()
1188 BUS_DMA_COHERENT, &req->dmap) != 0) { in safexcel_attach()
1190 bus_dmamap_destroy(ring->data_dtag, in safexcel_attach()
1191 ring->requests[j].dmap); in safexcel_attach()
1194 if (safexcel_dma_alloc_mem(sc, &req->ctx, in safexcel_attach()
1197 bus_dmamap_destroy(ring->data_dtag, in safexcel_attach()
1198 ring->requests[j].dmap); in safexcel_attach()
1200 &ring->requests[j].ctx); in safexcel_attach()
1209 OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->sc_debug, 0, in safexcel_attach()
1212 oid = device_get_sysctl_tree(sc->sc_dev); in safexcel_attach()
1218 sc->sc_req_alloc_failures = counter_u64_alloc(M_WAITOK); in safexcel_attach()
1220 CTLFLAG_RD, &sc->sc_req_alloc_failures, in safexcel_attach()
1222 sc->sc_cdesc_alloc_failures = counter_u64_alloc(M_WAITOK); in safexcel_attach()
1224 CTLFLAG_RD, &sc->sc_cdesc_alloc_failures, in safexcel_attach()
1226 sc->sc_rdesc_alloc_failures = counter_u64_alloc(M_WAITOK); in safexcel_attach()
1228 CTLFLAG_RD, &sc->sc_rdesc_alloc_failures, in safexcel_attach()
1231 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safexcel_session), in safexcel_attach()
1233 if (sc->sc_cid < 0) in safexcel_attach()
1255 if (sc->sc_cid >= 0) in safexcel_detach()
1256 crypto_unregister_all(sc->sc_cid); in safexcel_detach()
1258 counter_u64_free(sc->sc_req_alloc_failures); in safexcel_detach()
1259 counter_u64_free(sc->sc_cdesc_alloc_failures); in safexcel_detach()
1260 counter_u64_free(sc->sc_rdesc_alloc_failures); in safexcel_detach()
1262 for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { in safexcel_detach()
1263 ring = &sc->sc_ring[ringidx]; in safexcel_detach()
1265 bus_dmamap_destroy(ring->data_dtag, in safexcel_detach()
1266 ring->requests[i].dmap); in safexcel_detach()
1267 safexcel_dma_free_mem(&ring->requests[i].ctx); in safexcel_detach()
1269 sglist_free(ring->cmd_data); in safexcel_detach()
1270 sglist_free(ring->res_data); in safexcel_detach()
1280 * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted
1301 * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3
1303 * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC
1323 switch (ahash->type) { in safexcel_setkey_hmac_digest()
1325 hashwords = ahash->hashsize / sizeof(uint32_t); in safexcel_setkey_hmac_digest()
1327 ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); in safexcel_setkey_hmac_digest()
1332 ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); in safexcel_setkey_hmac_digest()
1335 hashwords = ahash->hashsize / sizeof(uint32_t); in safexcel_setkey_hmac_digest()
1337 ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); in safexcel_setkey_hmac_digest()
1342 ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); in safexcel_setkey_hmac_digest()
1345 hashwords = ahash->hashsize / sizeof(uint64_t); in safexcel_setkey_hmac_digest()
1347 ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); in safexcel_setkey_hmac_digest()
1353 * Pre-compute the inner and outer digests used in the HMAC algorithm.
1367 explicit_bzero(&ctx, ahash->ctxsize); in safexcel_setkey_hmac()
1378 * specify per-request keys, in which case the context must be re-initialized
1390 csp = crypto_get_params(sess->cses); in safexcel_set_context()
1391 aklen = csp->csp_auth_klen; in safexcel_set_context()
1392 cklen = csp->csp_cipher_klen; in safexcel_set_context()
1393 if (csp->csp_cipher_alg == CRYPTO_AES_XTS) in safexcel_set_context()
1396 ctrl0 = sess->alg | sess->digest | sess->hash; in safexcel_set_context()
1397 ctrl1 = sess->mode; in safexcel_set_context()
1399 data = (uint8_t *)ctx->data; in safexcel_set_context()
1400 if (csp->csp_cipher_alg != 0) { in safexcel_set_context()
1403 } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { in safexcel_set_context()
1410 switch (csp->csp_cipher_alg) { in safexcel_set_context()
1424 switch (csp->csp_auth_alg) { in safexcel_set_context()
1435 data + off, data + off + sess->statelen); in safexcel_set_context()
1436 off += sess->statelen * 2; in safexcel_set_context()
1441 alg = csp->csp_cipher_alg; in safexcel_set_context()
1443 alg = csp->csp_auth_alg; in safexcel_set_context()
1463 if (csp->csp_auth_alg != 0) in safexcel_set_context()
1469 if (csp->csp_auth_alg != 0) in safexcel_set_context()
1475 if (CRYPTO_OP_IS_ENCRYPT(op) || csp->csp_auth_alg != 0) { in safexcel_set_context()
1484 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { in safexcel_set_context()
1506 ctx->control0 = ctrl0; in safexcel_set_context()
1507 ctx->control1 = ctrl1; in safexcel_set_context()
1513 * Construct a no-op instruction, used to pad input tokens.
1521 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; in safexcel_instr_nop()
1522 instr->length = (1 << 2); in safexcel_instr_nop()
1523 instr->status = 0; in safexcel_instr_nop()
1524 instr->instructions = 0; in safexcel_instr_nop()
1539 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; in safexcel_instr_insert_digest()
1540 instr->length = len; in safexcel_instr_insert_digest()
1541 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | in safexcel_instr_insert_digest()
1543 instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | in safexcel_instr_insert_digest()
1558 instr->opcode = SAFEXCEL_INSTR_OPCODE_RETRIEVE; in safexcel_instr_retrieve_digest()
1559 instr->length = len; in safexcel_instr_retrieve_digest()
1560 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | in safexcel_instr_retrieve_digest()
1562 instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST; in safexcel_instr_retrieve_digest()
1565 instr->opcode = SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS; in safexcel_instr_retrieve_digest()
1566 instr->length = len | SAFEXCEL_INSTR_VERIFY_HASH; in safexcel_instr_retrieve_digest()
1567 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH | in safexcel_instr_retrieve_digest()
1569 instr->instructions = SAFEXCEL_INSTR_VERIFY_PADDING; in safexcel_instr_retrieve_digest()
1580 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT; in safexcel_instr_temp_aes_block()
1581 instr->length = 0; in safexcel_instr_temp_aes_block()
1582 instr->status = 0; in safexcel_instr_temp_aes_block()
1583 instr->instructions = AES_BLOCK_LEN; in safexcel_instr_temp_aes_block()
1586 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; in safexcel_instr_temp_aes_block()
1587 instr->length = AES_BLOCK_LEN; in safexcel_instr_temp_aes_block()
1588 instr->status = 0; in safexcel_instr_temp_aes_block()
1589 instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | in safexcel_instr_temp_aes_block()
1604 crp = req->crp; in safexcel_instr_cipher()
1607 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_cipher()
1608 instr->length = crp->crp_payload_length; in safexcel_instr_cipher()
1609 instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET | in safexcel_instr_cipher()
1611 instr->instructions = SAFEXCEL_INSTR_INS_LAST | in safexcel_instr_cipher()
1614 cdesc->additional_cdata_size = 1; in safexcel_instr_cipher()
1624 crp = req->crp; in safexcel_instr_eta()
1628 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_eta()
1629 instr->length = crp->crp_aad_length; in safexcel_instr_eta()
1630 instr->status = crp->crp_payload_length == 0 ? in safexcel_instr_eta()
1632 instr->instructions = SAFEXCEL_INSTR_INS_LAST | in safexcel_instr_eta()
1637 if (crp->crp_payload_length > 0) { in safexcel_instr_eta()
1638 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_eta()
1639 instr->length = crp->crp_payload_length; in safexcel_instr_eta()
1640 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; in safexcel_instr_eta()
1641 instr->instructions = SAFEXCEL_INSTR_INS_LAST | in safexcel_instr_eta()
1651 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) in safexcel_instr_eta()
1652 safexcel_instr_insert_digest(&instr, req->sess->digestlen); in safexcel_instr_eta()
1654 safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); in safexcel_instr_eta()
1655 cdesc->additional_cdata_size = instr - start; in safexcel_instr_eta()
1665 crp = req->crp; in safexcel_instr_sha_hash()
1669 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_sha_hash()
1670 instr->length = crp->crp_payload_length; in safexcel_instr_sha_hash()
1671 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; in safexcel_instr_sha_hash()
1672 instr->instructions = SAFEXCEL_INSTR_DEST_HASH; in safexcel_instr_sha_hash()
1676 safexcel_instr_insert_digest(&instr, req->sess->digestlen); in safexcel_instr_sha_hash()
1693 crp = req->crp; in safexcel_instr_ccm()
1694 csp = crypto_get_params(crp->crp_session); in safexcel_instr_ccm()
1705 * the smallest length field used for a 13-byte nonce. in safexcel_instr_ccm()
1708 L = 15 - csp->csp_ivlen; in safexcel_instr_ccm()
1710 a0 = (uint8_t *)&cdesc->control_data.token[0]; in safexcel_instr_ccm()
1712 a0[0] = L - 1; in safexcel_instr_ccm()
1713 memcpy(&a0[1], req->iv, csp->csp_ivlen); in safexcel_instr_ccm()
1718 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; in safexcel_instr_ccm()
1719 instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0); in safexcel_instr_ccm()
1720 instr->status = 0; in safexcel_instr_ccm()
1721 instr->instructions = SAFEXCEL_INSTR_DEST_HASH | in safexcel_instr_ccm()
1728 (L - 1) | /* payload length size */ in safexcel_instr_ccm()
1729 ((req->sess->digestlen - 2) / 2) << 3 /* digest length */ | in safexcel_instr_ccm()
1730 (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */; in safexcel_instr_ccm()
1731 memcpy(&b0[1], req->iv, csp->csp_ivlen); in safexcel_instr_ccm()
1732 b0[14] = crp->crp_payload_length >> 8; in safexcel_instr_ccm()
1733 b0[15] = crp->crp_payload_length & 0xff; in safexcel_instr_ccm()
1737 if (crp->crp_aad_length > 0) { in safexcel_instr_ccm()
1739 alenp[0] = crp->crp_aad_length >> 8; in safexcel_instr_ccm()
1740 alenp[1] = crp->crp_aad_length & 0xff; in safexcel_instr_ccm()
1745 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_ccm()
1746 instr->length = crp->crp_aad_length; in safexcel_instr_ccm()
1747 instr->status = 0; in safexcel_instr_ccm()
1748 instr->instructions = SAFEXCEL_INSTR_DEST_HASH; in safexcel_instr_ccm()
1752 aalign = (crp->crp_aad_length + 2) & (blen - 1); in safexcel_instr_ccm()
1753 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; in safexcel_instr_ccm()
1754 instr->length = aalign == 0 ? 0 : in safexcel_instr_ccm()
1755 blen - ((crp->crp_aad_length + 2) & (blen - 1)); in safexcel_instr_ccm()
1756 instr->status = crp->crp_payload_length == 0 ? in safexcel_instr_ccm()
1758 instr->instructions = SAFEXCEL_INSTR_DEST_HASH; in safexcel_instr_ccm()
1765 if (crp->crp_payload_length > 0) { in safexcel_instr_ccm()
1766 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_ccm()
1767 instr->length = crp->crp_payload_length; in safexcel_instr_ccm()
1768 instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ? in safexcel_instr_ccm()
1770 instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | in safexcel_instr_ccm()
1777 if (crp->crp_payload_length & (blen - 1)) { in safexcel_instr_ccm()
1778 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; in safexcel_instr_ccm()
1779 instr->length = blen - in safexcel_instr_ccm()
1780 (crp->crp_payload_length & (blen - 1)); in safexcel_instr_ccm()
1781 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; in safexcel_instr_ccm()
1782 instr->instructions = SAFEXCEL_INSTR_DEST_HASH; in safexcel_instr_ccm()
1790 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) in safexcel_instr_ccm()
1791 safexcel_instr_insert_digest(&instr, req->sess->digestlen); in safexcel_instr_ccm()
1793 safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); in safexcel_instr_ccm()
1795 cdesc->additional_cdata_size = instr - start; in safexcel_instr_ccm()
1805 memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); in safexcel_instr_gcm()
1806 cdesc->control_data.token[3] = htobe32(1); in safexcel_instr_gcm()
1808 crp = req->crp; in safexcel_instr_gcm()
1812 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_gcm()
1813 instr->length = crp->crp_aad_length; in safexcel_instr_gcm()
1814 instr->status = crp->crp_payload_length == 0 ? in safexcel_instr_gcm()
1816 instr->instructions = SAFEXCEL_INSTR_INS_LAST | in safexcel_instr_gcm()
1823 if (crp->crp_payload_length > 0) { in safexcel_instr_gcm()
1824 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_gcm()
1825 instr->length = crp->crp_payload_length; in safexcel_instr_gcm()
1826 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; in safexcel_instr_gcm()
1827 instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | in safexcel_instr_gcm()
1836 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) in safexcel_instr_gcm()
1837 safexcel_instr_insert_digest(&instr, req->sess->digestlen); in safexcel_instr_gcm()
1839 safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); in safexcel_instr_gcm()
1841 cdesc->additional_cdata_size = instr - start; in safexcel_instr_gcm()
1851 memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); in safexcel_instr_gmac()
1852 cdesc->control_data.token[3] = htobe32(1); in safexcel_instr_gmac()
1854 crp = req->crp; in safexcel_instr_gmac()
1857 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; in safexcel_instr_gmac()
1858 instr->length = crp->crp_payload_length; in safexcel_instr_gmac()
1859 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; in safexcel_instr_gmac()
1860 instr->instructions = SAFEXCEL_INSTR_INS_LAST | in safexcel_instr_gmac()
1866 safexcel_instr_insert_digest(&instr, req->sess->digestlen); in safexcel_instr_gmac()
1868 cdesc->additional_cdata_size = instr - start; in safexcel_instr_gmac()
1884 crp = req->crp; in safexcel_set_token()
1885 csp = crypto_get_params(crp->crp_session); in safexcel_set_token()
1886 cdesc = req->cdesc; in safexcel_set_token()
1887 sc = req->sc; in safexcel_set_token()
1888 ringidx = req->ringidx; in safexcel_set_token()
1890 akey = crp->crp_auth_key; in safexcel_set_token()
1891 ckey = crp->crp_cipher_key; in safexcel_set_token()
1894 * If we have a per-request key we have to generate the context in safexcel_set_token()
1898 akey = csp->csp_auth_key; in safexcel_set_token()
1900 ckey = csp->csp_cipher_key; in safexcel_set_token()
1901 ctx = (struct safexcel_context_record *)req->ctx.vaddr; in safexcel_set_token()
1902 (void)safexcel_set_context(ctx, crp->crp_op, ckey, akey, in safexcel_set_token()
1903 req->sess); in safexcel_set_token()
1909 ctxtmp = CRYPTO_OP_IS_ENCRYPT(crp->crp_op) ? in safexcel_set_token()
1910 &req->sess->encctx : &req->sess->decctx; in safexcel_set_token()
1911 ctx = &ctxtmp->ctx; in safexcel_set_token()
1912 memcpy(req->ctx.vaddr + 2 * sizeof(uint32_t), ctx->data, in safexcel_set_token()
1913 ctxtmp->len); in safexcel_set_token()
1915 cdesc->control_data.control0 = ctx->control0; in safexcel_set_token()
1916 cdesc->control_data.control1 = ctx->control1; in safexcel_set_token()
1923 if (csp->csp_cipher_alg == 0 && in safexcel_set_token()
1924 csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { in safexcel_set_token()
1925 instr = (void *)cdesc->control_data.token; in safexcel_set_token()
1927 instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr + in safexcel_set_token()
1928 sc->sc_config.atok_offset * in safexcel_set_token()
1929 (cdesc - sc->sc_ring[ringidx].cdr.desc)); in safexcel_set_token()
1930 cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD; in safexcel_set_token()
1933 switch (csp->csp_cipher_alg) { in safexcel_set_token()
1941 memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); in safexcel_set_token()
1942 memset(cdesc->control_data.token + in safexcel_set_token()
1949 memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); in safexcel_set_token()
1950 if (csp->csp_auth_alg != 0) in safexcel_set_token()
1956 switch (csp->csp_auth_alg) { in safexcel_set_token()
1973 panic("unhandled auth request %d", csp->csp_auth_alg); in safexcel_set_token()
1986 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_res_descr_add()
1988 rring = &ring->rdr; in safexcel_res_descr_add()
1989 if ((rring->write + 1) % SAFEXCEL_RING_SIZE == rring->read) in safexcel_res_descr_add()
1992 rdesc = &rring->desc[rring->write]; in safexcel_res_descr_add()
1993 rring->write = (rring->write + 1) % SAFEXCEL_RING_SIZE; in safexcel_res_descr_add()
1995 rdesc->particle_size = len; in safexcel_res_descr_add()
1996 rdesc->rsvd0 = 0; in safexcel_res_descr_add()
1997 rdesc->descriptor_overflow = 0; in safexcel_res_descr_add()
1998 rdesc->buffer_overflow = 0; in safexcel_res_descr_add()
1999 rdesc->last_seg = last; in safexcel_res_descr_add()
2000 rdesc->first_seg = first; in safexcel_res_descr_add()
2001 rdesc->result_size = in safexcel_res_descr_add()
2003 rdesc->rsvd1 = 0; in safexcel_res_descr_add()
2004 rdesc->data_lo = SAFEXCEL_ADDR_LO(data); in safexcel_res_descr_add()
2005 rdesc->data_hi = SAFEXCEL_ADDR_HI(data); in safexcel_res_descr_add()
2008 rdesc->result_data.packet_length = 0; in safexcel_res_descr_add()
2009 rdesc->result_data.error_code = 0; in safexcel_res_descr_add()
2024 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_cmd_descr_add()
2026 cring = &ring->cdr; in safexcel_cmd_descr_add()
2027 if ((cring->write + 1) % SAFEXCEL_RING_SIZE == cring->read) in safexcel_cmd_descr_add()
2030 cdesc = &cring->desc[cring->write]; in safexcel_cmd_descr_add()
2031 cring->write = (cring->write + 1) % SAFEXCEL_RING_SIZE; in safexcel_cmd_descr_add()
2033 cdesc->particle_size = seglen; in safexcel_cmd_descr_add()
2034 cdesc->rsvd0 = 0; in safexcel_cmd_descr_add()
2035 cdesc->last_seg = last; in safexcel_cmd_descr_add()
2036 cdesc->first_seg = first; in safexcel_cmd_descr_add()
2037 cdesc->additional_cdata_size = 0; in safexcel_cmd_descr_add()
2038 cdesc->rsvd1 = 0; in safexcel_cmd_descr_add()
2039 cdesc->data_lo = SAFEXCEL_ADDR_LO(data); in safexcel_cmd_descr_add()
2040 cdesc->data_hi = SAFEXCEL_ADDR_HI(data); in safexcel_cmd_descr_add()
2042 cdesc->control_data.packet_length = reqlen; in safexcel_cmd_descr_add()
2043 cdesc->control_data.options = SAFEXCEL_OPTION_IP | in safexcel_cmd_descr_add()
2046 cdesc->control_data.type = SAFEXCEL_TOKEN_TYPE_BYPASS; in safexcel_cmd_descr_add()
2047 cdesc->control_data.context_lo = SAFEXCEL_ADDR_LO(context) | in safexcel_cmd_descr_add()
2049 cdesc->control_data.context_hi = SAFEXCEL_ADDR_HI(context); in safexcel_cmd_descr_add()
2060 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_cmd_descr_rollback()
2062 cring = &ring->cdr; in safexcel_cmd_descr_rollback()
2063 cring->write -= count; in safexcel_cmd_descr_rollback()
2064 if (cring->write < 0) in safexcel_cmd_descr_rollback()
2065 cring->write += SAFEXCEL_RING_SIZE; in safexcel_cmd_descr_rollback()
2073 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_res_descr_rollback()
2075 rring = &ring->rdr; in safexcel_res_descr_rollback()
2076 rring->write -= count; in safexcel_res_descr_rollback()
2077 if (rring->write < 0) in safexcel_res_descr_rollback()
2078 rring->write += SAFEXCEL_RING_SIZE; in safexcel_res_descr_rollback()
2092 if (seg->ds_len <= start) { in safexcel_append_segs()
2093 start -= seg->ds_len; in safexcel_append_segs()
2097 seglen = MIN(len, seg->ds_len - start); in safexcel_append_segs()
2098 error = sglist_append_phys(sg, seg->ds_addr + start, seglen); in safexcel_append_segs()
2101 len -= seglen; in safexcel_append_segs()
2123 req->error = error; in safexcel_create_chain_cb()
2127 crp = req->crp; in safexcel_create_chain_cb()
2128 csp = crypto_get_params(crp->crp_session); in safexcel_create_chain_cb()
2129 sess = req->sess; in safexcel_create_chain_cb()
2130 ring = &req->sc->sc_ring[req->ringidx]; in safexcel_create_chain_cb()
2132 mtx_assert(&ring->mtx, MA_OWNED); in safexcel_create_chain_cb()
2145 sglist_reset(ring->cmd_data); in safexcel_create_chain_cb()
2146 sglist_reset(ring->res_data); in safexcel_create_chain_cb()
2147 if (crp->crp_aad_length != 0) { in safexcel_create_chain_cb()
2148 safexcel_append_segs(segs, nseg, ring->cmd_data, in safexcel_create_chain_cb()
2149 crp->crp_aad_start, crp->crp_aad_length); in safexcel_create_chain_cb()
2151 safexcel_append_segs(segs, nseg, ring->cmd_data, in safexcel_create_chain_cb()
2152 crp->crp_payload_start, crp->crp_payload_length); in safexcel_create_chain_cb()
2153 if (csp->csp_cipher_alg != 0) { in safexcel_create_chain_cb()
2154 safexcel_append_segs(segs, nseg, ring->res_data, in safexcel_create_chain_cb()
2155 crp->crp_payload_start, crp->crp_payload_length); in safexcel_create_chain_cb()
2157 if (sess->digestlen > 0) { in safexcel_create_chain_cb()
2158 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { in safexcel_create_chain_cb()
2159 safexcel_append_segs(segs, nseg, ring->cmd_data, in safexcel_create_chain_cb()
2160 crp->crp_digest_start, sess->digestlen); in safexcel_create_chain_cb()
2162 safexcel_append_segs(segs, nseg, ring->res_data, in safexcel_create_chain_cb()
2163 crp->crp_digest_start, sess->digestlen); in safexcel_create_chain_cb()
2167 sg = ring->cmd_data; in safexcel_create_chain_cb()
2168 if (sg->sg_nseg == 0) { in safexcel_create_chain_cb()
2172 * zero-length packets properly since subsequent requests return in safexcel_create_chain_cb()
2177 (void)sglist_append_phys(sg, req->ctx.paddr, 1); in safexcel_create_chain_cb()
2179 for (i = 0, inlen = 0; i < sg->sg_nseg; i++) in safexcel_create_chain_cb()
2180 inlen += sg->sg_segs[i].ss_len; in safexcel_create_chain_cb()
2181 for (i = 0; i < sg->sg_nseg; i++) { in safexcel_create_chain_cb()
2183 last = i == sg->sg_nseg - 1; in safexcel_create_chain_cb()
2186 sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len, in safexcel_create_chain_cb()
2187 (uint32_t)inlen, req->ctx.paddr); in safexcel_create_chain_cb()
2190 counter_u64_add(req->sc->sc_cdesc_alloc_failures, 1); in safexcel_create_chain_cb()
2191 req->error = ERESTART; in safexcel_create_chain_cb()
2195 req->cdesc = cdesc; in safexcel_create_chain_cb()
2197 req->cdescs = sg->sg_nseg; in safexcel_create_chain_cb()
2199 sg = ring->res_data; in safexcel_create_chain_cb()
2200 if (sg->sg_nseg == 0) { in safexcel_create_chain_cb()
2205 sg->sg_segs[0].ss_paddr = 0; in safexcel_create_chain_cb()
2206 sg->sg_segs[0].ss_len = 0; in safexcel_create_chain_cb()
2207 sg->sg_nseg = 1; in safexcel_create_chain_cb()
2209 for (i = 0; i < sg->sg_nseg; i++) { in safexcel_create_chain_cb()
2211 last = i == sg->sg_nseg - 1; in safexcel_create_chain_cb()
2214 sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len) == NULL) { in safexcel_create_chain_cb()
2216 ring->cmd_data->sg_nseg); in safexcel_create_chain_cb()
2218 counter_u64_add(req->sc->sc_rdesc_alloc_failures, 1); in safexcel_create_chain_cb()
2219 req->error = ERESTART; in safexcel_create_chain_cb()
2223 req->rdescs = sg->sg_nseg; in safexcel_create_chain_cb()
2231 req->error = 0; in safexcel_create_chain()
2232 req->cdescs = req->rdescs = 0; in safexcel_create_chain()
2234 error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp, in safexcel_create_chain()
2237 req->dmap_loaded = true; in safexcel_create_chain()
2239 if (req->error != 0) in safexcel_create_chain()
2240 error = req->error; in safexcel_create_chain()
2248 switch (csp->csp_cipher_alg) { in safexcel_probe_cipher()
2251 if (csp->csp_ivlen != AES_BLOCK_LEN) in safexcel_probe_cipher()
2255 if (csp->csp_ivlen != AES_XTS_IV_LEN) in safexcel_probe_cipher()
2272 if (csp->csp_flags != 0) in safexcel_probesession()
2275 switch (csp->csp_mode) { in safexcel_probesession()
2281 switch (csp->csp_auth_alg) { in safexcel_probesession()
2283 if (csp->csp_ivlen != AES_GCM_IV_LEN) in safexcel_probesession()
2302 switch (csp->csp_cipher_alg) { in safexcel_probesession()
2313 switch (csp->csp_cipher_alg) { in safexcel_probesession()
2317 * The EIP-97 does not support combining AES-XTS with in safexcel_probesession()
2320 if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC && in safexcel_probesession()
2321 csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC && in safexcel_probesession()
2322 csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC && in safexcel_probesession()
2323 csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC && in safexcel_probesession()
2324 csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC) in safexcel_probesession()
2445 sess->cses = cses; in safexcel_newsession()
2447 switch (csp->csp_auth_alg) { in safexcel_newsession()
2453 sess->digest = SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED; in safexcel_newsession()
2454 sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); in safexcel_newsession()
2455 sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); in safexcel_newsession()
2456 sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); in safexcel_newsession()
2463 sess->digest = SAFEXCEL_CONTROL0_DIGEST_HMAC; in safexcel_newsession()
2464 sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); in safexcel_newsession()
2465 sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); in safexcel_newsession()
2466 sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); in safexcel_newsession()
2469 sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; in safexcel_newsession()
2470 sess->digestlen = GMAC_DIGEST_LEN; in safexcel_newsession()
2471 sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; in safexcel_newsession()
2472 sess->alg = safexcel_aes_algid(csp->csp_auth_klen); in safexcel_newsession()
2473 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; in safexcel_newsession()
2477 switch (csp->csp_cipher_alg) { in safexcel_newsession()
2479 sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC; in safexcel_newsession()
2480 sess->digestlen = GMAC_DIGEST_LEN; in safexcel_newsession()
2481 sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH; in safexcel_newsession()
2482 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); in safexcel_newsession()
2483 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM; in safexcel_newsession()
2486 sess->hash = safexcel_aes_ccm_hashid(csp->csp_cipher_klen); in safexcel_newsession()
2487 sess->digest = SAFEXCEL_CONTROL0_DIGEST_CCM; in safexcel_newsession()
2488 sess->digestlen = CCM_CBC_MAX_DIGEST_LEN; in safexcel_newsession()
2489 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); in safexcel_newsession()
2490 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM; in safexcel_newsession()
2493 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); in safexcel_newsession()
2494 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC; in safexcel_newsession()
2497 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); in safexcel_newsession()
2498 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR; in safexcel_newsession()
2501 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen / 2); in safexcel_newsession()
2502 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS; in safexcel_newsession()
2506 if (csp->csp_auth_mlen != 0) in safexcel_newsession()
2507 sess->digestlen = csp->csp_auth_mlen; in safexcel_newsession()
2509 sess->encctx.len = safexcel_set_context(&sess->encctx.ctx, in safexcel_newsession()
2510 CRYPTO_OP_ENCRYPT, csp->csp_cipher_key, csp->csp_auth_key, in safexcel_newsession()
2512 sess->decctx.len = safexcel_set_context(&sess->decctx.ctx, in safexcel_newsession()
2513 CRYPTO_OP_DECRYPT, csp->csp_cipher_key, csp->csp_auth_key, in safexcel_newsession()
2529 sess = crypto_get_driver_session(crp->crp_session); in safexcel_process()
2531 if (__predict_false(crypto_buffer_len(&crp->crp_buf) > in safexcel_process()
2533 crp->crp_etype = E2BIG; in safexcel_process()
2538 ring = &sc->sc_ring[curcpu % sc->sc_config.rings]; in safexcel_process()
2539 mtx_lock(&ring->mtx); in safexcel_process()
2542 ring->blocked = CRYPTO_SYMQ; in safexcel_process()
2543 mtx_unlock(&ring->mtx); in safexcel_process()
2544 counter_u64_add(sc->sc_req_alloc_failures, 1); in safexcel_process()
2548 req->crp = crp; in safexcel_process()
2549 req->sess = sess; in safexcel_process()
2551 crypto_read_iv(crp, req->iv); in safexcel_process()
2557 ring->blocked = CRYPTO_SYMQ; in safexcel_process()
2558 mtx_unlock(&ring->mtx); in safexcel_process()
2560 crp->crp_etype = error; in safexcel_process()
2570 bus_dmamap_sync(ring->data_dtag, req->dmap, in safexcel_process()
2572 bus_dmamap_sync(req->ctx.tag, req->ctx.map, in safexcel_process()
2574 bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, in safexcel_process()
2576 bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, in safexcel_process()
2578 bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, in safexcel_process()
2583 mtx_unlock(&ring->mtx); in safexcel_process()