Lines Matching +full:dma +full:- +full:safe +full:- +full:map
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
32 * SafeNet SafeXcel-1141 hardware crypto accelerator
71 #include <dev/safe/safereg.h>
72 #include <dev/safe/safevar.h>
111 "safe",
116 DRIVER_MODULE(safe, pci, safe_driver, 0, 0);
117 MODULE_DEPEND(safe, crypto, 1, 1, 1);
119 MODULE_DEPEND(safe, rndtest, 1, 1, 1);
133 bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
146 static SYSCTL_NODE(_hw, OID_AUTO, safe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
156 static struct safe_softc *safec; /* for use by hw.safe.dump */
167 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
170 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
181 0, "RNG polling buffer size (32-bit words)");
200 switch (pci_get_vendor(sc->sc_dev)) { in safe_partname()
202 switch (pci_get_device(sc->sc_dev)) { in safe_partname()
203 case PCI_PRODUCT_SAFEXCEL: return "SafeNet SafeXcel-1141"; in safe_partname()
205 return "SafeNet unknown-part"; in safe_partname()
207 return "Unknown-vendor unknown-part"; in safe_partname()
228 sc->sc_dev = dev; in safe_attach()
235 * Setup memory-mapping of PCI registers. in safe_attach()
238 sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, in safe_attach()
240 if (sc->sc_sr == NULL) { in safe_attach()
241 device_printf(dev, "cannot map register space\n"); in safe_attach()
244 sc->sc_st = rman_get_bustag(sc->sc_sr); in safe_attach()
245 sc->sc_sh = rman_get_bushandle(sc->sc_sr); in safe_attach()
251 sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, in safe_attach()
253 if (sc->sc_irq == NULL) { in safe_attach()
254 device_printf(dev, "could not map interrupt\n"); in safe_attach()
261 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, in safe_attach()
262 NULL, safe_intr, sc, &sc->sc_ih)) { in safe_attach()
267 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safe_session), in safe_attach()
269 if (sc->sc_cid < 0) { in safe_attach()
274 sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) & in safe_attach()
278 * Setup DMA descriptor area. in safe_attach()
291 &sc->sc_srcdmat)) { in safe_attach()
292 device_printf(dev, "cannot allocate DMA tag\n"); in safe_attach()
306 &sc->sc_dstdmat)) { in safe_attach()
307 device_printf(dev, "cannot allocate DMA tag\n"); in safe_attach()
316 &sc->sc_ringalloc, 0)) { in safe_attach()
318 bus_dma_tag_destroy(sc->sc_srcdmat); in safe_attach()
324 sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr; in safe_attach()
325 sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE; in safe_attach()
326 sc->sc_front = sc->sc_ring; in safe_attach()
327 sc->sc_back = sc->sc_ring; in safe_attach()
328 raddr = sc->sc_ringalloc.dma_paddr; in safe_attach()
329 bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry)); in safe_attach()
331 struct safe_ringentry *re = &sc->sc_ring[i]; in safe_attach()
333 re->re_desc.d_sa = raddr + in safe_attach()
335 re->re_sa.sa_staterec = raddr + in safe_attach()
340 mtx_init(&sc->sc_ringmtx, device_get_nameunit(dev), in safe_attach()
347 &sc->sc_spalloc, 0)) { in safe_attach()
350 mtx_destroy(&sc->sc_ringmtx); in safe_attach()
351 safe_dma_free(sc, &sc->sc_ringalloc); in safe_attach()
352 bus_dma_tag_destroy(sc->sc_srcdmat); in safe_attach()
355 sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr; in safe_attach()
356 sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART; in safe_attach()
357 sc->sc_spfree = sc->sc_spring; in safe_attach()
358 bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc)); in safe_attach()
361 &sc->sc_dpalloc, 0)) { in safe_attach()
364 mtx_destroy(&sc->sc_ringmtx); in safe_attach()
365 safe_dma_free(sc, &sc->sc_spalloc); in safe_attach()
366 safe_dma_free(sc, &sc->sc_ringalloc); in safe_attach()
367 bus_dma_tag_destroy(sc->sc_dstdmat); in safe_attach()
370 sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr; in safe_attach()
371 sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART; in safe_attach()
372 sc->sc_dpfree = sc->sc_dpring; in safe_attach()
373 bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc)); in safe_attach()
375 device_printf(sc->sc_dev, "%s", safe_partname(sc)); in safe_attach()
377 sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO); in safe_attach()
378 if (sc->sc_devinfo & SAFE_DEVINFO_RNG) { in safe_attach()
379 sc->sc_flags |= SAFE_FLAGS_RNG; in safe_attach()
382 if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) { in safe_attach()
385 sc->sc_flags |= SAFE_FLAGS_KEY; in safe_attach()
388 if (sc->sc_devinfo & SAFE_DEVINFO_DES) { in safe_attach()
391 if (sc->sc_devinfo & SAFE_DEVINFO_AES) { in safe_attach()
394 if (sc->sc_devinfo & SAFE_DEVINFO_MD5) { in safe_attach()
397 if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) { in safe_attach()
408 if (sc->sc_flags & SAFE_FLAGS_RNG) { in safe_attach()
410 sc->sc_rndtest = rndtest_attach(dev); in safe_attach()
411 if (sc->sc_rndtest) in safe_attach()
412 sc->sc_harvest = rndtest_harvest; in safe_attach()
414 sc->sc_harvest = default_harvest; in safe_attach()
416 sc->sc_harvest = default_harvest; in safe_attach()
420 callout_init(&sc->sc_rngto, 1); in safe_attach()
421 callout_reset(&sc->sc_rngto, hz*safe_rnginterval, safe_rng, sc); in safe_attach()
425 safec = sc; /* for use by hw.safe.dump */ in safe_attach()
429 crypto_unregister_all(sc->sc_cid); in safe_attach()
431 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); in safe_attach()
433 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); in safe_attach()
435 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); in safe_attach()
452 callout_stop(&sc->sc_rngto); in safe_detach()
454 crypto_unregister_all(sc->sc_cid); in safe_detach()
457 if (sc->sc_rndtest) in safe_detach()
458 rndtest_detach(sc->sc_rndtest); in safe_detach()
462 safe_dma_free(sc, &sc->sc_dpalloc); in safe_detach()
463 safe_dma_free(sc, &sc->sc_spalloc); in safe_detach()
464 mtx_destroy(&sc->sc_ringmtx); in safe_detach()
465 safe_dma_free(sc, &sc->sc_ringalloc); in safe_detach()
467 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); in safe_detach()
468 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); in safe_detach()
470 bus_dma_tag_destroy(sc->sc_srcdmat); in safe_detach()
471 bus_dma_tag_destroy(sc->sc_dstdmat); in safe_detach()
472 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); in safe_detach()
501 sc->sc_suspended = 1; in safe_suspend()
514 sc->sc_suspended = 0; in safe_resume()
538 mtx_lock(&sc->sc_ringmtx); in safe_intr()
539 while (sc->sc_back != sc->sc_front) { in safe_intr()
540 struct safe_ringentry *re = sc->sc_back; in safe_intr()
554 if (re->re_desc.d_csr != 0) { in safe_intr()
555 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr)) in safe_intr()
557 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len)) in safe_intr()
559 sc->sc_nqchip--; in safe_intr()
562 if (++(sc->sc_back) == sc->sc_ringtop) in safe_intr()
563 sc->sc_back = sc->sc_ring; in safe_intr()
565 mtx_unlock(&sc->sc_ringmtx); in safe_intr()
569 * Check to see if we got any DMA Error in safe_intr()
581 if (sc->sc_needwakeup) { /* XXX check high watermark */ in safe_intr()
582 int wakeup = sc->sc_needwakeup & CRYPTO_SYMQ; in safe_intr()
584 sc->sc_needwakeup)); in safe_intr()
585 sc->sc_needwakeup &= ~wakeup; in safe_intr()
586 crypto_unblock(sc->sc_cid, wakeup); in safe_intr()
591 * safe_feed() - post a request to chip
596 bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE); in safe_feed()
597 if (re->re_dst_map != NULL) in safe_feed()
598 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map, in safe_feed()
601 safe_dma_sync(&sc->sc_ringalloc, in safe_feed()
603 safe_dma_sync(&sc->sc_spalloc, BUS_DMASYNC_PREWRITE); in safe_feed()
604 safe_dma_sync(&sc->sc_dpalloc, BUS_DMASYNC_PREWRITE); in safe_feed()
612 sc->sc_nqchip++; in safe_feed()
613 if (sc->sc_nqchip > safestats.st_maxqchip) in safe_feed()
614 safestats.st_maxqchip = sc->sc_nqchip; in safe_feed()
625 bcopy(key, ses->ses_key, ses->ses_klen); in safe_setup_enckey()
627 /* PE is little-endian, insure proper byte order */ in safe_setup_enckey()
628 for (i = 0; i < N(ses->ses_key); i++) in safe_setup_enckey()
629 ses->ses_key[i] = htole32(ses->ses_key[i]); in safe_setup_enckey()
640 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); in safe_setup_mackey()
643 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); in safe_setup_mackey()
647 /* PE is little-endian, insure proper byte order */ in safe_setup_mackey()
648 for (i = 0; i < N(ses->ses_hminner); i++) { in safe_setup_mackey()
649 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]); in safe_setup_mackey()
650 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]); in safe_setup_mackey()
660 switch (csp->csp_auth_alg) { in safe_auth_supported()
662 if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0) in safe_auth_supported()
676 switch (csp->csp_cipher_alg) { in safe_cipher_supported()
678 if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0) in safe_cipher_supported()
680 if (csp->csp_ivlen != 16) in safe_cipher_supported()
682 if (csp->csp_cipher_klen != 16 && in safe_cipher_supported()
683 csp->csp_cipher_klen != 24 && in safe_cipher_supported()
684 csp->csp_cipher_klen != 32) in safe_cipher_supported()
696 if (csp->csp_flags != 0) in safe_probesession()
698 switch (csp->csp_mode) { in safe_probesession()
729 if (csp->csp_cipher_alg != 0) { in safe_newsession()
730 ses->ses_klen = csp->csp_cipher_klen; in safe_newsession()
731 if (csp->csp_cipher_key != NULL) in safe_newsession()
732 safe_setup_enckey(ses, csp->csp_cipher_key); in safe_newsession()
735 if (csp->csp_auth_alg != 0) { in safe_newsession()
736 ses->ses_mlen = csp->csp_auth_mlen; in safe_newsession()
737 if (ses->ses_mlen == 0) { in safe_newsession()
738 ses->ses_mlen = SHA1_HASH_LEN; in safe_newsession()
741 if (csp->csp_auth_key != NULL) { in safe_newsession()
742 safe_setup_mackey(ses, csp->csp_auth_alg, in safe_newsession()
743 csp->csp_auth_key, csp->csp_auth_klen); in safe_newsession()
759 op->nsegs = nsegs; in safe_op_cb()
760 bcopy(seg, op->segs, nsegs * sizeof (seg[0])); in safe_op_cb()
777 mtx_lock(&sc->sc_ringmtx); in safe_process()
778 if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) { in safe_process()
780 sc->sc_needwakeup |= CRYPTO_SYMQ; in safe_process()
781 mtx_unlock(&sc->sc_ringmtx); in safe_process()
784 re = sc->sc_front; in safe_process()
786 staterec = re->re_sa.sa_staterec; /* save */ in safe_process()
788 bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc)); in safe_process()
789 re->re_sa.sa_staterec = staterec; /* restore */ in safe_process()
791 re->re_crp = crp; in safe_process()
793 sa = &re->re_sa; in safe_process()
794 ses = crypto_get_driver_session(crp->crp_session); in safe_process()
795 csp = crypto_get_params(crp->crp_session); in safe_process()
799 switch (csp->csp_mode) { in safe_process()
811 if (csp->csp_cipher_alg != 0) { in safe_process()
812 if (crp->crp_cipher_key != NULL) in safe_process()
813 safe_setup_enckey(ses, crp->crp_cipher_key); in safe_process()
815 switch (csp->csp_cipher_alg) { in safe_process()
819 if (ses->ses_klen * 8 == 128) in safe_process()
821 else if (ses->ses_klen * 8 == 192) in safe_process()
836 crypto_read_iv(crp, re->re_sastate.sa_saved_iv); in safe_process()
839 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { in safe_process()
852 * This pads results to an 8-byte boundary and in safe_process()
856 * NB: Not sure if the 8-byte pad boundary is a problem. in safe_process()
861 bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key)); in safe_process()
864 if (csp->csp_auth_alg != 0) { in safe_process()
865 if (crp->crp_auth_key != NULL) { in safe_process()
866 safe_setup_mackey(ses, csp->csp_auth_alg, in safe_process()
867 crp->crp_auth_key, csp->csp_auth_klen); in safe_process()
870 switch (csp->csp_auth_alg) { in safe_process()
883 bcopy(ses->ses_hminner, sa->sa_indigest, in safe_process()
884 sizeof(sa->sa_indigest)); in safe_process()
885 bcopy(ses->ses_hmouter, sa->sa_outdigest, in safe_process()
886 sizeof(sa->sa_outdigest)); in safe_process()
889 re->re_flags |= SAFE_QFLAGS_COPYOUTICV; in safe_process()
892 if (csp->csp_mode == CSP_MODE_ETA) { in safe_process()
897 if (crp->crp_aad_length != 0 && in safe_process()
898 crp->crp_aad_start + crp->crp_aad_length != in safe_process()
899 crp->crp_payload_start) { in safe_process()
904 if (crp->crp_aad_length != 0) in safe_process()
905 bypass = crp->crp_aad_start; in safe_process()
907 bypass = crp->crp_payload_start; in safe_process()
908 coffset = crp->crp_aad_length; in safe_process()
909 oplen = crp->crp_payload_start + crp->crp_payload_length; in safe_process()
913 crp->crp_aad_start, crp->crp_aad_length, in safe_process()
914 crp->crp_digest_start); in safe_process()
916 crp->crp_payload_start, crp->crp_payload_length, in safe_process()
917 crp->crp_iv_start); in safe_process()
922 if (coffset & 3) { /* offset must be 32-bit aligned */ in safe_process()
953 bypass = crp->crp_payload_start; in safe_process()
954 oplen = bypass + crp->crp_payload_length; in safe_process()
965 if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) { in safe_process()
970 if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb, in safe_process()
971 &re->re_src, BUS_DMA_NOWAIT) != 0) { in safe_process()
972 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); in safe_process()
973 re->re_src_map = NULL; in safe_process()
978 re->re_src_mapsize = crypto_buffer_len(&crp->crp_buf); in safe_process()
979 nicealign = safe_dmamap_aligned(&re->re_src); in safe_process()
980 uniform = safe_dmamap_uniform(&re->re_src); in safe_process()
983 nicealign, uniform, re->re_src.nsegs)); in safe_process()
984 if (re->re_src.nsegs > 1) { in safe_process()
985 re->re_desc.d_src = sc->sc_spalloc.dma_paddr + in safe_process()
986 ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring); in safe_process()
987 for (i = 0; i < re->re_src_nsegs; i++) { in safe_process()
989 pd = sc->sc_spfree; in safe_process()
990 if (++(sc->sc_spfree) == sc->sc_springtop) in safe_process()
991 sc->sc_spfree = sc->sc_spring; in safe_process()
993 KASSERT((pd->pd_flags&3) == 0 || in safe_process()
994 (pd->pd_flags&3) == SAFE_PD_DONE, in safe_process()
996 pd->pd_flags)); in safe_process()
997 pd->pd_addr = re->re_src_segs[i].ds_addr; in safe_process()
998 pd->pd_size = re->re_src_segs[i].ds_len; in safe_process()
999 pd->pd_flags = SAFE_PD_READY; in safe_process()
1006 re->re_desc.d_src = re->re_src_segs[0].ds_addr; in safe_process()
1009 if (csp->csp_mode == CSP_MODE_DIGEST) { in safe_process()
1017 * sharing of the DMA map and segment list. in safe_process()
1019 re->re_dst = re->re_src; in safe_process()
1023 * different particle list to handle DMA of the in safe_process()
1024 * result. Create a new map and do the load to in safe_process()
1029 if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT, in safe_process()
1030 &re->re_dst_map)) { in safe_process()
1035 if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map, in safe_process()
1036 crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) != in safe_process()
1038 bus_dmamap_destroy(sc->sc_dstdmat, in safe_process()
1039 re->re_dst_map); in safe_process()
1040 re->re_dst_map = NULL; in safe_process()
1045 } else if (crp->crp_buf.cb_type == CRYPTO_BUF_MBUF) { in safe_process()
1050 * DMA constraints require that we allocate a in safe_process()
1061 totlen = re->re_src_mapsize; in safe_process()
1062 if (crp->crp_buf.cb_mbuf->m_flags & M_PKTHDR) { in safe_process()
1065 if (m && !m_dup_pkthdr(m, crp->crp_buf.cb_mbuf, in safe_process()
1076 err = sc->sc_nqchip ? ERESTART : ENOMEM; in safe_process()
1083 err = sc->sc_nqchip ? in safe_process()
1089 m->m_len = len; in safe_process()
1099 err = sc->sc_nqchip ? in safe_process()
1110 err = sc->sc_nqchip ? in safe_process()
1116 m->m_len = len = min(totlen, len); in safe_process()
1117 totlen -= len; in safe_process()
1119 mp = &m->m_next; in safe_process()
1121 re->re_dst_m = top; in safe_process()
1122 if (bus_dmamap_create(sc->sc_dstdmat, in safe_process()
1123 BUS_DMA_NOWAIT, &re->re_dst_map) != 0) { in safe_process()
1128 if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat, in safe_process()
1129 re->re_dst_map, top, re->re_dst_segs, in safe_process()
1130 &re->re_dst_nsegs, 0) != 0) { in safe_process()
1131 bus_dmamap_destroy(sc->sc_dstdmat, in safe_process()
1132 re->re_dst_map); in safe_process()
1133 re->re_dst_map = NULL; in safe_process()
1138 re->re_dst_mapsize = re->re_src_mapsize; in safe_process()
1139 if (re->re_src.mapsize > oplen) { in safe_process()
1147 if (!(csp->csp_mode == CSP_MODE_ETA && in safe_process()
1148 (re->re_src.mapsize-oplen) == ses->ses_mlen && in safe_process()
1149 crp->crp_digest_start == oplen)) in safe_process()
1150 safe_mcopy(crp->crp_buf.cb_mbuf, in safe_process()
1151 re->re_dst_m, oplen); in safe_process()
1162 * There's no way to handle the DMA in safe_process()
1164 * could create a separate DMA area for in safe_process()
1169 * the DMA map and segment list for the in safe_process()
1172 * the necessary scatter DMA. in safe_process()
1180 if (re->re_dst.nsegs > 1) { in safe_process()
1181 re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr + in safe_process()
1182 ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring); in safe_process()
1183 for (i = 0; i < re->re_dst_nsegs; i++) { in safe_process()
1184 pd = sc->sc_dpfree; in safe_process()
1185 KASSERT((pd->pd_flags&3) == 0 || in safe_process()
1186 (pd->pd_flags&3) == SAFE_PD_DONE, in safe_process()
1188 pd->pd_flags)); in safe_process()
1189 if (++(sc->sc_dpfree) == sc->sc_dpringtop) in safe_process()
1190 sc->sc_dpfree = sc->sc_dpring; in safe_process()
1191 pd->pd_addr = re->re_dst_segs[i].ds_addr; in safe_process()
1192 pd->pd_flags = SAFE_PD_READY; in safe_process()
1199 re->re_desc.d_dst = re->re_dst_segs[0].ds_addr; in safe_process()
1208 sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI; in safe_process()
1209 sa->sa_cmd1 = cmd1 in safe_process()
1222 re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI; in safe_process()
1223 if (csp->csp_auth_alg != 0) in safe_process()
1224 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL; in safe_process()
1225 re->re_desc.d_len = oplen in safe_process()
1233 if (++(sc->sc_front) == sc->sc_ringtop) in safe_process()
1234 sc->sc_front = sc->sc_ring; in safe_process()
1238 mtx_unlock(&sc->sc_ringmtx); in safe_process()
1242 if (re->re_dst_m != NULL) in safe_process()
1243 m_freem(re->re_dst_m); in safe_process()
1245 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) { in safe_process()
1246 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map); in safe_process()
1247 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map); in safe_process()
1249 if (re->re_src_map != NULL) { in safe_process()
1250 bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map); in safe_process()
1251 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); in safe_process()
1253 mtx_unlock(&sc->sc_ringmtx); in safe_process()
1255 crp->crp_etype = err; in safe_process()
1259 sc->sc_needwakeup |= CRYPTO_SYMQ; in safe_process()
1268 struct cryptop *crp = (struct cryptop *)re->re_crp; in safe_callback()
1272 ses = crypto_get_driver_session(crp->crp_session); in safe_callback()
1273 csp = crypto_get_params(crp->crp_session); in safe_callback()
1276 safestats.st_obytes += re->re_dst.mapsize; in safe_callback()
1278 safe_dma_sync(&sc->sc_ringalloc, in safe_callback()
1280 if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) { in safe_callback()
1281 device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n", in safe_callback()
1282 re->re_desc.d_csr, in safe_callback()
1283 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1); in safe_callback()
1285 crp->crp_etype = EIO; /* something more meaningful? */ in safe_callback()
1289 * XXX: Should crp_buf.cb_mbuf be updated to re->re_dst_m if in safe_callback()
1290 * it is non-NULL? in safe_callback()
1293 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) { in safe_callback()
1294 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map, in safe_callback()
1296 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map); in safe_callback()
1297 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map); in safe_callback()
1299 bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE); in safe_callback()
1300 bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map); in safe_callback()
1301 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); in safe_callback()
1303 if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) { in safe_callback()
1304 if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) { in safe_callback()
1306 * SHA-1 ICV's are byte-swapped; fix 'em up in safe_callback()
1309 re->re_sastate.sa_saved_indigest[0] = in safe_callback()
1310 bswap32(re->re_sastate.sa_saved_indigest[0]); in safe_callback()
1311 re->re_sastate.sa_saved_indigest[1] = in safe_callback()
1312 bswap32(re->re_sastate.sa_saved_indigest[1]); in safe_callback()
1313 re->re_sastate.sa_saved_indigest[2] = in safe_callback()
1314 bswap32(re->re_sastate.sa_saved_indigest[2]); in safe_callback()
1317 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { in safe_callback()
1318 crypto_copydata(crp, crp->crp_digest_start, in safe_callback()
1319 ses->ses_mlen, hash); in safe_callback()
1320 if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest, in safe_callback()
1321 hash, ses->ses_mlen) != 0) in safe_callback()
1322 crp->crp_etype = EBADMSG; in safe_callback()
1324 crypto_copyback(crp, crp->crp_digest_start, in safe_callback()
1325 ses->ses_mlen, re->re_sastate.sa_saved_indigest); in safe_callback()
1343 while (j >= srcm->m_len) { in safe_mcopy()
1344 j -= srcm->m_len; in safe_mcopy()
1345 srcm = srcm->m_next; in safe_mcopy()
1350 slen = srcm->m_len - j; in safe_mcopy()
1353 while (j >= dstm->m_len) { in safe_mcopy()
1354 j -= dstm->m_len; in safe_mcopy()
1355 dstm = dstm->m_next; in safe_mcopy()
1360 dlen = dstm->m_len - j; in safe_mcopy()
1369 srcm = srcm->m_next; in safe_mcopy()
1372 sptr = srcm->m_data; in safe_mcopy()
1373 slen = srcm->m_len; in safe_mcopy()
1375 sptr += j, slen -= j; in safe_mcopy()
1377 dstm = dstm->m_next; in safe_mcopy()
1380 dptr = dstm->m_data; in safe_mcopy()
1381 dlen = dstm->m_len; in safe_mcopy()
1383 dptr += j, dlen -= j; in safe_mcopy()
1412 * This could take between 750ms - 1000ms in time. in safe_rng_init()
1512 (*sc->sc_harvest)(sc->sc_rndtest, buf, maxwords*sizeof (u_int32_t)); in safe_rng()
1513 callout_reset(&sc->sc_rngto, in safe_rng()
1522 *paddr = segs->ds_addr; in safe_dmamap_cb()
1529 struct safe_dma_alloc *dma, in safe_dma_malloc() argument
1535 r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ in safe_dma_malloc()
1545 &dma->dma_tag); in safe_dma_malloc()
1547 device_printf(sc->sc_dev, "safe_dma_malloc: " in safe_dma_malloc()
1552 r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, in safe_dma_malloc()
1553 BUS_DMA_NOWAIT, &dma->dma_map); in safe_dma_malloc()
1555 device_printf(sc->sc_dev, "safe_dma_malloc: " in safe_dma_malloc()
1561 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, in safe_dma_malloc()
1564 &dma->dma_paddr, in safe_dma_malloc()
1567 device_printf(sc->sc_dev, "safe_dma_malloc: " in safe_dma_malloc()
1572 dma->dma_size = size; in safe_dma_malloc()
1575 bus_dmamap_unload(dma->dma_tag, dma->dma_map); in safe_dma_malloc()
1577 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); in safe_dma_malloc()
1579 bus_dma_tag_destroy(dma->dma_tag); in safe_dma_malloc()
1581 dma->dma_tag = NULL; in safe_dma_malloc()
1586 safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma) in safe_dma_free() argument
1588 bus_dmamap_unload(dma->dma_tag, dma->dma_map); in safe_dma_free()
1589 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); in safe_dma_free()
1590 bus_dma_tag_destroy(dma->dma_tag); in safe_dma_free()
1628 | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */ in safe_init_board()
1629 | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */ in safe_init_board()
1630 | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */ in safe_init_board()
1637 if (sc->sc_chiprev == SAFE_REV(1,0)) { in safe_init_board()
1639 * Avoid large PCI DMA transfers. Rev 1.0 has a bug where in safe_init_board()
1640 * "target mode transfers" done while the chip is DMA'ing in safe_init_board()
1646 device_printf(sc->sc_dev, in safe_init_board()
1647 "Reduce max DMA size to %u words for rev %u.%u WAR\n", in safe_init_board()
1649 SAFE_REV_MAJ(sc->sc_chiprev), in safe_init_board()
1650 SAFE_REV_MIN(sc->sc_chiprev)); in safe_init_board()
1654 WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr); in safe_init_board()
1655 WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr); in safe_init_board()
1660 ("PE ring entry not 32-bit aligned!")); in safe_init_board()
1666 WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr); in safe_init_board()
1667 WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr); in safe_init_board()
1677 /* it's now safe to enable PE mode, do it */ in safe_init_board()
1681 * Configure hardware to use level-triggered interrupts and in safe_init_board()
1705 if (sc->sc_nqchip != 0) { in safe_cleanchip()
1706 struct safe_ringentry *re = sc->sc_back; in safe_cleanchip()
1708 while (re != sc->sc_front) { in safe_cleanchip()
1709 if (re->re_desc.d_csr != 0) in safe_cleanchip()
1711 if (++re == sc->sc_ringtop) in safe_cleanchip()
1712 re = sc->sc_ring; in safe_cleanchip()
1714 sc->sc_back = re; in safe_cleanchip()
1715 sc->sc_nqchip = 0; in safe_cleanchip()
1731 if (re->re_dst_m != NULL) in safe_free_entry()
1732 m_freem(re->re_dst_m); in safe_free_entry()
1734 crp = (struct cryptop *)re->re_crp; in safe_free_entry()
1736 re->re_desc.d_csr = 0; in safe_free_entry()
1738 crp->crp_etype = EFAULT; in safe_free_entry()
1756 * Is the operand suitable aligned for direct DMA. Each
1757 * segment must be aligned on a 32-bit boundary and all
1765 for (i = 0; i < op->nsegs; i++) { in safe_dmamap_aligned()
1766 if (op->segs[i].ds_addr & 3) in safe_dmamap_aligned()
1768 if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3)) in safe_dmamap_aligned()
1775 * Is the operand suitable for direct DMA as the destination
1788 if (op->nsegs > 0) { in safe_dmamap_uniform()
1791 for (i = 0; i < op->nsegs-1; i++) { in safe_dmamap_uniform()
1792 if (op->segs[i].ds_len % SAFE_MAX_DSIZE) in safe_dmamap_uniform()
1794 if (op->segs[i].ds_len != SAFE_MAX_DSIZE) in safe_dmamap_uniform()
1836 (unsigned long)(sc->sc_back - sc->sc_ring), in safe_dump_ringstate()
1837 (unsigned long)(sc->sc_front - sc->sc_ring)); in safe_dump_ringstate()
1845 ix = re - sc->sc_ring; in safe_dump_request()
1849 , re->re_desc.d_csr in safe_dump_request()
1850 , re->re_desc.d_src in safe_dump_request()
1851 , re->re_desc.d_dst in safe_dump_request()
1852 , re->re_desc.d_sa in safe_dump_request()
1853 , re->re_desc.d_len in safe_dump_request()
1855 if (re->re_src.nsegs > 1) { in safe_dump_request()
1856 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) / in safe_dump_request()
1858 for (nsegs = re->re_src.nsegs; nsegs; nsegs--) { in safe_dump_request()
1860 , ix, &sc->sc_spring[ix] in safe_dump_request()
1861 , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr in safe_dump_request()
1862 , sc->sc_spring[ix].pd_size in safe_dump_request()
1863 , sc->sc_spring[ix].pd_flags in safe_dump_request()
1865 if (sc->sc_spring[ix].pd_size == 0) in safe_dump_request()
1872 if (re->re_dst.nsegs > 1) { in safe_dump_request()
1873 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) / in safe_dump_request()
1875 for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) { in safe_dump_request()
1877 , ix, &sc->sc_dpring[ix] in safe_dump_request()
1878 , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr in safe_dump_request()
1879 , sc->sc_dpring[ix].pd_flags in safe_dump_request()
1886 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec); in safe_dump_request()
1888 , re->re_sa.sa_key[0] in safe_dump_request()
1889 , re->re_sa.sa_key[1] in safe_dump_request()
1890 , re->re_sa.sa_key[2] in safe_dump_request()
1891 , re->re_sa.sa_key[3] in safe_dump_request()
1892 , re->re_sa.sa_key[4] in safe_dump_request()
1893 , re->re_sa.sa_key[5] in safe_dump_request()
1894 , re->re_sa.sa_key[6] in safe_dump_request()
1895 , re->re_sa.sa_key[7] in safe_dump_request()
1898 , re->re_sa.sa_indigest[0] in safe_dump_request()
1899 , re->re_sa.sa_indigest[1] in safe_dump_request()
1900 , re->re_sa.sa_indigest[2] in safe_dump_request()
1901 , re->re_sa.sa_indigest[3] in safe_dump_request()
1902 , re->re_sa.sa_indigest[4] in safe_dump_request()
1905 , re->re_sa.sa_outdigest[0] in safe_dump_request()
1906 , re->re_sa.sa_outdigest[1] in safe_dump_request()
1907 , re->re_sa.sa_outdigest[2] in safe_dump_request()
1908 , re->re_sa.sa_outdigest[3] in safe_dump_request()
1909 , re->re_sa.sa_outdigest[4] in safe_dump_request()
1912 , re->re_sastate.sa_saved_iv[0] in safe_dump_request()
1913 , re->re_sastate.sa_saved_iv[1] in safe_dump_request()
1914 , re->re_sastate.sa_saved_iv[2] in safe_dump_request()
1915 , re->re_sastate.sa_saved_iv[3] in safe_dump_request()
1918 , re->re_sastate.sa_saved_hashbc in safe_dump_request()
1919 , re->re_sastate.sa_saved_indigest[0] in safe_dump_request()
1920 , re->re_sastate.sa_saved_indigest[1] in safe_dump_request()
1921 , re->re_sastate.sa_saved_indigest[2] in safe_dump_request()
1922 , re->re_sastate.sa_saved_indigest[3] in safe_dump_request()
1923 , re->re_sastate.sa_saved_indigest[4] in safe_dump_request()
1930 mtx_lock(&sc->sc_ringmtx); in safe_dump_ring()
1935 if (sc->sc_nqchip) { in safe_dump_ring()
1936 struct safe_ringentry *re = sc->sc_back; in safe_dump_ring()
1939 if (++re == sc->sc_ringtop) in safe_dump_ring()
1940 re = sc->sc_ring; in safe_dump_ring()
1941 } while (re != sc->sc_front); in safe_dump_ring()
1943 mtx_unlock(&sc->sc_ringmtx); in safe_dump_ring()
1952 strncpy(dmode, "", sizeof(dmode) - 1); in sysctl_hw_safe_dump()
1953 dmode[sizeof(dmode) - 1] = '\0'; in sysctl_hw_safe_dump()
1956 if (error == 0 && req->newptr != NULL) { in sysctl_hw_safe_dump()
1961 if (strncmp(dmode, "dma", 3) == 0) in sysctl_hw_safe_dump()