Lines Matching +full:re +full:-
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
32 * SafeNet SafeXcel-1141 hardware crypto accelerator
133 bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
167 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
170 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
181 0, "RNG polling buffer size (32-bit words)");
200 switch (pci_get_vendor(sc->sc_dev)) { in safe_partname()
202 switch (pci_get_device(sc->sc_dev)) { in safe_partname()
203 case PCI_PRODUCT_SAFEXCEL: return "SafeNet SafeXcel-1141"; in safe_partname()
205 return "SafeNet unknown-part"; in safe_partname()
207 return "Unknown-vendor unknown-part"; in safe_partname()
228 sc->sc_dev = dev; in safe_attach()
235 * Setup memory-mapping of PCI registers. in safe_attach()
238 sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, in safe_attach()
240 if (sc->sc_sr == NULL) { in safe_attach()
244 sc->sc_st = rman_get_bustag(sc->sc_sr); in safe_attach()
245 sc->sc_sh = rman_get_bushandle(sc->sc_sr); in safe_attach()
251 sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, in safe_attach()
253 if (sc->sc_irq == NULL) { in safe_attach()
261 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, in safe_attach()
262 NULL, safe_intr, sc, &sc->sc_ih)) { in safe_attach()
267 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safe_session), in safe_attach()
269 if (sc->sc_cid < 0) { in safe_attach()
274 sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) & in safe_attach()
291 &sc->sc_srcdmat)) { in safe_attach()
306 &sc->sc_dstdmat)) { in safe_attach()
316 &sc->sc_ringalloc, 0)) { in safe_attach()
318 bus_dma_tag_destroy(sc->sc_srcdmat); in safe_attach()
324 sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr; in safe_attach()
325 sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE; in safe_attach()
326 sc->sc_front = sc->sc_ring; in safe_attach()
327 sc->sc_back = sc->sc_ring; in safe_attach()
328 raddr = sc->sc_ringalloc.dma_paddr; in safe_attach()
329 bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry)); in safe_attach()
331 struct safe_ringentry *re = &sc->sc_ring[i]; in safe_attach() local
333 re->re_desc.d_sa = raddr + in safe_attach()
335 re->re_sa.sa_staterec = raddr + in safe_attach()
340 mtx_init(&sc->sc_ringmtx, device_get_nameunit(dev), in safe_attach()
347 &sc->sc_spalloc, 0)) { in safe_attach()
350 mtx_destroy(&sc->sc_ringmtx); in safe_attach()
351 safe_dma_free(sc, &sc->sc_ringalloc); in safe_attach()
352 bus_dma_tag_destroy(sc->sc_srcdmat); in safe_attach()
355 sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr; in safe_attach()
356 sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART; in safe_attach()
357 sc->sc_spfree = sc->sc_spring; in safe_attach()
358 bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc)); in safe_attach()
361 &sc->sc_dpalloc, 0)) { in safe_attach()
364 mtx_destroy(&sc->sc_ringmtx); in safe_attach()
365 safe_dma_free(sc, &sc->sc_spalloc); in safe_attach()
366 safe_dma_free(sc, &sc->sc_ringalloc); in safe_attach()
367 bus_dma_tag_destroy(sc->sc_dstdmat); in safe_attach()
370 sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr; in safe_attach()
371 sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART; in safe_attach()
372 sc->sc_dpfree = sc->sc_dpring; in safe_attach()
373 bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc)); in safe_attach()
375 device_printf(sc->sc_dev, "%s", safe_partname(sc)); in safe_attach()
377 sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO); in safe_attach()
378 if (sc->sc_devinfo & SAFE_DEVINFO_RNG) { in safe_attach()
379 sc->sc_flags |= SAFE_FLAGS_RNG; in safe_attach()
382 if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) { in safe_attach()
385 sc->sc_flags |= SAFE_FLAGS_KEY; in safe_attach()
388 if (sc->sc_devinfo & SAFE_DEVINFO_DES) { in safe_attach()
391 if (sc->sc_devinfo & SAFE_DEVINFO_AES) { in safe_attach()
394 if (sc->sc_devinfo & SAFE_DEVINFO_MD5) { in safe_attach()
397 if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) { in safe_attach()
408 if (sc->sc_flags & SAFE_FLAGS_RNG) { in safe_attach()
410 sc->sc_rndtest = rndtest_attach(dev); in safe_attach()
411 if (sc->sc_rndtest) in safe_attach()
412 sc->sc_harvest = rndtest_harvest; in safe_attach()
414 sc->sc_harvest = default_harvest; in safe_attach()
416 sc->sc_harvest = default_harvest; in safe_attach()
420 callout_init(&sc->sc_rngto, 1); in safe_attach()
421 callout_reset(&sc->sc_rngto, hz*safe_rnginterval, safe_rng, sc); in safe_attach()
431 crypto_unregister_all(sc->sc_cid); in safe_attach()
433 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); in safe_attach()
435 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); in safe_attach()
437 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); in safe_attach()
454 callout_stop(&sc->sc_rngto); in safe_detach()
456 crypto_unregister_all(sc->sc_cid); in safe_detach()
459 if (sc->sc_rndtest) in safe_detach()
460 rndtest_detach(sc->sc_rndtest); in safe_detach()
464 safe_dma_free(sc, &sc->sc_dpalloc); in safe_detach()
465 safe_dma_free(sc, &sc->sc_spalloc); in safe_detach()
466 mtx_destroy(&sc->sc_ringmtx); in safe_detach()
467 safe_dma_free(sc, &sc->sc_ringalloc); in safe_detach()
469 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); in safe_detach()
470 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); in safe_detach()
472 bus_dma_tag_destroy(sc->sc_srcdmat); in safe_detach()
473 bus_dma_tag_destroy(sc->sc_dstdmat); in safe_detach()
474 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); in safe_detach()
503 sc->sc_suspended = 1; in safe_suspend()
516 sc->sc_suspended = 0; in safe_resume()
540 mtx_lock(&sc->sc_ringmtx); in safe_intr()
541 while (sc->sc_back != sc->sc_front) { in safe_intr()
542 struct safe_ringentry *re = sc->sc_back; in safe_intr() local
546 safe_dump_request(sc, __func__, re); in safe_intr()
556 if (re->re_desc.d_csr != 0) { in safe_intr()
557 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr)) in safe_intr()
559 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len)) in safe_intr()
561 sc->sc_nqchip--; in safe_intr()
562 safe_callback(sc, re); in safe_intr()
564 if (++(sc->sc_back) == sc->sc_ringtop) in safe_intr()
565 sc->sc_back = sc->sc_ring; in safe_intr()
567 mtx_unlock(&sc->sc_ringmtx); in safe_intr()
583 if (sc->sc_needwakeup) { /* XXX check high watermark */ in safe_intr()
584 int wakeup = sc->sc_needwakeup & CRYPTO_SYMQ; in safe_intr()
586 sc->sc_needwakeup)); in safe_intr()
587 sc->sc_needwakeup &= ~wakeup; in safe_intr()
588 crypto_unblock(sc->sc_cid, wakeup); in safe_intr()
593 * safe_feed() - post a request to chip
596 safe_feed(struct safe_softc *sc, struct safe_ringentry *re) in safe_feed() argument
598 bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE); in safe_feed()
599 if (re->re_dst_map != NULL) in safe_feed()
600 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map, in safe_feed()
603 safe_dma_sync(&sc->sc_ringalloc, in safe_feed()
605 safe_dma_sync(&sc->sc_spalloc, BUS_DMASYNC_PREWRITE); in safe_feed()
606 safe_dma_sync(&sc->sc_dpalloc, BUS_DMASYNC_PREWRITE); in safe_feed()
611 safe_dump_request(sc, __func__, re); in safe_feed()
614 sc->sc_nqchip++; in safe_feed()
615 if (sc->sc_nqchip > safestats.st_maxqchip) in safe_feed()
616 safestats.st_maxqchip = sc->sc_nqchip; in safe_feed()
627 bcopy(key, ses->ses_key, ses->ses_klen); in safe_setup_enckey()
629 /* PE is little-endian, insure proper byte order */ in safe_setup_enckey()
630 for (i = 0; i < N(ses->ses_key); i++) in safe_setup_enckey()
631 ses->ses_key[i] = htole32(ses->ses_key[i]); in safe_setup_enckey()
642 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32)); in safe_setup_mackey()
645 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32)); in safe_setup_mackey()
649 /* PE is little-endian, insure proper byte order */ in safe_setup_mackey()
650 for (i = 0; i < N(ses->ses_hminner); i++) { in safe_setup_mackey()
651 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]); in safe_setup_mackey()
652 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]); in safe_setup_mackey()
662 switch (csp->csp_auth_alg) { in safe_auth_supported()
664 if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0) in safe_auth_supported()
678 switch (csp->csp_cipher_alg) { in safe_cipher_supported()
680 if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0) in safe_cipher_supported()
682 if (csp->csp_ivlen != 16) in safe_cipher_supported()
684 if (csp->csp_cipher_klen != 16 && in safe_cipher_supported()
685 csp->csp_cipher_klen != 24 && in safe_cipher_supported()
686 csp->csp_cipher_klen != 32) in safe_cipher_supported()
698 if (csp->csp_flags != 0) in safe_probesession()
700 switch (csp->csp_mode) { in safe_probesession()
731 if (csp->csp_cipher_alg != 0) { in safe_newsession()
732 ses->ses_klen = csp->csp_cipher_klen; in safe_newsession()
733 if (csp->csp_cipher_key != NULL) in safe_newsession()
734 safe_setup_enckey(ses, csp->csp_cipher_key); in safe_newsession()
737 if (csp->csp_auth_alg != 0) { in safe_newsession()
738 ses->ses_mlen = csp->csp_auth_mlen; in safe_newsession()
739 if (ses->ses_mlen == 0) { in safe_newsession()
740 ses->ses_mlen = SHA1_HASH_LEN; in safe_newsession()
743 if (csp->csp_auth_key != NULL) { in safe_newsession()
744 safe_setup_mackey(ses, csp->csp_auth_alg, in safe_newsession()
745 csp->csp_auth_key, csp->csp_auth_klen); in safe_newsession()
761 op->nsegs = nsegs; in safe_op_cb()
762 bcopy(seg, op->segs, nsegs * sizeof (seg[0])); in safe_op_cb()
774 struct safe_ringentry *re; in safe_process() local
779 mtx_lock(&sc->sc_ringmtx); in safe_process()
780 if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) { in safe_process()
782 sc->sc_needwakeup |= CRYPTO_SYMQ; in safe_process()
783 mtx_unlock(&sc->sc_ringmtx); in safe_process()
786 re = sc->sc_front; in safe_process()
788 staterec = re->re_sa.sa_staterec; /* save */ in safe_process()
790 bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc)); in safe_process()
791 re->re_sa.sa_staterec = staterec; /* restore */ in safe_process()
793 re->re_crp = crp; in safe_process()
795 sa = &re->re_sa; in safe_process()
796 ses = crypto_get_driver_session(crp->crp_session); in safe_process()
797 csp = crypto_get_params(crp->crp_session); in safe_process()
801 switch (csp->csp_mode) { in safe_process()
813 if (csp->csp_cipher_alg != 0) { in safe_process()
814 if (crp->crp_cipher_key != NULL) in safe_process()
815 safe_setup_enckey(ses, crp->crp_cipher_key); in safe_process()
817 switch (csp->csp_cipher_alg) { in safe_process()
821 if (ses->ses_klen * 8 == 128) in safe_process()
823 else if (ses->ses_klen * 8 == 192) in safe_process()
838 crypto_read_iv(crp, re->re_sastate.sa_saved_iv); in safe_process()
841 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { in safe_process()
854 * This pads results to an 8-byte boundary and in safe_process()
858 * NB: Not sure if the 8-byte pad boundary is a problem. in safe_process()
863 bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key)); in safe_process()
866 if (csp->csp_auth_alg != 0) { in safe_process()
867 if (crp->crp_auth_key != NULL) { in safe_process()
868 safe_setup_mackey(ses, csp->csp_auth_alg, in safe_process()
869 crp->crp_auth_key, csp->csp_auth_klen); in safe_process()
872 switch (csp->csp_auth_alg) { in safe_process()
885 bcopy(ses->ses_hminner, sa->sa_indigest, in safe_process()
886 sizeof(sa->sa_indigest)); in safe_process()
887 bcopy(ses->ses_hmouter, sa->sa_outdigest, in safe_process()
888 sizeof(sa->sa_outdigest)); in safe_process()
891 re->re_flags |= SAFE_QFLAGS_COPYOUTICV; in safe_process()
894 if (csp->csp_mode == CSP_MODE_ETA) { in safe_process()
899 if (crp->crp_aad_length != 0 && in safe_process()
900 crp->crp_aad_start + crp->crp_aad_length != in safe_process()
901 crp->crp_payload_start) { in safe_process()
906 if (crp->crp_aad_length != 0) in safe_process()
907 bypass = crp->crp_aad_start; in safe_process()
909 bypass = crp->crp_payload_start; in safe_process()
910 coffset = crp->crp_aad_length; in safe_process()
911 oplen = crp->crp_payload_start + crp->crp_payload_length; in safe_process()
915 crp->crp_aad_start, crp->crp_aad_length, in safe_process()
916 crp->crp_digest_start); in safe_process()
918 crp->crp_payload_start, crp->crp_payload_length, in safe_process()
919 crp->crp_iv_start); in safe_process()
924 if (coffset & 3) { /* offset must be 32-bit aligned */ in safe_process()
955 bypass = crp->crp_payload_start; in safe_process()
956 oplen = bypass + crp->crp_payload_length; in safe_process()
967 if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) { in safe_process()
972 if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb, in safe_process()
973 &re->re_src, BUS_DMA_NOWAIT) != 0) { in safe_process()
974 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); in safe_process()
975 re->re_src_map = NULL; in safe_process()
980 re->re_src_mapsize = crypto_buffer_len(&crp->crp_buf); in safe_process()
981 nicealign = safe_dmamap_aligned(&re->re_src); in safe_process()
982 uniform = safe_dmamap_uniform(&re->re_src); in safe_process()
985 nicealign, uniform, re->re_src.nsegs)); in safe_process()
986 if (re->re_src.nsegs > 1) { in safe_process()
987 re->re_desc.d_src = sc->sc_spalloc.dma_paddr + in safe_process()
988 ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring); in safe_process()
989 for (i = 0; i < re->re_src_nsegs; i++) { in safe_process()
991 pd = sc->sc_spfree; in safe_process()
992 if (++(sc->sc_spfree) == sc->sc_springtop) in safe_process()
993 sc->sc_spfree = sc->sc_spring; in safe_process()
995 KASSERT((pd->pd_flags&3) == 0 || in safe_process()
996 (pd->pd_flags&3) == SAFE_PD_DONE, in safe_process()
998 pd->pd_flags)); in safe_process()
999 pd->pd_addr = re->re_src_segs[i].ds_addr; in safe_process()
1000 pd->pd_size = re->re_src_segs[i].ds_len; in safe_process()
1001 pd->pd_flags = SAFE_PD_READY; in safe_process()
1008 re->re_desc.d_src = re->re_src_segs[0].ds_addr; in safe_process()
1011 if (csp->csp_mode == CSP_MODE_DIGEST) { in safe_process()
1021 re->re_dst = re->re_src; in safe_process()
1031 if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT, in safe_process()
1032 &re->re_dst_map)) { in safe_process()
1037 if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map, in safe_process()
1038 crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) != in safe_process()
1040 bus_dmamap_destroy(sc->sc_dstdmat, in safe_process()
1041 re->re_dst_map); in safe_process()
1042 re->re_dst_map = NULL; in safe_process()
1047 } else if (crp->crp_buf.cb_type == CRYPTO_BUF_MBUF) { in safe_process()
1063 totlen = re->re_src_mapsize; in safe_process()
1064 if (crp->crp_buf.cb_mbuf->m_flags & M_PKTHDR) { in safe_process()
1067 if (m && !m_dup_pkthdr(m, crp->crp_buf.cb_mbuf, in safe_process()
1078 err = sc->sc_nqchip ? ERESTART : ENOMEM; in safe_process()
1085 err = sc->sc_nqchip ? in safe_process()
1091 m->m_len = len; in safe_process()
1101 err = sc->sc_nqchip ? in safe_process()
1112 err = sc->sc_nqchip ? in safe_process()
1118 m->m_len = len = min(totlen, len); in safe_process()
1119 totlen -= len; in safe_process()
1121 mp = &m->m_next; in safe_process()
1123 re->re_dst_m = top; in safe_process()
1124 if (bus_dmamap_create(sc->sc_dstdmat, in safe_process()
1125 BUS_DMA_NOWAIT, &re->re_dst_map) != 0) { in safe_process()
1130 if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat, in safe_process()
1131 re->re_dst_map, top, re->re_dst_segs, in safe_process()
1132 &re->re_dst_nsegs, 0) != 0) { in safe_process()
1133 bus_dmamap_destroy(sc->sc_dstdmat, in safe_process()
1134 re->re_dst_map); in safe_process()
1135 re->re_dst_map = NULL; in safe_process()
1140 re->re_dst_mapsize = re->re_src_mapsize; in safe_process()
1141 if (re->re_src.mapsize > oplen) { in safe_process()
1149 if (!(csp->csp_mode == CSP_MODE_ETA && in safe_process()
1150 (re->re_src.mapsize-oplen) == ses->ses_mlen && in safe_process()
1151 crp->crp_digest_start == oplen)) in safe_process()
1152 safe_mcopy(crp->crp_buf.cb_mbuf, in safe_process()
1153 re->re_dst_m, oplen); in safe_process()
1182 if (re->re_dst.nsegs > 1) { in safe_process()
1183 re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr + in safe_process()
1184 ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring); in safe_process()
1185 for (i = 0; i < re->re_dst_nsegs; i++) { in safe_process()
1186 pd = sc->sc_dpfree; in safe_process()
1187 KASSERT((pd->pd_flags&3) == 0 || in safe_process()
1188 (pd->pd_flags&3) == SAFE_PD_DONE, in safe_process()
1190 pd->pd_flags)); in safe_process()
1191 if (++(sc->sc_dpfree) == sc->sc_dpringtop) in safe_process()
1192 sc->sc_dpfree = sc->sc_dpring; in safe_process()
1193 pd->pd_addr = re->re_dst_segs[i].ds_addr; in safe_process()
1194 pd->pd_flags = SAFE_PD_READY; in safe_process()
1201 re->re_desc.d_dst = re->re_dst_segs[0].ds_addr; in safe_process()
1210 sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI; in safe_process()
1211 sa->sa_cmd1 = cmd1 in safe_process()
1224 re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI; in safe_process()
1225 if (csp->csp_auth_alg != 0) in safe_process()
1226 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL; in safe_process()
1227 re->re_desc.d_len = oplen in safe_process()
1235 if (++(sc->sc_front) == sc->sc_ringtop) in safe_process()
1236 sc->sc_front = sc->sc_ring; in safe_process()
1239 safe_feed(sc, re); in safe_process()
1240 mtx_unlock(&sc->sc_ringmtx); in safe_process()
1244 if (re->re_dst_m != NULL) in safe_process()
1245 m_freem(re->re_dst_m); in safe_process()
1247 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) { in safe_process()
1248 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map); in safe_process()
1249 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map); in safe_process()
1251 if (re->re_src_map != NULL) { in safe_process()
1252 bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map); in safe_process()
1253 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); in safe_process()
1255 mtx_unlock(&sc->sc_ringmtx); in safe_process()
1257 crp->crp_etype = err; in safe_process()
1261 sc->sc_needwakeup |= CRYPTO_SYMQ; in safe_process()
1267 safe_callback(struct safe_softc *sc, struct safe_ringentry *re) in safe_callback() argument
1270 struct cryptop *crp = (struct cryptop *)re->re_crp; in safe_callback()
1274 ses = crypto_get_driver_session(crp->crp_session); in safe_callback()
1275 csp = crypto_get_params(crp->crp_session); in safe_callback()
1278 safestats.st_obytes += re->re_dst.mapsize; in safe_callback()
1280 safe_dma_sync(&sc->sc_ringalloc, in safe_callback()
1282 if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) { in safe_callback()
1283 device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n", in safe_callback()
1284 re->re_desc.d_csr, in safe_callback()
1285 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1); in safe_callback()
1287 crp->crp_etype = EIO; /* something more meaningful? */ in safe_callback()
1291 * XXX: Should crp_buf.cb_mbuf be updated to re->re_dst_m if in safe_callback()
1292 * it is non-NULL? in safe_callback()
1295 if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) { in safe_callback()
1296 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map, in safe_callback()
1298 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map); in safe_callback()
1299 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map); in safe_callback()
1301 bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE); in safe_callback()
1302 bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map); in safe_callback()
1303 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map); in safe_callback()
1305 if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) { in safe_callback()
1306 if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) { in safe_callback()
1308 * SHA-1 ICV's are byte-swapped; fix 'em up in safe_callback()
1311 re->re_sastate.sa_saved_indigest[0] = in safe_callback()
1312 bswap32(re->re_sastate.sa_saved_indigest[0]); in safe_callback()
1313 re->re_sastate.sa_saved_indigest[1] = in safe_callback()
1314 bswap32(re->re_sastate.sa_saved_indigest[1]); in safe_callback()
1315 re->re_sastate.sa_saved_indigest[2] = in safe_callback()
1316 bswap32(re->re_sastate.sa_saved_indigest[2]); in safe_callback()
1319 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { in safe_callback()
1320 crypto_copydata(crp, crp->crp_digest_start, in safe_callback()
1321 ses->ses_mlen, hash); in safe_callback()
1322 if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest, in safe_callback()
1323 hash, ses->ses_mlen) != 0) in safe_callback()
1324 crp->crp_etype = EBADMSG; in safe_callback()
1326 crypto_copyback(crp, crp->crp_digest_start, in safe_callback()
1327 ses->ses_mlen, re->re_sastate.sa_saved_indigest); in safe_callback()
1345 while (j >= srcm->m_len) { in safe_mcopy()
1346 j -= srcm->m_len; in safe_mcopy()
1347 srcm = srcm->m_next; in safe_mcopy()
1352 slen = srcm->m_len - j; in safe_mcopy()
1355 while (j >= dstm->m_len) { in safe_mcopy()
1356 j -= dstm->m_len; in safe_mcopy()
1357 dstm = dstm->m_next; in safe_mcopy()
1362 dlen = dstm->m_len - j; in safe_mcopy()
1371 srcm = srcm->m_next; in safe_mcopy()
1374 sptr = srcm->m_data; in safe_mcopy()
1375 slen = srcm->m_len; in safe_mcopy()
1377 sptr += j, slen -= j; in safe_mcopy()
1379 dstm = dstm->m_next; in safe_mcopy()
1382 dptr = dstm->m_data; in safe_mcopy()
1383 dlen = dstm->m_len; in safe_mcopy()
1385 dptr += j, dlen -= j; in safe_mcopy()
1414 * This could take between 750ms - 1000ms in time. in safe_rng_init()
1514 (*sc->sc_harvest)(sc->sc_rndtest, buf, maxwords*sizeof (u_int32_t)); in safe_rng()
1515 callout_reset(&sc->sc_rngto, in safe_rng()
1524 *paddr = segs->ds_addr; in safe_dmamap_cb()
1537 r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ in safe_dma_malloc()
1547 &dma->dma_tag); in safe_dma_malloc()
1549 device_printf(sc->sc_dev, "safe_dma_malloc: " in safe_dma_malloc()
1554 r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, in safe_dma_malloc()
1555 BUS_DMA_NOWAIT, &dma->dma_map); in safe_dma_malloc()
1557 device_printf(sc->sc_dev, "safe_dma_malloc: " in safe_dma_malloc()
1563 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, in safe_dma_malloc()
1566 &dma->dma_paddr, in safe_dma_malloc()
1569 device_printf(sc->sc_dev, "safe_dma_malloc: " in safe_dma_malloc()
1574 dma->dma_size = size; in safe_dma_malloc()
1577 bus_dmamap_unload(dma->dma_tag, dma->dma_map); in safe_dma_malloc()
1579 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); in safe_dma_malloc()
1581 bus_dma_tag_destroy(dma->dma_tag); in safe_dma_malloc()
1583 dma->dma_tag = NULL; in safe_dma_malloc()
1590 bus_dmamap_unload(dma->dma_tag, dma->dma_map); in safe_dma_free()
1591 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); in safe_dma_free()
1592 bus_dma_tag_destroy(dma->dma_tag); in safe_dma_free()
1630 | SAFE_PE_DMACFG_ESDESC /* endian-swap descriptors */ in safe_init_board()
1631 | SAFE_PE_DMACFG_ESSA /* endian-swap SA's */ in safe_init_board()
1632 | SAFE_PE_DMACFG_ESPDESC /* endian-swap part. desc's */ in safe_init_board()
1639 if (sc->sc_chiprev == SAFE_REV(1,0)) { in safe_init_board()
1648 device_printf(sc->sc_dev, in safe_init_board()
1651 SAFE_REV_MAJ(sc->sc_chiprev), in safe_init_board()
1652 SAFE_REV_MIN(sc->sc_chiprev)); in safe_init_board()
1656 WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr); in safe_init_board()
1657 WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr); in safe_init_board()
1662 ("PE ring entry not 32-bit aligned!")); in safe_init_board()
1668 WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr); in safe_init_board()
1669 WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr); in safe_init_board()
1683 * Configure hardware to use level-triggered interrupts and in safe_init_board()
1707 if (sc->sc_nqchip != 0) { in safe_cleanchip()
1708 struct safe_ringentry *re = sc->sc_back; in safe_cleanchip() local
1710 while (re != sc->sc_front) { in safe_cleanchip()
1711 if (re->re_desc.d_csr != 0) in safe_cleanchip()
1712 safe_free_entry(sc, re); in safe_cleanchip()
1713 if (++re == sc->sc_ringtop) in safe_cleanchip()
1714 re = sc->sc_ring; in safe_cleanchip()
1716 sc->sc_back = re; in safe_cleanchip()
1717 sc->sc_nqchip = 0; in safe_cleanchip()
1726 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re) in safe_free_entry() argument
1733 if (re->re_dst_m != NULL) in safe_free_entry()
1734 m_freem(re->re_dst_m); in safe_free_entry()
1736 crp = (struct cryptop *)re->re_crp; in safe_free_entry()
1738 re->re_desc.d_csr = 0; in safe_free_entry()
1740 crp->crp_etype = EFAULT; in safe_free_entry()
1759 * segment must be aligned on a 32-bit boundary and all
1767 for (i = 0; i < op->nsegs; i++) { in safe_dmamap_aligned()
1768 if (op->segs[i].ds_addr & 3) in safe_dmamap_aligned()
1770 if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3)) in safe_dmamap_aligned()
1790 if (op->nsegs > 0) { in safe_dmamap_uniform()
1793 for (i = 0; i < op->nsegs-1; i++) { in safe_dmamap_uniform()
1794 if (op->segs[i].ds_len % SAFE_MAX_DSIZE) in safe_dmamap_uniform()
1796 if (op->segs[i].ds_len != SAFE_MAX_DSIZE) in safe_dmamap_uniform()
1838 (unsigned long)(sc->sc_back - sc->sc_ring), in safe_dump_ringstate()
1839 (unsigned long)(sc->sc_front - sc->sc_ring)); in safe_dump_ringstate()
1843 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re) in safe_dump_request() argument
1847 ix = re - sc->sc_ring; in safe_dump_request()
1850 , re, ix in safe_dump_request()
1851 , re->re_desc.d_csr in safe_dump_request()
1852 , re->re_desc.d_src in safe_dump_request()
1853 , re->re_desc.d_dst in safe_dump_request()
1854 , re->re_desc.d_sa in safe_dump_request()
1855 , re->re_desc.d_len in safe_dump_request()
1857 if (re->re_src.nsegs > 1) { in safe_dump_request()
1858 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) / in safe_dump_request()
1860 for (nsegs = re->re_src.nsegs; nsegs; nsegs--) { in safe_dump_request()
1862 , ix, &sc->sc_spring[ix] in safe_dump_request()
1863 , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr in safe_dump_request()
1864 , sc->sc_spring[ix].pd_size in safe_dump_request()
1865 , sc->sc_spring[ix].pd_flags in safe_dump_request()
1867 if (sc->sc_spring[ix].pd_size == 0) in safe_dump_request()
1874 if (re->re_dst.nsegs > 1) { in safe_dump_request()
1875 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) / in safe_dump_request()
1877 for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) { in safe_dump_request()
1879 , ix, &sc->sc_dpring[ix] in safe_dump_request()
1880 , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr in safe_dump_request()
1881 , sc->sc_dpring[ix].pd_flags in safe_dump_request()
1888 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec); in safe_dump_request()
1890 , re->re_sa.sa_key[0] in safe_dump_request()
1891 , re->re_sa.sa_key[1] in safe_dump_request()
1892 , re->re_sa.sa_key[2] in safe_dump_request()
1893 , re->re_sa.sa_key[3] in safe_dump_request()
1894 , re->re_sa.sa_key[4] in safe_dump_request()
1895 , re->re_sa.sa_key[5] in safe_dump_request()
1896 , re->re_sa.sa_key[6] in safe_dump_request()
1897 , re->re_sa.sa_key[7] in safe_dump_request()
1900 , re->re_sa.sa_indigest[0] in safe_dump_request()
1901 , re->re_sa.sa_indigest[1] in safe_dump_request()
1902 , re->re_sa.sa_indigest[2] in safe_dump_request()
1903 , re->re_sa.sa_indigest[3] in safe_dump_request()
1904 , re->re_sa.sa_indigest[4] in safe_dump_request()
1907 , re->re_sa.sa_outdigest[0] in safe_dump_request()
1908 , re->re_sa.sa_outdigest[1] in safe_dump_request()
1909 , re->re_sa.sa_outdigest[2] in safe_dump_request()
1910 , re->re_sa.sa_outdigest[3] in safe_dump_request()
1911 , re->re_sa.sa_outdigest[4] in safe_dump_request()
1914 , re->re_sastate.sa_saved_iv[0] in safe_dump_request()
1915 , re->re_sastate.sa_saved_iv[1] in safe_dump_request()
1916 , re->re_sastate.sa_saved_iv[2] in safe_dump_request()
1917 , re->re_sastate.sa_saved_iv[3] in safe_dump_request()
1920 , re->re_sastate.sa_saved_hashbc in safe_dump_request()
1921 , re->re_sastate.sa_saved_indigest[0] in safe_dump_request()
1922 , re->re_sastate.sa_saved_indigest[1] in safe_dump_request()
1923 , re->re_sastate.sa_saved_indigest[2] in safe_dump_request()
1924 , re->re_sastate.sa_saved_indigest[3] in safe_dump_request()
1925 , re->re_sastate.sa_saved_indigest[4] in safe_dump_request()
1932 mtx_lock(&sc->sc_ringmtx); in safe_dump_ring()
1937 if (sc->sc_nqchip) { in safe_dump_ring()
1938 struct safe_ringentry *re = sc->sc_back; in safe_dump_ring() local
1940 safe_dump_request(sc, tag, re); in safe_dump_ring()
1941 if (++re == sc->sc_ringtop) in safe_dump_ring()
1942 re = sc->sc_ring; in safe_dump_ring()
1943 } while (re != sc->sc_front); in safe_dump_ring()
1945 mtx_unlock(&sc->sc_ringmtx); in safe_dump_ring()
1954 strncpy(dmode, "", sizeof(dmode) - 1); in sysctl_hw_safe_dump()
1955 dmode[sizeof(dmode) - 1] = '\0'; in sysctl_hw_safe_dump()
1958 if (error == 0 && req->newptr != NULL) { in sysctl_hw_safe_dump()