331 		struct safe_ringentry *re = &sc->sc_ring[i];  in safe_attach()  local
333 		re->re_desc.d_sa = raddr +  in safe_attach()
335 		re->re_sa.sa_staterec = raddr +  in safe_attach()
542 			struct safe_ringentry *re = sc->sc_back;  in safe_intr()  local
546 				safe_dump_request(sc, __func__, re);  in safe_intr()
556 			if (re->re_desc.d_csr != 0) {  in safe_intr()
557 				if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))  in safe_intr()
559 				if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))  in safe_intr()
562 				safe_callback(sc, re);  in safe_intr()
596 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)  in safe_feed()  argument
598 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE);  in safe_feed()
599 	if (re->re_dst_map != NULL)  in safe_feed()
600 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,  in safe_feed()
611 		safe_dump_request(sc, __func__, re);  in safe_feed()
774 	struct safe_ringentry *re;  in safe_process()  local
786 	re = sc->sc_front;  in safe_process()
788 	staterec = re->re_sa.sa_staterec;	/* save */  in safe_process()
790 	bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));  in safe_process()
791 	re->re_sa.sa_staterec = staterec;	/* restore */  in safe_process()
793 	re->re_crp = crp;  in safe_process()
795 	sa = &re->re_sa;  in safe_process()
838 		crypto_read_iv(crp, re->re_sastate.sa_saved_iv);  in safe_process()
891 		re->re_flags |= SAFE_QFLAGS_COPYOUTICV;  in safe_process()
967 	if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) {  in safe_process()
972 	if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb,  in safe_process()
973 	    &re->re_src, BUS_DMA_NOWAIT) != 0) {  in safe_process()
974 		bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);  in safe_process()
975 		re->re_src_map = NULL;  in safe_process()
980 	re->re_src_mapsize = crypto_buffer_len(&crp->crp_buf);  in safe_process()
981 	nicealign = safe_dmamap_aligned(&re->re_src);  in safe_process()
982 	uniform = safe_dmamap_uniform(&re->re_src);  in safe_process()
985 		nicealign, uniform, re->re_src.nsegs));  in safe_process()
986 	if (re->re_src.nsegs > 1) {  in safe_process()
987 		re->re_desc.d_src = sc->sc_spalloc.dma_paddr +  in safe_process()
989 		for (i = 0; i < re->re_src_nsegs; i++) {  in safe_process()
999 			pd->pd_addr = re->re_src_segs[i].ds_addr;  in safe_process()
1000 			pd->pd_size = re->re_src_segs[i].ds_len;  in safe_process()
1008 		re->re_desc.d_src = re->re_src_segs[0].ds_addr;  in safe_process()
1021 			re->re_dst = re->re_src;  in safe_process()
1032 			    &re->re_dst_map)) {  in safe_process()
1037 			if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map,  in safe_process()
1038 			    crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) !=  in safe_process()
1041 				    re->re_dst_map);  in safe_process()
1042 				re->re_dst_map = NULL;  in safe_process()
1063 			totlen = re->re_src_mapsize;  in safe_process()
1123 			re->re_dst_m = top;  in safe_process()
1125 			    BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {  in safe_process()
1131 			    re->re_dst_map, top, re->re_dst_segs,  in safe_process()
1132 			    &re->re_dst_nsegs, 0) != 0) {  in safe_process()
1134 				    re->re_dst_map);  in safe_process()
1135 				re->re_dst_map = NULL;  in safe_process()
1140 			re->re_dst_mapsize = re->re_src_mapsize;  in safe_process()
1141 			if (re->re_src.mapsize > oplen) {  in safe_process()
1150 				    (re->re_src.mapsize-oplen) == ses->ses_mlen &&  in safe_process()
1153 					    re->re_dst_m, oplen);  in safe_process()
1182 		if (re->re_dst.nsegs > 1) {  in safe_process()
1183 			re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +  in safe_process()
1185 			for (i = 0; i < re->re_dst_nsegs; i++) {  in safe_process()
1193 				pd->pd_addr = re->re_dst_segs[i].ds_addr;  in safe_process()
1201 			re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;  in safe_process()
1224 	re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;  in safe_process()
1226 		re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;  in safe_process()
1227 	re->re_desc.d_len = oplen  in safe_process()
1239 	safe_feed(sc, re);  in safe_process()
1244 	if (re->re_dst_m != NULL)  in safe_process()
1245 		m_freem(re->re_dst_m);  in safe_process()
1247 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {  in safe_process()
1248 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);  in safe_process()
1249 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);  in safe_process()
1251 	if (re->re_src_map != NULL) {  in safe_process()
1252 		bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);  in safe_process()
1253 		bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);  in safe_process()
1267 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)  in safe_callback()  argument
1270 	struct cryptop *crp = (struct cryptop *)re->re_crp;  in safe_callback()
1278 	safestats.st_obytes += re->re_dst.mapsize;  in safe_callback()
1282 	if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {  in safe_callback()
1284 			re->re_desc.d_csr,  in safe_callback()
1285 			re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);  in safe_callback()
1291 	 * XXX: Should crp_buf.cb_mbuf be updated to re->re_dst_m if  in safe_callback()
1295 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {  in safe_callback()
1296 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,  in safe_callback()
1298 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);  in safe_callback()
1299 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);  in safe_callback()
1301 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE);  in safe_callback()
1302 	bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);  in safe_callback()
1303 	bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);  in safe_callback()
1305 	if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {  in safe_callback()
1311 			re->re_sastate.sa_saved_indigest[0] =  in safe_callback()
1312 			    bswap32(re->re_sastate.sa_saved_indigest[0]);  in safe_callback()
1313 			re->re_sastate.sa_saved_indigest[1] =  in safe_callback()
1314 			    bswap32(re->re_sastate.sa_saved_indigest[1]);  in safe_callback()
1315 			re->re_sastate.sa_saved_indigest[2] =  in safe_callback()
1316 			    bswap32(re->re_sastate.sa_saved_indigest[2]);  in safe_callback()
1322 			if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest,  in safe_callback()
1327 			    ses->ses_mlen, re->re_sastate.sa_saved_indigest);  in safe_callback()
1708 		struct safe_ringentry *re = sc->sc_back;  in safe_cleanchip()  local
1710 		while (re != sc->sc_front) {  in safe_cleanchip()
1711 			if (re->re_desc.d_csr != 0)  in safe_cleanchip()
1712 				safe_free_entry(sc, re);  in safe_cleanchip()
1713 			if (++re == sc->sc_ringtop)  in safe_cleanchip()
1714 				re = sc->sc_ring;  in safe_cleanchip()
1716 		sc->sc_back = re;  in safe_cleanchip()
1726 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)  in safe_free_entry()  argument
1733 	if (re->re_dst_m != NULL)  in safe_free_entry()
1734 		m_freem(re->re_dst_m);  in safe_free_entry()
1736 	crp = (struct cryptop *)re->re_crp;  in safe_free_entry()
1738 	re->re_desc.d_csr = 0;  in safe_free_entry()
1843 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)  in safe_dump_request()  argument
1847 	ix = re - sc->sc_ring;  in safe_dump_request()
1850 		, re, ix  in safe_dump_request()
1851 		, re->re_desc.d_csr  in safe_dump_request()
1852 		, re->re_desc.d_src  in safe_dump_request()
1853 		, re->re_desc.d_dst  in safe_dump_request()
1854 		, re->re_desc.d_sa  in safe_dump_request()
1855 		, re->re_desc.d_len  in safe_dump_request()
1857 	if (re->re_src.nsegs > 1) {  in safe_dump_request()
1858 		ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /  in safe_dump_request()
1860 		for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {  in safe_dump_request()
1874 	if (re->re_dst.nsegs > 1) {  in safe_dump_request()
1875 		ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /  in safe_dump_request()
1877 		for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {  in safe_dump_request()
1888 		re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);  in safe_dump_request()
1890 		, re->re_sa.sa_key[0]  in safe_dump_request()
1891 		, re->re_sa.sa_key[1]  in safe_dump_request()
1892 		, re->re_sa.sa_key[2]  in safe_dump_request()
1893 		, re->re_sa.sa_key[3]  in safe_dump_request()
1894 		, re->re_sa.sa_key[4]  in safe_dump_request()
1895 		, re->re_sa.sa_key[5]  in safe_dump_request()
1896 		, re->re_sa.sa_key[6]  in safe_dump_request()
1897 		, re->re_sa.sa_key[7]  in safe_dump_request()
1900 		, re->re_sa.sa_indigest[0]  in safe_dump_request()
1901 		, re->re_sa.sa_indigest[1]  in safe_dump_request()
1902 		, re->re_sa.sa_indigest[2]  in safe_dump_request()
1903 		, re->re_sa.sa_indigest[3]  in safe_dump_request()
1904 		, re->re_sa.sa_indigest[4]  in safe_dump_request()
1907 		, re->re_sa.sa_outdigest[0]  in safe_dump_request()
1908 		, re->re_sa.sa_outdigest[1]  in safe_dump_request()
1909 		, re->re_sa.sa_outdigest[2]  in safe_dump_request()
1910 		, re->re_sa.sa_outdigest[3]  in safe_dump_request()
1911 		, re->re_sa.sa_outdigest[4]  in safe_dump_request()
1914 		, re->re_sastate.sa_saved_iv[0]  in safe_dump_request()
1915 		, re->re_sastate.sa_saved_iv[1]  in safe_dump_request()
1916 		, re->re_sastate.sa_saved_iv[2]  in safe_dump_request()
1917 		, re->re_sastate.sa_saved_iv[3]  in safe_dump_request()
1920 		, re->re_sastate.sa_saved_hashbc  in safe_dump_request()
1921 		, re->re_sastate.sa_saved_indigest[0]  in safe_dump_request()
1922 		, re->re_sastate.sa_saved_indigest[1]  in safe_dump_request()
1923 		, re->re_sastate.sa_saved_indigest[2]  in safe_dump_request()
1924 		, re->re_sastate.sa_saved_indigest[3]  in safe_dump_request()
1925 		, re->re_sastate.sa_saved_indigest[4]  in safe_dump_request()
1938 		struct safe_ringentry *re = sc->sc_back;  in safe_dump_ring()  local
1940 			safe_dump_request(sc, tag, re);  in safe_dump_ring()
1941 			if (++re == sc->sc_ringtop)  in safe_dump_ring()
1942 				re = sc->sc_ring;  in safe_dump_ring()
1943 		} while (re != sc->sc_front);  in safe_dump_ring()