331 		struct safe_ringentry *re = &sc->sc_ring[i];  in safe_attach()  local
333 		re->re_desc.d_sa = raddr +  in safe_attach()
335 		re->re_sa.sa_staterec = raddr +  in safe_attach()
540 			struct safe_ringentry *re = sc->sc_back;  in safe_intr()  local
544 				safe_dump_request(sc, __func__, re);  in safe_intr()
554 			if (re->re_desc.d_csr != 0) {  in safe_intr()
555 				if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))  in safe_intr()
557 				if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))  in safe_intr()
560 				safe_callback(sc, re);  in safe_intr()
594 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)  in safe_feed()  argument
596 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE);  in safe_feed()
597 	if (re->re_dst_map != NULL)  in safe_feed()
598 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,  in safe_feed()
609 		safe_dump_request(sc, __func__, re);  in safe_feed()
772 	struct safe_ringentry *re;  in safe_process()  local
784 	re = sc->sc_front;  in safe_process()
786 	staterec = re->re_sa.sa_staterec;	/* save */  in safe_process()
788 	bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));  in safe_process()
789 	re->re_sa.sa_staterec = staterec;	/* restore */  in safe_process()
791 	re->re_crp = crp;  in safe_process()
793 	sa = &re->re_sa;  in safe_process()
836 		crypto_read_iv(crp, re->re_sastate.sa_saved_iv);  in safe_process()
889 		re->re_flags |= SAFE_QFLAGS_COPYOUTICV;  in safe_process()
965 	if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) {  in safe_process()
970 	if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb,  in safe_process()
971 	    &re->re_src, BUS_DMA_NOWAIT) != 0) {  in safe_process()
972 		bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);  in safe_process()
973 		re->re_src_map = NULL;  in safe_process()
978 	re->re_src_mapsize = crypto_buffer_len(&crp->crp_buf);  in safe_process()
979 	nicealign = safe_dmamap_aligned(&re->re_src);  in safe_process()
980 	uniform = safe_dmamap_uniform(&re->re_src);  in safe_process()
983 		nicealign, uniform, re->re_src.nsegs));  in safe_process()
984 	if (re->re_src.nsegs > 1) {  in safe_process()
985 		re->re_desc.d_src = sc->sc_spalloc.dma_paddr +  in safe_process()
987 		for (i = 0; i < re->re_src_nsegs; i++) {  in safe_process()
997 			pd->pd_addr = re->re_src_segs[i].ds_addr;  in safe_process()
998 			pd->pd_size = re->re_src_segs[i].ds_len;  in safe_process()
1006 		re->re_desc.d_src = re->re_src_segs[0].ds_addr;  in safe_process()
1019 			re->re_dst = re->re_src;  in safe_process()
1030 			    &re->re_dst_map)) {  in safe_process()
1035 			if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map,  in safe_process()
1036 			    crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) !=  in safe_process()
1039 				    re->re_dst_map);  in safe_process()
1040 				re->re_dst_map = NULL;  in safe_process()
1061 			totlen = re->re_src_mapsize;  in safe_process()
1121 			re->re_dst_m = top;  in safe_process()
1123 			    BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {  in safe_process()
1129 			    re->re_dst_map, top, re->re_dst_segs,  in safe_process()
1130 			    &re->re_dst_nsegs, 0) != 0) {  in safe_process()
1132 				    re->re_dst_map);  in safe_process()
1133 				re->re_dst_map = NULL;  in safe_process()
1138 			re->re_dst_mapsize = re->re_src_mapsize;  in safe_process()
1139 			if (re->re_src.mapsize > oplen) {  in safe_process()
1148 				    (re->re_src.mapsize-oplen) == ses->ses_mlen &&  in safe_process()
1151 					    re->re_dst_m, oplen);  in safe_process()
1180 		if (re->re_dst.nsegs > 1) {  in safe_process()
1181 			re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +  in safe_process()
1183 			for (i = 0; i < re->re_dst_nsegs; i++) {  in safe_process()
1191 				pd->pd_addr = re->re_dst_segs[i].ds_addr;  in safe_process()
1199 			re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;  in safe_process()
1222 	re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;  in safe_process()
1224 		re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;  in safe_process()
1225 	re->re_desc.d_len = oplen  in safe_process()
1237 	safe_feed(sc, re);  in safe_process()
1242 	if (re->re_dst_m != NULL)  in safe_process()
1243 		m_freem(re->re_dst_m);  in safe_process()
1245 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {  in safe_process()
1246 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);  in safe_process()
1247 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);  in safe_process()
1249 	if (re->re_src_map != NULL) {  in safe_process()
1250 		bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);  in safe_process()
1251 		bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);  in safe_process()
1265 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)  in safe_callback()  argument
1268 	struct cryptop *crp = (struct cryptop *)re->re_crp;  in safe_callback()
1276 	safestats.st_obytes += re->re_dst.mapsize;  in safe_callback()
1280 	if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {  in safe_callback()
1282 			re->re_desc.d_csr,  in safe_callback()
1283 			re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);  in safe_callback()
1289 	 * XXX: Should crp_buf.cb_mbuf be updated to re->re_dst_m if  in safe_callback()
1293 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {  in safe_callback()
1294 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,  in safe_callback()
1296 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);  in safe_callback()
1297 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);  in safe_callback()
1299 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE);  in safe_callback()
1300 	bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);  in safe_callback()
1301 	bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);  in safe_callback()
1303 	if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {  in safe_callback()
1309 			re->re_sastate.sa_saved_indigest[0] =  in safe_callback()
1310 			    bswap32(re->re_sastate.sa_saved_indigest[0]);  in safe_callback()
1311 			re->re_sastate.sa_saved_indigest[1] =  in safe_callback()
1312 			    bswap32(re->re_sastate.sa_saved_indigest[1]);  in safe_callback()
1313 			re->re_sastate.sa_saved_indigest[2] =  in safe_callback()
1314 			    bswap32(re->re_sastate.sa_saved_indigest[2]);  in safe_callback()
1320 			if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest,  in safe_callback()
1325 			    ses->ses_mlen, re->re_sastate.sa_saved_indigest);  in safe_callback()
1706 		struct safe_ringentry *re = sc->sc_back;  in safe_cleanchip()  local
1708 		while (re != sc->sc_front) {  in safe_cleanchip()
1709 			if (re->re_desc.d_csr != 0)  in safe_cleanchip()
1710 				safe_free_entry(sc, re);  in safe_cleanchip()
1711 			if (++re == sc->sc_ringtop)  in safe_cleanchip()
1712 				re = sc->sc_ring;  in safe_cleanchip()
1714 		sc->sc_back = re;  in safe_cleanchip()
1724 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)  in safe_free_entry()  argument
1731 	if (re->re_dst_m != NULL)  in safe_free_entry()
1732 		m_freem(re->re_dst_m);  in safe_free_entry()
1734 	crp = (struct cryptop *)re->re_crp;  in safe_free_entry()
1736 	re->re_desc.d_csr = 0;  in safe_free_entry()
1841 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)  in safe_dump_request()  argument
1845 	ix = re - sc->sc_ring;  in safe_dump_request()
1848 		, re, ix  in safe_dump_request()
1849 		, re->re_desc.d_csr  in safe_dump_request()
1850 		, re->re_desc.d_src  in safe_dump_request()
1851 		, re->re_desc.d_dst  in safe_dump_request()
1852 		, re->re_desc.d_sa  in safe_dump_request()
1853 		, re->re_desc.d_len  in safe_dump_request()
1855 	if (re->re_src.nsegs > 1) {  in safe_dump_request()
1856 		ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /  in safe_dump_request()
1858 		for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {  in safe_dump_request()
1872 	if (re->re_dst.nsegs > 1) {  in safe_dump_request()
1873 		ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /  in safe_dump_request()
1875 		for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {  in safe_dump_request()
1886 		re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);  in safe_dump_request()
1888 		, re->re_sa.sa_key[0]  in safe_dump_request()
1889 		, re->re_sa.sa_key[1]  in safe_dump_request()
1890 		, re->re_sa.sa_key[2]  in safe_dump_request()
1891 		, re->re_sa.sa_key[3]  in safe_dump_request()
1892 		, re->re_sa.sa_key[4]  in safe_dump_request()
1893 		, re->re_sa.sa_key[5]  in safe_dump_request()
1894 		, re->re_sa.sa_key[6]  in safe_dump_request()
1895 		, re->re_sa.sa_key[7]  in safe_dump_request()
1898 		, re->re_sa.sa_indigest[0]  in safe_dump_request()
1899 		, re->re_sa.sa_indigest[1]  in safe_dump_request()
1900 		, re->re_sa.sa_indigest[2]  in safe_dump_request()
1901 		, re->re_sa.sa_indigest[3]  in safe_dump_request()
1902 		, re->re_sa.sa_indigest[4]  in safe_dump_request()
1905 		, re->re_sa.sa_outdigest[0]  in safe_dump_request()
1906 		, re->re_sa.sa_outdigest[1]  in safe_dump_request()
1907 		, re->re_sa.sa_outdigest[2]  in safe_dump_request()
1908 		, re->re_sa.sa_outdigest[3]  in safe_dump_request()
1909 		, re->re_sa.sa_outdigest[4]  in safe_dump_request()
1912 		, re->re_sastate.sa_saved_iv[0]  in safe_dump_request()
1913 		, re->re_sastate.sa_saved_iv[1]  in safe_dump_request()
1914 		, re->re_sastate.sa_saved_iv[2]  in safe_dump_request()
1915 		, re->re_sastate.sa_saved_iv[3]  in safe_dump_request()
1918 		, re->re_sastate.sa_saved_hashbc  in safe_dump_request()
1919 		, re->re_sastate.sa_saved_indigest[0]  in safe_dump_request()
1920 		, re->re_sastate.sa_saved_indigest[1]  in safe_dump_request()
1921 		, re->re_sastate.sa_saved_indigest[2]  in safe_dump_request()
1922 		, re->re_sastate.sa_saved_indigest[3]  in safe_dump_request()
1923 		, re->re_sastate.sa_saved_indigest[4]  in safe_dump_request()
1936 		struct safe_ringentry *re = sc->sc_back;  in safe_dump_ring()  local
1938 			safe_dump_request(sc, tag, re);  in safe_dump_ring()
1939 			if (++re == sc->sc_ringtop)  in safe_dump_ring()
1940 				re = sc->sc_ring;  in safe_dump_ring()
1941 		} while (re != sc->sc_front);  in safe_dump_ring()