xref: /freebsd/sys/opencrypto/cryptosoft.c (revision 190cef3d52236565eb22e18b33e9e865ec634aa3)
1 /*	$OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $	*/
2 
3 /*-
4  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
5  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
6  *
7  * This code was written by Angelos D. Keromytis in Athens, Greece, in
8  * February 2000. Network Security Technologies Inc. (NSTI) kindly
9  * supported the development of this code.
10  *
11  * Copyright (c) 2000, 2001 Angelos D. Keromytis
12  * Copyright (c) 2014 The FreeBSD Foundation
13  * All rights reserved.
14  *
15  * Portions of this software were developed by John-Mark Gurney
16  * under sponsorship of the FreeBSD Foundation and
17  * Rubicon Communications, LLC (Netgate).
18  *
19  * Permission to use, copy, and modify this software with or without fee
20  * is hereby granted, provided that this entire notice is included in
21  * all source code copies of any software which is or includes a copy or
22  * modification of this software.
23  *
24  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
25  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
26  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
27  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
28  * PURPOSE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/malloc.h>
37 #include <sys/mbuf.h>
38 #include <sys/module.h>
39 #include <sys/sysctl.h>
40 #include <sys/errno.h>
41 #include <sys/random.h>
42 #include <sys/kernel.h>
43 #include <sys/uio.h>
44 #include <sys/lock.h>
45 #include <sys/rwlock.h>
46 #include <sys/endian.h>
47 #include <sys/limits.h>
48 
49 #include <crypto/blowfish/blowfish.h>
50 #include <crypto/sha1.h>
51 #include <opencrypto/rmd160.h>
52 #include <opencrypto/cast.h>
53 #include <opencrypto/skipjack.h>
54 #include <sys/md5.h>
55 
56 #include <opencrypto/cryptodev.h>
57 #include <opencrypto/cryptosoft.h>
58 #include <opencrypto/xform.h>
59 
60 #include <sys/kobj.h>
61 #include <sys/bus.h>
62 #include "cryptodev_if.h"
63 
64 static	int32_t swcr_id;
65 
66 u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
67 u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
68 
69 static	int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
70 static	int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
71 static	int swcr_authenc(struct cryptop *crp);
72 static	int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
73 static	void swcr_freesession(device_t dev, crypto_session_t cses);
74 
75 /*
76  * Apply a symmetric encryption/decryption algorithm.
77  */
78 static int
79 swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
80     int flags)
81 {
82 	unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN];
83 	unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN];
84 	struct enc_xform *exf;
85 	int i, j, k, blks, ind, count, ivlen;
86 	struct uio *uio, uiolcl;
87 	struct iovec iovlcl[4];
88 	struct iovec *iov;
89 	int iovcnt, iovalloc;
90 	int error;
91 
92 	error = 0;
93 
94 	exf = sw->sw_exf;
95 	blks = exf->blocksize;
96 	ivlen = exf->ivsize;
97 
98 	/* Check for non-padded data */
99 	if (crd->crd_len % blks)
100 		return EINVAL;
101 
102 	if (crd->crd_alg == CRYPTO_AES_ICM &&
103 	    (crd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
104 		return (EINVAL);
105 
106 	/* Initialize the IV */
107 	if (crd->crd_flags & CRD_F_ENCRYPT) {
108 		/* IV explicitly provided ? */
109 		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
110 			bcopy(crd->crd_iv, iv, ivlen);
111 		else
112 			arc4rand(iv, ivlen, 0);
113 
114 		/* Do we need to write the IV */
115 		if (!(crd->crd_flags & CRD_F_IV_PRESENT))
116 			crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
117 
118 	} else {	/* Decryption */
119 		/* IV explicitly provided ? */
120 		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
121 			bcopy(crd->crd_iv, iv, ivlen);
122 		else {
123 			/* Get IV off buf */
124 			crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
125 		}
126 	}
127 
128 	if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
129 		int error;
130 
131 		if (sw->sw_kschedule)
132 			exf->zerokey(&(sw->sw_kschedule));
133 
134 		error = exf->setkey(&sw->sw_kschedule,
135 				crd->crd_key, crd->crd_klen / 8);
136 		if (error)
137 			return (error);
138 	}
139 
140 	iov = iovlcl;
141 	iovcnt = nitems(iovlcl);
142 	iovalloc = 0;
143 	uio = &uiolcl;
144 	if ((flags & CRYPTO_F_IMBUF) != 0) {
145 		error = crypto_mbuftoiov((struct mbuf *)buf, &iov, &iovcnt,
146 		    &iovalloc);
147 		if (error)
148 			return (error);
149 		uio->uio_iov = iov;
150 		uio->uio_iovcnt = iovcnt;
151 	} else if ((flags & CRYPTO_F_IOV) != 0)
152 		uio = (struct uio *)buf;
153 	else {
154 		iov[0].iov_base = buf;
155 		iov[0].iov_len = crd->crd_skip + crd->crd_len;
156 		uio->uio_iov = iov;
157 		uio->uio_iovcnt = 1;
158 	}
159 
160 	ivp = iv;
161 
162 	if (exf->reinit) {
163 		/*
164 		 * xforms that provide a reinit method perform all IV
165 		 * handling themselves.
166 		 */
167 		exf->reinit(sw->sw_kschedule, iv);
168 	}
169 
170 	count = crd->crd_skip;
171 	ind = cuio_getptr(uio, count, &k);
172 	if (ind == -1) {
173 		error = EINVAL;
174 		goto out;
175 	}
176 
177 	i = crd->crd_len;
178 
179 	while (i > 0) {
180 		/*
181 		 * If there's insufficient data at the end of
182 		 * an iovec, we have to do some copying.
183 		 */
184 		if (uio->uio_iov[ind].iov_len < k + blks &&
185 		    uio->uio_iov[ind].iov_len != k) {
186 			cuio_copydata(uio, count, blks, blk);
187 
188 			/* Actual encryption/decryption */
189 			if (exf->reinit) {
190 				if (crd->crd_flags & CRD_F_ENCRYPT) {
191 					exf->encrypt(sw->sw_kschedule,
192 					    blk);
193 				} else {
194 					exf->decrypt(sw->sw_kschedule,
195 					    blk);
196 				}
197 			} else if (crd->crd_flags & CRD_F_ENCRYPT) {
198 				/* XOR with previous block */
199 				for (j = 0; j < blks; j++)
200 					blk[j] ^= ivp[j];
201 
202 				exf->encrypt(sw->sw_kschedule, blk);
203 
204 				/*
205 				 * Keep encrypted block for XOR'ing
206 				 * with next block
207 				 */
208 				bcopy(blk, iv, blks);
209 				ivp = iv;
210 			} else {	/* decrypt */
211 				/*
212 				 * Keep encrypted block for XOR'ing
213 				 * with next block
214 				 */
215 				nivp = (ivp == iv) ? iv2 : iv;
216 				bcopy(blk, nivp, blks);
217 
218 				exf->decrypt(sw->sw_kschedule, blk);
219 
220 				/* XOR with previous block */
221 				for (j = 0; j < blks; j++)
222 					blk[j] ^= ivp[j];
223 
224 				ivp = nivp;
225 			}
226 
227 			/* Copy back decrypted block */
228 			cuio_copyback(uio, count, blks, blk);
229 
230 			count += blks;
231 
232 			/* Advance pointer */
233 			ind = cuio_getptr(uio, count, &k);
234 			if (ind == -1) {
235 				error = EINVAL;
236 				goto out;
237 			}
238 
239 			i -= blks;
240 
241 			/* Could be done... */
242 			if (i == 0)
243 				break;
244 		}
245 
246 		while (uio->uio_iov[ind].iov_len >= k + blks && i > 0) {
247 			uint8_t *idat;
248 			size_t nb, rem;
249 
250 			nb = blks;
251 			rem = MIN((size_t)i,
252 			    uio->uio_iov[ind].iov_len - (size_t)k);
253 			idat = (uint8_t *)uio->uio_iov[ind].iov_base + k;
254 
255 			if (exf->reinit) {
256 				if ((crd->crd_flags & CRD_F_ENCRYPT) != 0 &&
257 				    exf->encrypt_multi == NULL)
258 					exf->encrypt(sw->sw_kschedule,
259 					    idat);
260 				else if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) {
261 					nb = rounddown(rem, blks);
262 					exf->encrypt_multi(sw->sw_kschedule,
263 					    idat, nb);
264 				} else if (exf->decrypt_multi == NULL)
265 					exf->decrypt(sw->sw_kschedule,
266 					    idat);
267 				else {
268 					nb = rounddown(rem, blks);
269 					exf->decrypt_multi(sw->sw_kschedule,
270 					    idat, nb);
271 				}
272 			} else if (crd->crd_flags & CRD_F_ENCRYPT) {
273 				/* XOR with previous block/IV */
274 				for (j = 0; j < blks; j++)
275 					idat[j] ^= ivp[j];
276 
277 				exf->encrypt(sw->sw_kschedule, idat);
278 				ivp = idat;
279 			} else {	/* decrypt */
280 				/*
281 				 * Keep encrypted block to be used
282 				 * in next block's processing.
283 				 */
284 				nivp = (ivp == iv) ? iv2 : iv;
285 				bcopy(idat, nivp, blks);
286 
287 				exf->decrypt(sw->sw_kschedule, idat);
288 
289 				/* XOR with previous block/IV */
290 				for (j = 0; j < blks; j++)
291 					idat[j] ^= ivp[j];
292 
293 				ivp = nivp;
294 			}
295 
296 			count += nb;
297 			k += nb;
298 			i -= nb;
299 		}
300 
301 		/*
302 		 * Advance to the next iov if the end of the current iov
303 		 * is aligned with the end of a cipher block.
304 		 * Note that the code is equivalent to calling:
305 		 *      ind = cuio_getptr(uio, count, &k);
306 		 */
307 		if (i > 0 && k == uio->uio_iov[ind].iov_len) {
308 			k = 0;
309 			ind++;
310 			if (ind >= uio->uio_iovcnt) {
311 				error = EINVAL;
312 				goto out;
313 			}
314 		}
315 	}
316 
317 out:
318 	if (iovalloc)
319 		free(iov, M_CRYPTO_DATA);
320 
321 	return (error);
322 }
323 
324 static int __result_use_check
325 swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
326     int klen)
327 {
328 	int k;
329 
330 	klen /= 8;
331 
332 	switch (axf->type) {
333 	case CRYPTO_MD5_HMAC:
334 	case CRYPTO_SHA1_HMAC:
335 	case CRYPTO_SHA2_224_HMAC:
336 	case CRYPTO_SHA2_256_HMAC:
337 	case CRYPTO_SHA2_384_HMAC:
338 	case CRYPTO_SHA2_512_HMAC:
339 	case CRYPTO_NULL_HMAC:
340 	case CRYPTO_RIPEMD160_HMAC:
341 		for (k = 0; k < klen; k++)
342 			key[k] ^= HMAC_IPAD_VAL;
343 
344 		axf->Init(sw->sw_ictx);
345 		axf->Update(sw->sw_ictx, key, klen);
346 		axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
347 
348 		for (k = 0; k < klen; k++)
349 			key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
350 
351 		axf->Init(sw->sw_octx);
352 		axf->Update(sw->sw_octx, key, klen);
353 		axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
354 
355 		for (k = 0; k < klen; k++)
356 			key[k] ^= HMAC_OPAD_VAL;
357 		break;
358 	case CRYPTO_MD5_KPDK:
359 	case CRYPTO_SHA1_KPDK:
360 	{
361 		/*
362 		 * We need a buffer that can hold an md5 and a sha1 result
363 		 * just to throw it away.
364 		 * What we do here is the initial part of:
365 		 *   ALGO( key, keyfill, .. )
366 		 * adding the key to sw_ictx and abusing Final() to get the
367 		 * "keyfill" padding.
368 		 * In addition we abuse the sw_octx to save the key to have
369 		 * it to be able to append it at the end in swcr_authcompute().
370 		 */
371 		u_char buf[SHA1_RESULTLEN];
372 
373 		sw->sw_klen = klen;
374 		bcopy(key, sw->sw_octx, klen);
375 		axf->Init(sw->sw_ictx);
376 		axf->Update(sw->sw_ictx, key, klen);
377 		axf->Final(buf, sw->sw_ictx);
378 		break;
379 	}
380 	case CRYPTO_POLY1305:
381 		if (klen != POLY1305_KEY_LEN) {
382 			CRYPTDEB("bad poly1305 key size %d", klen);
383 			return EINVAL;
384 		}
385 		/* FALLTHROUGH */
386 	case CRYPTO_BLAKE2B:
387 	case CRYPTO_BLAKE2S:
388 		axf->Setkey(sw->sw_ictx, key, klen);
389 		axf->Init(sw->sw_ictx);
390 		break;
391 	default:
392 		printf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
393 		    "doesn't use keys.\n", __func__, axf->type);
394 		return EINVAL;
395 	}
396 	return 0;
397 }
398 
399 /*
400  * Compute keyed-hash authenticator.
401  */
402 static int
403 swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
404     int flags)
405 {
406 	unsigned char aalg[HASH_MAX_LEN];
407 	struct auth_hash *axf;
408 	union authctx ctx;
409 	int err;
410 
411 	if (sw->sw_ictx == 0)
412 		return EINVAL;
413 
414 	axf = sw->sw_axf;
415 
416 	if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
417 		err = swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
418 		if (err != 0)
419 			return err;
420 	}
421 
422 	bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
423 
424 	err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
425 	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
426 	if (err)
427 		return err;
428 
429 	switch (sw->sw_alg) {
430 	case CRYPTO_SHA1:
431 	case CRYPTO_SHA2_224:
432 	case CRYPTO_SHA2_256:
433 	case CRYPTO_SHA2_384:
434 	case CRYPTO_SHA2_512:
435 		axf->Final(aalg, &ctx);
436 		break;
437 
438 	case CRYPTO_MD5_HMAC:
439 	case CRYPTO_SHA1_HMAC:
440 	case CRYPTO_SHA2_224_HMAC:
441 	case CRYPTO_SHA2_256_HMAC:
442 	case CRYPTO_SHA2_384_HMAC:
443 	case CRYPTO_SHA2_512_HMAC:
444 	case CRYPTO_RIPEMD160_HMAC:
445 		if (sw->sw_octx == NULL)
446 			return EINVAL;
447 
448 		axf->Final(aalg, &ctx);
449 		bcopy(sw->sw_octx, &ctx, axf->ctxsize);
450 		axf->Update(&ctx, aalg, axf->hashsize);
451 		axf->Final(aalg, &ctx);
452 		break;
453 
454 	case CRYPTO_MD5_KPDK:
455 	case CRYPTO_SHA1_KPDK:
456 		/* If we have no key saved, return error. */
457 		if (sw->sw_octx == NULL)
458 			return EINVAL;
459 
460 		/*
461 		 * Add the trailing copy of the key (see comment in
462 		 * swcr_authprepare()) after the data:
463 		 *   ALGO( .., key, algofill )
464 		 * and let Final() do the proper, natural "algofill"
465 		 * padding.
466 		 */
467 		axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
468 		axf->Final(aalg, &ctx);
469 		break;
470 
471 	case CRYPTO_BLAKE2B:
472 	case CRYPTO_BLAKE2S:
473 	case CRYPTO_NULL_HMAC:
474 	case CRYPTO_POLY1305:
475 		axf->Final(aalg, &ctx);
476 		break;
477 	}
478 
479 	/* Inject the authentication data */
480 	crypto_copyback(flags, buf, crd->crd_inject,
481 	    sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
482 	return 0;
483 }
484 
485 CTASSERT(INT_MAX <= (1ll<<39) - 256);	/* GCM: plain text < 2^39-256 */
486 CTASSERT(INT_MAX <= (uint64_t)-1);	/* GCM: associated data <= 2^64-1 */
487 
488 /*
489  * Apply a combined encryption-authentication transformation
490  */
491 static int
492 swcr_authenc(struct cryptop *crp)
493 {
494 	uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
495 	u_char *blk = (u_char *)blkbuf;
496 	u_char aalg[AALG_MAX_RESULT_LEN];
497 	u_char uaalg[AALG_MAX_RESULT_LEN];
498 	u_char iv[EALG_MAX_BLOCK_LEN];
499 	union authctx ctx;
500 	struct swcr_session *ses;
501 	struct cryptodesc *crd, *crda = NULL, *crde = NULL;
502 	struct swcr_data *sw, *swa, *swe = NULL;
503 	struct auth_hash *axf = NULL;
504 	struct enc_xform *exf = NULL;
505 	caddr_t buf = (caddr_t)crp->crp_buf;
506 	uint32_t *blkp;
507 	int aadlen, blksz, i, ivlen, len, iskip, oskip, r;
508 
509 	ivlen = blksz = iskip = oskip = 0;
510 
511 	ses = crypto_get_driver_session(crp->crp_session);
512 
513 	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
514 		for (i = 0; i < nitems(ses->swcr_algorithms) &&
515 		    ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)
516 			;
517 		if (i == nitems(ses->swcr_algorithms))
518 			return (EINVAL);
519 
520 		sw = &ses->swcr_algorithms[i];
521 		switch (sw->sw_alg) {
522 		case CRYPTO_AES_NIST_GCM_16:
523 		case CRYPTO_AES_NIST_GMAC:
524 			swe = sw;
525 			crde = crd;
526 			exf = swe->sw_exf;
527 			ivlen = 12;
528 			break;
529 		case CRYPTO_AES_128_NIST_GMAC:
530 		case CRYPTO_AES_192_NIST_GMAC:
531 		case CRYPTO_AES_256_NIST_GMAC:
532 			swa = sw;
533 			crda = crd;
534 			axf = swa->sw_axf;
535 			if (swa->sw_ictx == 0)
536 				return (EINVAL);
537 			bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
538 			blksz = axf->blocksize;
539 			break;
540 		default:
541 			return (EINVAL);
542 		}
543 	}
544 	if (crde == NULL || crda == NULL)
545 		return (EINVAL);
546 
547 	if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16 &&
548 	    (crde->crd_flags & CRD_F_IV_EXPLICIT) == 0)
549 		return (EINVAL);
550 
551 	if (crde->crd_klen != crda->crd_klen)
552 		return (EINVAL);
553 
554 	/* Initialize the IV */
555 	if (crde->crd_flags & CRD_F_ENCRYPT) {
556 		/* IV explicitly provided ? */
557 		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
558 			bcopy(crde->crd_iv, iv, ivlen);
559 		else
560 			arc4rand(iv, ivlen, 0);
561 
562 		/* Do we need to write the IV */
563 		if (!(crde->crd_flags & CRD_F_IV_PRESENT))
564 			crypto_copyback(crp->crp_flags, buf, crde->crd_inject,
565 			    ivlen, iv);
566 
567 	} else {	/* Decryption */
568 			/* IV explicitly provided ? */
569 		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
570 			bcopy(crde->crd_iv, iv, ivlen);
571 		else {
572 			/* Get IV off buf */
573 			crypto_copydata(crp->crp_flags, buf, crde->crd_inject,
574 			    ivlen, iv);
575 		}
576 	}
577 
578 	/* Supply MAC with IV */
579 	if (axf->Reinit)
580 		axf->Reinit(&ctx, iv, ivlen);
581 
582 	/* Supply MAC with AAD */
583 	aadlen = crda->crd_len;
584 
585 	for (i = iskip; i < crda->crd_len; i += blksz) {
586 		len = MIN(crda->crd_len - i, blksz - oskip);
587 		crypto_copydata(crp->crp_flags, buf, crda->crd_skip + i, len,
588 		    blk + oskip);
589 		bzero(blk + len + oskip, blksz - len - oskip);
590 		axf->Update(&ctx, blk, blksz);
591 		oskip = 0; /* reset initial output offset */
592 	}
593 
594 	if (exf->reinit)
595 		exf->reinit(swe->sw_kschedule, iv);
596 
597 	/* Do encryption/decryption with MAC */
598 	for (i = 0; i < crde->crd_len; i += len) {
599 		if (exf->encrypt_multi != NULL) {
600 			len = rounddown(crde->crd_len - i, blksz);
601 			if (len == 0)
602 				len = blksz;
603 			else
604 				len = MIN(len, sizeof(blkbuf));
605 		} else
606 			len = blksz;
607 		len = MIN(crde->crd_len - i, len);
608 		if (len < blksz)
609 			bzero(blk, blksz);
610 		crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len,
611 		    blk);
612 		if (crde->crd_flags & CRD_F_ENCRYPT) {
613 			if (exf->encrypt_multi != NULL)
614 				exf->encrypt_multi(swe->sw_kschedule, blk,
615 				    len);
616 			else
617 				exf->encrypt(swe->sw_kschedule, blk);
618 			axf->Update(&ctx, blk, len);
619 			crypto_copyback(crp->crp_flags, buf,
620 			    crde->crd_skip + i, len, blk);
621 		} else {
622 			axf->Update(&ctx, blk, len);
623 		}
624 	}
625 
626 	/* Do any required special finalization */
627 	switch (crda->crd_alg) {
628 		case CRYPTO_AES_128_NIST_GMAC:
629 		case CRYPTO_AES_192_NIST_GMAC:
630 		case CRYPTO_AES_256_NIST_GMAC:
631 			/* length block */
632 			bzero(blk, blksz);
633 			blkp = (uint32_t *)blk + 1;
634 			*blkp = htobe32(aadlen * 8);
635 			blkp = (uint32_t *)blk + 3;
636 			*blkp = htobe32(crde->crd_len * 8);
637 			axf->Update(&ctx, blk, blksz);
638 			break;
639 	}
640 
641 	/* Finalize MAC */
642 	axf->Final(aalg, &ctx);
643 
644 	/* Validate tag */
645 	if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
646 		crypto_copydata(crp->crp_flags, buf, crda->crd_inject,
647 		    axf->hashsize, uaalg);
648 
649 		r = timingsafe_bcmp(aalg, uaalg, axf->hashsize);
650 		if (r == 0) {
651 			/* tag matches, decrypt data */
652 			for (i = 0; i < crde->crd_len; i += blksz) {
653 				len = MIN(crde->crd_len - i, blksz);
654 				if (len < blksz)
655 					bzero(blk, blksz);
656 				crypto_copydata(crp->crp_flags, buf,
657 				    crde->crd_skip + i, len, blk);
658 				exf->decrypt(swe->sw_kschedule, blk);
659 				crypto_copyback(crp->crp_flags, buf,
660 				    crde->crd_skip + i, len, blk);
661 			}
662 		} else
663 			return (EBADMSG);
664 	} else {
665 		/* Inject the authentication data */
666 		crypto_copyback(crp->crp_flags, buf, crda->crd_inject,
667 		    axf->hashsize, aalg);
668 	}
669 
670 	return (0);
671 }
672 
673 /*
674  * Apply a compression/decompression algorithm
675  */
676 static int
677 swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
678     caddr_t buf, int flags)
679 {
680 	u_int8_t *data, *out;
681 	struct comp_algo *cxf;
682 	int adj;
683 	u_int32_t result;
684 
685 	cxf = sw->sw_cxf;
686 
687 	/* We must handle the whole buffer of data in one time
688 	 * then if there is not all the data in the mbuf, we must
689 	 * copy in a buffer.
690 	 */
691 
692 	data = malloc(crd->crd_len, M_CRYPTO_DATA,  M_NOWAIT);
693 	if (data == NULL)
694 		return (EINVAL);
695 	crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
696 
697 	if (crd->crd_flags & CRD_F_COMP)
698 		result = cxf->compress(data, crd->crd_len, &out);
699 	else
700 		result = cxf->decompress(data, crd->crd_len, &out);
701 
702 	free(data, M_CRYPTO_DATA);
703 	if (result == 0)
704 		return EINVAL;
705 
706 	/* Copy back the (de)compressed data. m_copyback is
707 	 * extending the mbuf as necessary.
708 	 */
709 	sw->sw_size = result;
710 	/* Check the compressed size when doing compression */
711 	if (crd->crd_flags & CRD_F_COMP) {
712 		if (result >= crd->crd_len) {
713 			/* Compression was useless, we lost time */
714 			free(out, M_CRYPTO_DATA);
715 			return 0;
716 		}
717 	}
718 
719 	crypto_copyback(flags, buf, crd->crd_skip, result, out);
720 	if (result < crd->crd_len) {
721 		adj = result - crd->crd_len;
722 		if (flags & CRYPTO_F_IMBUF) {
723 			adj = result - crd->crd_len;
724 			m_adj((struct mbuf *)buf, adj);
725 		} else if (flags & CRYPTO_F_IOV) {
726 			struct uio *uio = (struct uio *)buf;
727 			int ind;
728 
729 			adj = crd->crd_len - result;
730 			ind = uio->uio_iovcnt - 1;
731 
732 			while (adj > 0 && ind >= 0) {
733 				if (adj < uio->uio_iov[ind].iov_len) {
734 					uio->uio_iov[ind].iov_len -= adj;
735 					break;
736 				}
737 
738 				adj -= uio->uio_iov[ind].iov_len;
739 				uio->uio_iov[ind].iov_len = 0;
740 				ind--;
741 				uio->uio_iovcnt--;
742 			}
743 		}
744 	}
745 	free(out, M_CRYPTO_DATA);
746 	return 0;
747 }
748 
749 /*
750  * Generate a new software session.
751  */
752 static int
753 swcr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
754 {
755 	struct swcr_session *ses;
756 	struct swcr_data *swd;
757 	struct auth_hash *axf;
758 	struct enc_xform *txf;
759 	struct comp_algo *cxf;
760 	size_t i;
761 	int len;
762 	int error;
763 
764 	if (cses == NULL || cri == NULL)
765 		return EINVAL;
766 
767 	ses = crypto_get_driver_session(cses);
768 
769 	for (i = 0; cri != NULL && i < nitems(ses->swcr_algorithms); i++) {
770 		swd = &ses->swcr_algorithms[i];
771 
772 		switch (cri->cri_alg) {
773 		case CRYPTO_DES_CBC:
774 			txf = &enc_xform_des;
775 			goto enccommon;
776 		case CRYPTO_3DES_CBC:
777 			txf = &enc_xform_3des;
778 			goto enccommon;
779 		case CRYPTO_BLF_CBC:
780 			txf = &enc_xform_blf;
781 			goto enccommon;
782 		case CRYPTO_CAST_CBC:
783 			txf = &enc_xform_cast5;
784 			goto enccommon;
785 		case CRYPTO_SKIPJACK_CBC:
786 			txf = &enc_xform_skipjack;
787 			goto enccommon;
788 		case CRYPTO_RIJNDAEL128_CBC:
789 			txf = &enc_xform_rijndael128;
790 			goto enccommon;
791 		case CRYPTO_AES_XTS:
792 			txf = &enc_xform_aes_xts;
793 			goto enccommon;
794 		case CRYPTO_AES_ICM:
795 			txf = &enc_xform_aes_icm;
796 			goto enccommon;
797 		case CRYPTO_AES_NIST_GCM_16:
798 			txf = &enc_xform_aes_nist_gcm;
799 			goto enccommon;
800 		case CRYPTO_AES_NIST_GMAC:
801 			txf = &enc_xform_aes_nist_gmac;
802 			swd->sw_exf = txf;
803 			break;
804 		case CRYPTO_CAMELLIA_CBC:
805 			txf = &enc_xform_camellia;
806 			goto enccommon;
807 		case CRYPTO_NULL_CBC:
808 			txf = &enc_xform_null;
809 			goto enccommon;
810 		case CRYPTO_CHACHA20:
811 			txf = &enc_xform_chacha20;
812 			goto enccommon;
813 		enccommon:
814 			if (cri->cri_key != NULL) {
815 				error = txf->setkey(&swd->sw_kschedule,
816 				    cri->cri_key, cri->cri_klen / 8);
817 				if (error) {
818 					swcr_freesession(dev, cses);
819 					return error;
820 				}
821 			}
822 			swd->sw_exf = txf;
823 			break;
824 
825 		case CRYPTO_MD5_HMAC:
826 			axf = &auth_hash_hmac_md5;
827 			goto authcommon;
828 		case CRYPTO_SHA1_HMAC:
829 			axf = &auth_hash_hmac_sha1;
830 			goto authcommon;
831 		case CRYPTO_SHA2_224_HMAC:
832 			axf = &auth_hash_hmac_sha2_224;
833 			goto authcommon;
834 		case CRYPTO_SHA2_256_HMAC:
835 			axf = &auth_hash_hmac_sha2_256;
836 			goto authcommon;
837 		case CRYPTO_SHA2_384_HMAC:
838 			axf = &auth_hash_hmac_sha2_384;
839 			goto authcommon;
840 		case CRYPTO_SHA2_512_HMAC:
841 			axf = &auth_hash_hmac_sha2_512;
842 			goto authcommon;
843 		case CRYPTO_NULL_HMAC:
844 			axf = &auth_hash_null;
845 			goto authcommon;
846 		case CRYPTO_RIPEMD160_HMAC:
847 			axf = &auth_hash_hmac_ripemd_160;
848 		authcommon:
849 			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
850 			    M_NOWAIT);
851 			if (swd->sw_ictx == NULL) {
852 				swcr_freesession(dev, cses);
853 				return ENOBUFS;
854 			}
855 
856 			swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
857 			    M_NOWAIT);
858 			if (swd->sw_octx == NULL) {
859 				swcr_freesession(dev, cses);
860 				return ENOBUFS;
861 			}
862 
863 			if (cri->cri_key != NULL) {
864 				error = swcr_authprepare(axf, swd,
865 				    cri->cri_key, cri->cri_klen);
866 				if (error != 0) {
867 					swcr_freesession(dev, cses);
868 					return error;
869 				}
870 			}
871 
872 			swd->sw_mlen = cri->cri_mlen;
873 			swd->sw_axf = axf;
874 			break;
875 
876 		case CRYPTO_MD5_KPDK:
877 			axf = &auth_hash_key_md5;
878 			goto auth2common;
879 
880 		case CRYPTO_SHA1_KPDK:
881 			axf = &auth_hash_key_sha1;
882 		auth2common:
883 			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
884 			    M_NOWAIT);
885 			if (swd->sw_ictx == NULL) {
886 				swcr_freesession(dev, cses);
887 				return ENOBUFS;
888 			}
889 
890 			swd->sw_octx = malloc(cri->cri_klen / 8,
891 			    M_CRYPTO_DATA, M_NOWAIT);
892 			if (swd->sw_octx == NULL) {
893 				swcr_freesession(dev, cses);
894 				return ENOBUFS;
895 			}
896 
897 			/* Store the key so we can "append" it to the payload */
898 			if (cri->cri_key != NULL) {
899 				error = swcr_authprepare(axf, swd,
900 				    cri->cri_key, cri->cri_klen);
901 				if (error != 0) {
902 					swcr_freesession(dev, cses);
903 					return error;
904 				}
905 			}
906 
907 			swd->sw_mlen = cri->cri_mlen;
908 			swd->sw_axf = axf;
909 			break;
910 #ifdef notdef
911 		case CRYPTO_MD5:
912 			axf = &auth_hash_md5;
913 			goto auth3common;
914 #endif
915 
916 		case CRYPTO_SHA1:
917 			axf = &auth_hash_sha1;
918 			goto auth3common;
919 		case CRYPTO_SHA2_224:
920 			axf = &auth_hash_sha2_224;
921 			goto auth3common;
922 		case CRYPTO_SHA2_256:
923 			axf = &auth_hash_sha2_256;
924 			goto auth3common;
925 		case CRYPTO_SHA2_384:
926 			axf = &auth_hash_sha2_384;
927 			goto auth3common;
928 		case CRYPTO_SHA2_512:
929 			axf = &auth_hash_sha2_512;
930 
931 		auth3common:
932 			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
933 			    M_NOWAIT);
934 			if (swd->sw_ictx == NULL) {
935 				swcr_freesession(dev, cses);
936 				return ENOBUFS;
937 			}
938 
939 			axf->Init(swd->sw_ictx);
940 			swd->sw_mlen = cri->cri_mlen;
941 			swd->sw_axf = axf;
942 			break;
943 
944 		case CRYPTO_AES_128_NIST_GMAC:
945 			axf = &auth_hash_nist_gmac_aes_128;
946 			goto auth4common;
947 
948 		case CRYPTO_AES_192_NIST_GMAC:
949 			axf = &auth_hash_nist_gmac_aes_192;
950 			goto auth4common;
951 
952 		case CRYPTO_AES_256_NIST_GMAC:
953 			axf = &auth_hash_nist_gmac_aes_256;
954 		auth4common:
955 			len = cri->cri_klen / 8;
956 			if (len != 16 && len != 24 && len != 32) {
957 				swcr_freesession(dev, cses);
958 				return EINVAL;
959 			}
960 
961 			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
962 			    M_NOWAIT);
963 			if (swd->sw_ictx == NULL) {
964 				swcr_freesession(dev, cses);
965 				return ENOBUFS;
966 			}
967 			axf->Init(swd->sw_ictx);
968 			axf->Setkey(swd->sw_ictx, cri->cri_key, len);
969 			swd->sw_axf = axf;
970 			break;
971 
972 		case CRYPTO_BLAKE2B:
973 			axf = &auth_hash_blake2b;
974 			goto auth5common;
975 		case CRYPTO_BLAKE2S:
976 			axf = &auth_hash_blake2s;
977 			goto auth5common;
978 		case CRYPTO_POLY1305:
979 			axf = &auth_hash_poly1305;
980 		auth5common:
981 			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
982 			    M_NOWAIT);
983 			if (swd->sw_ictx == NULL) {
984 				swcr_freesession(dev, cses);
985 				return ENOBUFS;
986 			}
987 			axf->Setkey(swd->sw_ictx, cri->cri_key,
988 			    cri->cri_klen / 8);
989 			axf->Init(swd->sw_ictx);
990 			swd->sw_axf = axf;
991 			break;
992 
993 		case CRYPTO_DEFLATE_COMP:
994 			cxf = &comp_algo_deflate;
995 			swd->sw_cxf = cxf;
996 			break;
997 		default:
998 			swcr_freesession(dev, cses);
999 			return EINVAL;
1000 		}
1001 
1002 		swd->sw_alg = cri->cri_alg;
1003 		cri = cri->cri_next;
1004 		ses->swcr_nalgs++;
1005 	}
1006 
1007 	if (cri != NULL) {
1008 		CRYPTDEB("Bogus session request for three or more algorithms");
1009 		return EINVAL;
1010 	}
1011 	return 0;
1012 }
1013 
1014 static void
1015 swcr_freesession(device_t dev, crypto_session_t cses)
1016 {
1017 	struct swcr_session *ses;
1018 	struct swcr_data *swd;
1019 	struct enc_xform *txf;
1020 	struct auth_hash *axf;
1021 	size_t i;
1022 
1023 	ses = crypto_get_driver_session(cses);
1024 
1025 	for (i = 0; i < nitems(ses->swcr_algorithms); i++) {
1026 		swd = &ses->swcr_algorithms[i];
1027 
1028 		switch (swd->sw_alg) {
1029 		case CRYPTO_DES_CBC:
1030 		case CRYPTO_3DES_CBC:
1031 		case CRYPTO_BLF_CBC:
1032 		case CRYPTO_CAST_CBC:
1033 		case CRYPTO_SKIPJACK_CBC:
1034 		case CRYPTO_RIJNDAEL128_CBC:
1035 		case CRYPTO_AES_XTS:
1036 		case CRYPTO_AES_ICM:
1037 		case CRYPTO_AES_NIST_GCM_16:
1038 		case CRYPTO_AES_NIST_GMAC:
1039 		case CRYPTO_CAMELLIA_CBC:
1040 		case CRYPTO_NULL_CBC:
1041 		case CRYPTO_CHACHA20:
1042 			txf = swd->sw_exf;
1043 
1044 			if (swd->sw_kschedule)
1045 				txf->zerokey(&(swd->sw_kschedule));
1046 			break;
1047 
1048 		case CRYPTO_MD5_HMAC:
1049 		case CRYPTO_SHA1_HMAC:
1050 		case CRYPTO_SHA2_224_HMAC:
1051 		case CRYPTO_SHA2_256_HMAC:
1052 		case CRYPTO_SHA2_384_HMAC:
1053 		case CRYPTO_SHA2_512_HMAC:
1054 		case CRYPTO_RIPEMD160_HMAC:
1055 		case CRYPTO_NULL_HMAC:
1056 			axf = swd->sw_axf;
1057 
1058 			if (swd->sw_ictx) {
1059 				bzero(swd->sw_ictx, axf->ctxsize);
1060 				free(swd->sw_ictx, M_CRYPTO_DATA);
1061 			}
1062 			if (swd->sw_octx) {
1063 				bzero(swd->sw_octx, axf->ctxsize);
1064 				free(swd->sw_octx, M_CRYPTO_DATA);
1065 			}
1066 			break;
1067 
1068 		case CRYPTO_MD5_KPDK:
1069 		case CRYPTO_SHA1_KPDK:
1070 			axf = swd->sw_axf;
1071 
1072 			if (swd->sw_ictx) {
1073 				bzero(swd->sw_ictx, axf->ctxsize);
1074 				free(swd->sw_ictx, M_CRYPTO_DATA);
1075 			}
1076 			if (swd->sw_octx) {
1077 				bzero(swd->sw_octx, swd->sw_klen);
1078 				free(swd->sw_octx, M_CRYPTO_DATA);
1079 			}
1080 			break;
1081 
1082 		case CRYPTO_BLAKE2B:
1083 		case CRYPTO_BLAKE2S:
1084 		case CRYPTO_MD5:
1085 		case CRYPTO_POLY1305:
1086 		case CRYPTO_SHA1:
1087 		case CRYPTO_SHA2_224:
1088 		case CRYPTO_SHA2_256:
1089 		case CRYPTO_SHA2_384:
1090 		case CRYPTO_SHA2_512:
1091 			axf = swd->sw_axf;
1092 
1093 			if (swd->sw_ictx) {
1094 				explicit_bzero(swd->sw_ictx, axf->ctxsize);
1095 				free(swd->sw_ictx, M_CRYPTO_DATA);
1096 			}
1097 			break;
1098 
1099 		case CRYPTO_DEFLATE_COMP:
1100 			/* Nothing to do */
1101 			break;
1102 		}
1103 	}
1104 }
1105 
1106 /*
1107  * Process a software request.
1108  */
1109 static int
1110 swcr_process(device_t dev, struct cryptop *crp, int hint)
1111 {
1112 	struct swcr_session *ses;
1113 	struct cryptodesc *crd;
1114 	struct swcr_data *sw;
1115 	size_t i;
1116 
1117 	/* Sanity check */
1118 	if (crp == NULL)
1119 		return EINVAL;
1120 
1121 	if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
1122 		crp->crp_etype = EINVAL;
1123 		goto done;
1124 	}
1125 
1126 	ses = crypto_get_driver_session(crp->crp_session);
1127 
1128 	/* Go through crypto descriptors, processing as we go */
1129 	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1130 		/*
1131 		 * Find the crypto context.
1132 		 *
1133 		 * XXX Note that the logic here prevents us from having
1134 		 * XXX the same algorithm multiple times in a session
1135 		 * XXX (or rather, we can but it won't give us the right
1136 		 * XXX results). To do that, we'd need some way of differentiating
1137 		 * XXX between the various instances of an algorithm (so we can
1138 		 * XXX locate the correct crypto context).
1139 		 */
1140 		for (i = 0; i < nitems(ses->swcr_algorithms) &&
1141 		    ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)
1142 			;
1143 
1144 		/* No such context ? */
1145 		if (i == nitems(ses->swcr_algorithms)) {
1146 			crp->crp_etype = EINVAL;
1147 			goto done;
1148 		}
1149 		sw = &ses->swcr_algorithms[i];
1150 		switch (sw->sw_alg) {
1151 		case CRYPTO_DES_CBC:
1152 		case CRYPTO_3DES_CBC:
1153 		case CRYPTO_BLF_CBC:
1154 		case CRYPTO_CAST_CBC:
1155 		case CRYPTO_SKIPJACK_CBC:
1156 		case CRYPTO_RIJNDAEL128_CBC:
1157 		case CRYPTO_AES_XTS:
1158 		case CRYPTO_AES_ICM:
1159 		case CRYPTO_CAMELLIA_CBC:
1160 		case CRYPTO_CHACHA20:
1161 			if ((crp->crp_etype = swcr_encdec(crd, sw,
1162 			    crp->crp_buf, crp->crp_flags)) != 0)
1163 				goto done;
1164 			break;
1165 		case CRYPTO_NULL_CBC:
1166 			crp->crp_etype = 0;
1167 			break;
1168 		case CRYPTO_MD5_HMAC:
1169 		case CRYPTO_SHA1_HMAC:
1170 		case CRYPTO_SHA2_224_HMAC:
1171 		case CRYPTO_SHA2_256_HMAC:
1172 		case CRYPTO_SHA2_384_HMAC:
1173 		case CRYPTO_SHA2_512_HMAC:
1174 		case CRYPTO_RIPEMD160_HMAC:
1175 		case CRYPTO_NULL_HMAC:
1176 		case CRYPTO_MD5_KPDK:
1177 		case CRYPTO_SHA1_KPDK:
1178 		case CRYPTO_MD5:
1179 		case CRYPTO_SHA1:
1180 		case CRYPTO_SHA2_224:
1181 		case CRYPTO_SHA2_256:
1182 		case CRYPTO_SHA2_384:
1183 		case CRYPTO_SHA2_512:
1184 		case CRYPTO_BLAKE2B:
1185 		case CRYPTO_BLAKE2S:
1186 		case CRYPTO_POLY1305:
1187 			if ((crp->crp_etype = swcr_authcompute(crd, sw,
1188 			    crp->crp_buf, crp->crp_flags)) != 0)
1189 				goto done;
1190 			break;
1191 
1192 		case CRYPTO_AES_NIST_GCM_16:
1193 		case CRYPTO_AES_NIST_GMAC:
1194 		case CRYPTO_AES_128_NIST_GMAC:
1195 		case CRYPTO_AES_192_NIST_GMAC:
1196 		case CRYPTO_AES_256_NIST_GMAC:
1197 			crp->crp_etype = swcr_authenc(crp);
1198 			goto done;
1199 
1200 		case CRYPTO_DEFLATE_COMP:
1201 			if ((crp->crp_etype = swcr_compdec(crd, sw,
1202 			    crp->crp_buf, crp->crp_flags)) != 0)
1203 				goto done;
1204 			else
1205 				crp->crp_olen = (int)sw->sw_size;
1206 			break;
1207 
1208 		default:
1209 			/* Unknown/unsupported algorithm */
1210 			crp->crp_etype = EINVAL;
1211 			goto done;
1212 		}
1213 	}
1214 
1215 done:
1216 	crypto_done(crp);
1217 	return 0;
1218 }
1219 
1220 static void
1221 swcr_identify(driver_t *drv, device_t parent)
1222 {
1223 	/* NB: order 10 is so we get attached after h/w devices */
1224 	if (device_find_child(parent, "cryptosoft", -1) == NULL &&
1225 	    BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0)
1226 		panic("cryptosoft: could not attach");
1227 }
1228 
1229 static int
1230 swcr_probe(device_t dev)
1231 {
1232 	device_set_desc(dev, "software crypto");
1233 	return (BUS_PROBE_NOWILDCARD);
1234 }
1235 
1236 static int
1237 swcr_attach(device_t dev)
1238 {
1239 	memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
1240 	memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
1241 
1242 	swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session),
1243 			CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
1244 	if (swcr_id < 0) {
1245 		device_printf(dev, "cannot initialize!");
1246 		return ENOMEM;
1247 	}
1248 #define	REGISTER(alg) \
1249 	crypto_register(swcr_id, alg, 0,0)
1250 	REGISTER(CRYPTO_DES_CBC);
1251 	REGISTER(CRYPTO_3DES_CBC);
1252 	REGISTER(CRYPTO_BLF_CBC);
1253 	REGISTER(CRYPTO_CAST_CBC);
1254 	REGISTER(CRYPTO_SKIPJACK_CBC);
1255 	REGISTER(CRYPTO_NULL_CBC);
1256 	REGISTER(CRYPTO_MD5_HMAC);
1257 	REGISTER(CRYPTO_SHA1_HMAC);
1258 	REGISTER(CRYPTO_SHA2_224_HMAC);
1259 	REGISTER(CRYPTO_SHA2_256_HMAC);
1260 	REGISTER(CRYPTO_SHA2_384_HMAC);
1261 	REGISTER(CRYPTO_SHA2_512_HMAC);
1262 	REGISTER(CRYPTO_RIPEMD160_HMAC);
1263 	REGISTER(CRYPTO_NULL_HMAC);
1264 	REGISTER(CRYPTO_MD5_KPDK);
1265 	REGISTER(CRYPTO_SHA1_KPDK);
1266 	REGISTER(CRYPTO_MD5);
1267 	REGISTER(CRYPTO_SHA1);
1268 	REGISTER(CRYPTO_SHA2_224);
1269 	REGISTER(CRYPTO_SHA2_256);
1270 	REGISTER(CRYPTO_SHA2_384);
1271 	REGISTER(CRYPTO_SHA2_512);
1272 	REGISTER(CRYPTO_RIJNDAEL128_CBC);
1273 	REGISTER(CRYPTO_AES_XTS);
1274 	REGISTER(CRYPTO_AES_ICM);
1275 	REGISTER(CRYPTO_AES_NIST_GCM_16);
1276 	REGISTER(CRYPTO_AES_NIST_GMAC);
1277 	REGISTER(CRYPTO_AES_128_NIST_GMAC);
1278 	REGISTER(CRYPTO_AES_192_NIST_GMAC);
1279 	REGISTER(CRYPTO_AES_256_NIST_GMAC);
1280  	REGISTER(CRYPTO_CAMELLIA_CBC);
1281 	REGISTER(CRYPTO_DEFLATE_COMP);
1282 	REGISTER(CRYPTO_BLAKE2B);
1283 	REGISTER(CRYPTO_BLAKE2S);
1284 	REGISTER(CRYPTO_CHACHA20);
1285 	REGISTER(CRYPTO_POLY1305);
1286 #undef REGISTER
1287 
1288 	return 0;
1289 }
1290 
1291 static int
1292 swcr_detach(device_t dev)
1293 {
1294 	crypto_unregister_all(swcr_id);
1295 	return 0;
1296 }
1297 
1298 static device_method_t swcr_methods[] = {
1299 	DEVMETHOD(device_identify,	swcr_identify),
1300 	DEVMETHOD(device_probe,		swcr_probe),
1301 	DEVMETHOD(device_attach,	swcr_attach),
1302 	DEVMETHOD(device_detach,	swcr_detach),
1303 
1304 	DEVMETHOD(cryptodev_newsession,	swcr_newsession),
1305 	DEVMETHOD(cryptodev_freesession,swcr_freesession),
1306 	DEVMETHOD(cryptodev_process,	swcr_process),
1307 
1308 	{0, 0},
1309 };
1310 
1311 static driver_t swcr_driver = {
1312 	"cryptosoft",
1313 	swcr_methods,
1314 	0,		/* NB: no softc */
1315 };
1316 static devclass_t swcr_devclass;
1317 
1318 /*
1319  * NB: We explicitly reference the crypto module so we
1320  * get the necessary ordering when built as a loadable
1321  * module.  This is required because we bundle the crypto
1322  * module code together with the cryptosoft driver (otherwise
1323  * normal module dependencies would handle things).
1324  */
1325 extern int crypto_modevent(struct module *, int, void *);
1326 /* XXX where to attach */
1327 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0);
1328 MODULE_VERSION(cryptosoft, 1);
1329 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);
1330