xref: /freebsd/sys/opencrypto/crypto.c (revision 2e4311906d8c8dc7a7c726345268253bca6d4acc)
1 /*-
2  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
3  * Copyright (c) 2021 The FreeBSD Foundation
4  *
5  * Portions of this software were developed by Ararat River
6  * Consulting, LLC under sponsorship of the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 /*
33  * Cryptographic Subsystem.
34  *
35  * This code is derived from the Openbsd Cryptographic Framework (OCF)
36  * that has the copyright shown below.  Very little of the original
37  * code remains.
38  */
39 
40 /*-
41  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
42  *
43  * This code was written by Angelos D. Keromytis in Athens, Greece, in
44  * February 2000. Network Security Technologies Inc. (NSTI) kindly
45  * supported the development of this code.
46  *
47  * Copyright (c) 2000, 2001 Angelos D. Keromytis
48  *
49  * Permission to use, copy, and modify this software with or without fee
50  * is hereby granted, provided that this entire notice is included in
51  * all source code copies of any software which is or includes a copy or
52  * modification of this software.
53  *
54  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
55  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
56  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
57  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
58  * PURPOSE.
59  */
60 
61 #include "opt_compat.h"
62 #include "opt_ddb.h"
63 
64 #include <sys/param.h>
65 #include <sys/systm.h>
66 #include <sys/counter.h>
67 #include <sys/kernel.h>
68 #include <sys/kthread.h>
69 #include <sys/linker.h>
70 #include <sys/lock.h>
71 #include <sys/module.h>
72 #include <sys/mutex.h>
73 #include <sys/malloc.h>
74 #include <sys/mbuf.h>
75 #include <sys/proc.h>
76 #include <sys/refcount.h>
77 #include <sys/sdt.h>
78 #include <sys/smp.h>
79 #include <sys/sysctl.h>
80 #include <sys/taskqueue.h>
81 #include <sys/uio.h>
82 
83 #include <ddb/ddb.h>
84 
85 #include <machine/vmparam.h>
86 #include <vm/uma.h>
87 
88 #include <crypto/intake.h>
89 #include <opencrypto/cryptodev.h>
90 #include <opencrypto/xform_auth.h>
91 #include <opencrypto/xform_enc.h>
92 
93 #include <sys/kobj.h>
94 #include <sys/bus.h>
95 #include "cryptodev_if.h"
96 
97 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
98 #include <machine/pcb.h>
99 #endif
100 
101 SDT_PROVIDER_DEFINE(opencrypto);
102 
103 /*
104  * Crypto drivers register themselves by allocating a slot in the
105  * crypto_drivers table with crypto_get_driverid().
106  */
107 static	struct mtx crypto_drivers_mtx;		/* lock on driver table */
108 #define	CRYPTO_DRIVER_LOCK()	mtx_lock(&crypto_drivers_mtx)
109 #define	CRYPTO_DRIVER_UNLOCK()	mtx_unlock(&crypto_drivers_mtx)
110 #define	CRYPTO_DRIVER_ASSERT()	mtx_assert(&crypto_drivers_mtx, MA_OWNED)
111 
112 /*
113  * Crypto device/driver capabilities structure.
114  *
115  * Synchronization:
116  * (d) - protected by CRYPTO_DRIVER_LOCK()
117  * (q) - protected by CRYPTO_Q_LOCK()
118  * Not tagged fields are read-only.
119  */
120 struct cryptocap {
121 	device_t	cc_dev;
122 	uint32_t	cc_hid;
123 	uint32_t	cc_sessions;		/* (d) # of sessions */
124 
125 	int		cc_flags;		/* (d) flags */
126 #define CRYPTOCAP_F_CLEANUP	0x80000000	/* needs resource cleanup */
127 	int		cc_qblocked;		/* (q) symmetric q blocked */
128 	size_t		cc_session_size;
129 	volatile int	cc_refs;
130 };
131 
132 static	struct cryptocap **crypto_drivers = NULL;
133 static	int crypto_drivers_size = 0;
134 
135 struct crypto_session {
136 	struct cryptocap *cap;
137 	struct crypto_session_params csp;
138 	uint64_t id;
139 	/* Driver softc follows. */
140 };
141 
142 static	int crp_sleep = 0;
143 static	TAILQ_HEAD(cryptop_q ,cryptop) crp_q;		/* request queues */
144 static	struct mtx crypto_q_mtx;
145 #define	CRYPTO_Q_LOCK()		mtx_lock(&crypto_q_mtx)
146 #define	CRYPTO_Q_UNLOCK()	mtx_unlock(&crypto_q_mtx)
147 
148 SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
149     "In-kernel cryptography");
150 
151 /*
152  * Taskqueue used to dispatch the crypto requests
153  * that have the CRYPTO_F_ASYNC flag
154  */
155 static struct taskqueue *crypto_tq;
156 
157 /*
158  * Crypto seq numbers are operated on with modular arithmetic
159  */
160 #define	CRYPTO_SEQ_GT(a,b)	((int)((a)-(b)) > 0)
161 
162 struct crypto_ret_worker {
163 	struct mtx crypto_ret_mtx;
164 
165 	TAILQ_HEAD(,cryptop) crp_ordered_ret_q;	/* ordered callback queue for symetric jobs */
166 	TAILQ_HEAD(,cryptop) crp_ret_q;		/* callback queue for symetric jobs */
167 
168 	uint32_t reorder_ops;		/* total ordered sym jobs received */
169 	uint32_t reorder_cur_seq;	/* current sym job dispatched */
170 
171 	struct thread *td;
172 };
173 static struct crypto_ret_worker *crypto_ret_workers = NULL;
174 
175 #define CRYPTO_RETW(i)		(&crypto_ret_workers[i])
176 #define CRYPTO_RETW_ID(w)	((w) - crypto_ret_workers)
177 #define FOREACH_CRYPTO_RETW(w) \
178 	for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
179 
180 #define	CRYPTO_RETW_LOCK(w)	mtx_lock(&w->crypto_ret_mtx)
181 #define	CRYPTO_RETW_UNLOCK(w)	mtx_unlock(&w->crypto_ret_mtx)
182 
183 static int crypto_workers_num = 0;
184 SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
185 	   &crypto_workers_num, 0,
186 	   "Number of crypto workers used to dispatch crypto jobs");
187 #ifdef COMPAT_FREEBSD12
188 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
189 	   &crypto_workers_num, 0,
190 	   "Number of crypto workers used to dispatch crypto jobs");
191 #endif
192 
193 static	uma_zone_t cryptop_zone;
194 
195 int	crypto_devallowsoft = 0;
196 SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RWTUN,
197 	   &crypto_devallowsoft, 0,
198 	   "Enable use of software crypto by /dev/crypto");
199 #ifdef COMPAT_FREEBSD12
200 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RWTUN,
201 	   &crypto_devallowsoft, 0,
202 	   "Enable/disable use of software crypto by /dev/crypto");
203 #endif
204 
205 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
206 
207 static	void crypto_dispatch_thread(void *arg);
208 static	struct thread *cryptotd;
209 static	void crypto_ret_thread(void *arg);
210 static	void crypto_destroy(void);
211 static	int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
212 static	void crypto_task_invoke(void *ctx, int pending);
213 static void crypto_batch_enqueue(struct cryptop *crp);
214 
215 static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)];
216 SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW,
217     cryptostats, nitems(cryptostats),
218     "Crypto system statistics");
219 
220 #define	CRYPTOSTAT_INC(stat) do {					\
221 	counter_u64_add(						\
222 	    cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\
223 	    1);								\
224 } while (0)
225 
226 static void
227 cryptostats_init(void *arg __unused)
228 {
229 	COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK);
230 }
231 SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL);
232 
233 static void
234 cryptostats_fini(void *arg __unused)
235 {
236 	COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats));
237 }
238 SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini,
239     NULL);
240 
241 /* Try to avoid directly exposing the key buffer as a symbol */
242 static struct keybuf *keybuf;
243 
244 static struct keybuf empty_keybuf = {
245         .kb_nents = 0
246 };
247 
248 /* Obtain the key buffer from boot metadata */
249 static void
250 keybuf_init(void)
251 {
252 	caddr_t kmdp;
253 
254 	kmdp = preload_search_by_type("elf kernel");
255 
256 	if (kmdp == NULL)
257 		kmdp = preload_search_by_type("elf64 kernel");
258 
259 	keybuf = (struct keybuf *)preload_search_info(kmdp,
260 	    MODINFO_METADATA | MODINFOMD_KEYBUF);
261 
262         if (keybuf == NULL)
263                 keybuf = &empty_keybuf;
264 }
265 
266 /* It'd be nice if we could store these in some kind of secure memory... */
267 struct keybuf *
268 get_keybuf(void)
269 {
270 
271         return (keybuf);
272 }
273 
274 static struct cryptocap *
275 cap_ref(struct cryptocap *cap)
276 {
277 
278 	refcount_acquire(&cap->cc_refs);
279 	return (cap);
280 }
281 
282 static void
283 cap_rele(struct cryptocap *cap)
284 {
285 
286 	if (refcount_release(&cap->cc_refs) == 0)
287 		return;
288 
289 	KASSERT(cap->cc_sessions == 0,
290 	    ("freeing crypto driver with active sessions"));
291 
292 	free(cap, M_CRYPTO_DATA);
293 }
294 
295 static int
296 crypto_init(void)
297 {
298 	struct crypto_ret_worker *ret_worker;
299 	struct proc *p;
300 	int error;
301 
302 	mtx_init(&crypto_drivers_mtx, "crypto driver table", NULL, MTX_DEF);
303 
304 	TAILQ_INIT(&crp_q);
305 	mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF);
306 
307 	cryptop_zone = uma_zcreate("cryptop",
308 	    sizeof(struct cryptop), NULL, NULL, NULL, NULL,
309 	    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
310 
311 	crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
312 	crypto_drivers = malloc(crypto_drivers_size *
313 	    sizeof(struct cryptocap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
314 
315 	if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
316 		crypto_workers_num = mp_ncpus;
317 
318 	crypto_tq = taskqueue_create("crypto", M_WAITOK | M_ZERO,
319 	    taskqueue_thread_enqueue, &crypto_tq);
320 
321 	taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
322 	    "crypto");
323 
324 	p = NULL;
325 	error = kproc_kthread_add(crypto_dispatch_thread, NULL, &p, &cryptotd,
326 	    0, 0, "crypto", "crypto");
327 	if (error) {
328 		printf("crypto_init: cannot start crypto thread; error %d",
329 			error);
330 		goto bad;
331 	}
332 
333 	crypto_ret_workers = mallocarray(crypto_workers_num,
334 	    sizeof(struct crypto_ret_worker), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
335 
336 	FOREACH_CRYPTO_RETW(ret_worker) {
337 		TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
338 		TAILQ_INIT(&ret_worker->crp_ret_q);
339 
340 		ret_worker->reorder_ops = 0;
341 		ret_worker->reorder_cur_seq = 0;
342 
343 		mtx_init(&ret_worker->crypto_ret_mtx, "crypto return queues",
344 		    NULL, MTX_DEF);
345 
346 		error = kthread_add(crypto_ret_thread, ret_worker, p,
347 		    &ret_worker->td, 0, 0, "crypto returns %td",
348 		    CRYPTO_RETW_ID(ret_worker));
349 		if (error) {
350 			printf("crypto_init: cannot start cryptoret thread; error %d",
351 				error);
352 			goto bad;
353 		}
354 	}
355 
356 	keybuf_init();
357 
358 	return 0;
359 bad:
360 	crypto_destroy();
361 	return error;
362 }
363 
364 /*
365  * Signal a crypto thread to terminate.  We use the driver
366  * table lock to synchronize the sleep/wakeups so that we
367  * are sure the threads have terminated before we release
368  * the data structures they use.  See crypto_finis below
369  * for the other half of this song-and-dance.
370  */
371 static void
372 crypto_terminate(struct thread **tdp, void *q)
373 {
374 	struct thread *td;
375 
376 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
377 	td = *tdp;
378 	*tdp = NULL;
379 	if (td != NULL) {
380 		wakeup_one(q);
381 		mtx_sleep(td, &crypto_drivers_mtx, PWAIT, "crypto_destroy", 0);
382 	}
383 }
384 
385 static void
386 hmac_init_pad(const struct auth_hash *axf, const char *key, int klen,
387     void *auth_ctx, uint8_t padval)
388 {
389 	uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
390 	u_int i;
391 
392 	KASSERT(axf->blocksize <= sizeof(hmac_key),
393 	    ("Invalid HMAC block size %d", axf->blocksize));
394 
395 	/*
396 	 * If the key is larger than the block size, use the digest of
397 	 * the key as the key instead.
398 	 */
399 	memset(hmac_key, 0, sizeof(hmac_key));
400 	if (klen > axf->blocksize) {
401 		axf->Init(auth_ctx);
402 		axf->Update(auth_ctx, key, klen);
403 		axf->Final(hmac_key, auth_ctx);
404 		klen = axf->hashsize;
405 	} else
406 		memcpy(hmac_key, key, klen);
407 
408 	for (i = 0; i < axf->blocksize; i++)
409 		hmac_key[i] ^= padval;
410 
411 	axf->Init(auth_ctx);
412 	axf->Update(auth_ctx, hmac_key, axf->blocksize);
413 	explicit_bzero(hmac_key, sizeof(hmac_key));
414 }
415 
416 void
417 hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen,
418     void *auth_ctx)
419 {
420 
421 	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
422 }
423 
424 void
425 hmac_init_opad(const struct auth_hash *axf, const char *key, int klen,
426     void *auth_ctx)
427 {
428 
429 	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
430 }
431 
432 static void
433 crypto_destroy(void)
434 {
435 	struct crypto_ret_worker *ret_worker;
436 	int i;
437 
438 	/*
439 	 * Terminate any crypto threads.
440 	 */
441 	if (crypto_tq != NULL)
442 		taskqueue_drain_all(crypto_tq);
443 	CRYPTO_DRIVER_LOCK();
444 	crypto_terminate(&cryptotd, &crp_q);
445 	FOREACH_CRYPTO_RETW(ret_worker)
446 		crypto_terminate(&ret_worker->td, &ret_worker->crp_ret_q);
447 	CRYPTO_DRIVER_UNLOCK();
448 
449 	/* XXX flush queues??? */
450 
451 	/*
452 	 * Reclaim dynamically allocated resources.
453 	 */
454 	for (i = 0; i < crypto_drivers_size; i++) {
455 		if (crypto_drivers[i] != NULL)
456 			cap_rele(crypto_drivers[i]);
457 	}
458 	free(crypto_drivers, M_CRYPTO_DATA);
459 
460 	if (cryptop_zone != NULL)
461 		uma_zdestroy(cryptop_zone);
462 	mtx_destroy(&crypto_q_mtx);
463 	FOREACH_CRYPTO_RETW(ret_worker)
464 		mtx_destroy(&ret_worker->crypto_ret_mtx);
465 	free(crypto_ret_workers, M_CRYPTO_DATA);
466 	if (crypto_tq != NULL)
467 		taskqueue_free(crypto_tq);
468 	mtx_destroy(&crypto_drivers_mtx);
469 }
470 
471 uint32_t
472 crypto_ses2hid(crypto_session_t crypto_session)
473 {
474 	return (crypto_session->cap->cc_hid);
475 }
476 
477 uint32_t
478 crypto_ses2caps(crypto_session_t crypto_session)
479 {
480 	return (crypto_session->cap->cc_flags & 0xff000000);
481 }
482 
483 void *
484 crypto_get_driver_session(crypto_session_t crypto_session)
485 {
486 	return (crypto_session + 1);
487 }
488 
489 const struct crypto_session_params *
490 crypto_get_params(crypto_session_t crypto_session)
491 {
492 	return (&crypto_session->csp);
493 }
494 
495 const struct auth_hash *
496 crypto_auth_hash(const struct crypto_session_params *csp)
497 {
498 
499 	switch (csp->csp_auth_alg) {
500 	case CRYPTO_SHA1_HMAC:
501 		return (&auth_hash_hmac_sha1);
502 	case CRYPTO_SHA2_224_HMAC:
503 		return (&auth_hash_hmac_sha2_224);
504 	case CRYPTO_SHA2_256_HMAC:
505 		return (&auth_hash_hmac_sha2_256);
506 	case CRYPTO_SHA2_384_HMAC:
507 		return (&auth_hash_hmac_sha2_384);
508 	case CRYPTO_SHA2_512_HMAC:
509 		return (&auth_hash_hmac_sha2_512);
510 	case CRYPTO_NULL_HMAC:
511 		return (&auth_hash_null);
512 	case CRYPTO_RIPEMD160_HMAC:
513 		return (&auth_hash_hmac_ripemd_160);
514 	case CRYPTO_RIPEMD160:
515 		return (&auth_hash_ripemd_160);
516 	case CRYPTO_SHA1:
517 		return (&auth_hash_sha1);
518 	case CRYPTO_SHA2_224:
519 		return (&auth_hash_sha2_224);
520 	case CRYPTO_SHA2_256:
521 		return (&auth_hash_sha2_256);
522 	case CRYPTO_SHA2_384:
523 		return (&auth_hash_sha2_384);
524 	case CRYPTO_SHA2_512:
525 		return (&auth_hash_sha2_512);
526 	case CRYPTO_AES_NIST_GMAC:
527 		switch (csp->csp_auth_klen) {
528 		case 128 / 8:
529 			return (&auth_hash_nist_gmac_aes_128);
530 		case 192 / 8:
531 			return (&auth_hash_nist_gmac_aes_192);
532 		case 256 / 8:
533 			return (&auth_hash_nist_gmac_aes_256);
534 		default:
535 			return (NULL);
536 		}
537 	case CRYPTO_BLAKE2B:
538 		return (&auth_hash_blake2b);
539 	case CRYPTO_BLAKE2S:
540 		return (&auth_hash_blake2s);
541 	case CRYPTO_POLY1305:
542 		return (&auth_hash_poly1305);
543 	case CRYPTO_AES_CCM_CBC_MAC:
544 		switch (csp->csp_auth_klen) {
545 		case 128 / 8:
546 			return (&auth_hash_ccm_cbc_mac_128);
547 		case 192 / 8:
548 			return (&auth_hash_ccm_cbc_mac_192);
549 		case 256 / 8:
550 			return (&auth_hash_ccm_cbc_mac_256);
551 		default:
552 			return (NULL);
553 		}
554 	default:
555 		return (NULL);
556 	}
557 }
558 
559 const struct enc_xform *
560 crypto_cipher(const struct crypto_session_params *csp)
561 {
562 
563 	switch (csp->csp_cipher_alg) {
564 	case CRYPTO_AES_CBC:
565 		return (&enc_xform_aes_cbc);
566 	case CRYPTO_AES_XTS:
567 		return (&enc_xform_aes_xts);
568 	case CRYPTO_AES_ICM:
569 		return (&enc_xform_aes_icm);
570 	case CRYPTO_AES_NIST_GCM_16:
571 		return (&enc_xform_aes_nist_gcm);
572 	case CRYPTO_CAMELLIA_CBC:
573 		return (&enc_xform_camellia);
574 	case CRYPTO_NULL_CBC:
575 		return (&enc_xform_null);
576 	case CRYPTO_CHACHA20:
577 		return (&enc_xform_chacha20);
578 	case CRYPTO_AES_CCM_16:
579 		return (&enc_xform_ccm);
580 	case CRYPTO_CHACHA20_POLY1305:
581 		return (&enc_xform_chacha20_poly1305);
582 	case CRYPTO_XCHACHA20_POLY1305:
583 		return (&enc_xform_xchacha20_poly1305);
584 	default:
585 		return (NULL);
586 	}
587 }
588 
589 static struct cryptocap *
590 crypto_checkdriver(uint32_t hid)
591 {
592 
593 	return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
594 }
595 
596 /*
597  * Select a driver for a new session that supports the specified
598  * algorithms and, optionally, is constrained according to the flags.
599  */
600 static struct cryptocap *
601 crypto_select_driver(const struct crypto_session_params *csp, int flags)
602 {
603 	struct cryptocap *cap, *best;
604 	int best_match, error, hid;
605 
606 	CRYPTO_DRIVER_ASSERT();
607 
608 	best = NULL;
609 	for (hid = 0; hid < crypto_drivers_size; hid++) {
610 		/*
611 		 * If there is no driver for this slot, or the driver
612 		 * is not appropriate (hardware or software based on
613 		 * match), then skip.
614 		 */
615 		cap = crypto_drivers[hid];
616 		if (cap == NULL ||
617 		    (cap->cc_flags & flags) == 0)
618 			continue;
619 
620 		error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
621 		if (error >= 0)
622 			continue;
623 
624 		/*
625 		 * Use the driver with the highest probe value.
626 		 * Hardware drivers use a higher probe value than
627 		 * software.  In case of a tie, prefer the driver with
628 		 * the fewest active sessions.
629 		 */
630 		if (best == NULL || error > best_match ||
631 		    (error == best_match &&
632 		    cap->cc_sessions < best->cc_sessions)) {
633 			best = cap;
634 			best_match = error;
635 		}
636 	}
637 	return best;
638 }
639 
640 static enum alg_type {
641 	ALG_NONE = 0,
642 	ALG_CIPHER,
643 	ALG_DIGEST,
644 	ALG_KEYED_DIGEST,
645 	ALG_COMPRESSION,
646 	ALG_AEAD
647 } alg_types[] = {
648 	[CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST,
649 	[CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST,
650 	[CRYPTO_AES_CBC] = ALG_CIPHER,
651 	[CRYPTO_SHA1] = ALG_DIGEST,
652 	[CRYPTO_NULL_HMAC] = ALG_DIGEST,
653 	[CRYPTO_NULL_CBC] = ALG_CIPHER,
654 	[CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION,
655 	[CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST,
656 	[CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST,
657 	[CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST,
658 	[CRYPTO_CAMELLIA_CBC] = ALG_CIPHER,
659 	[CRYPTO_AES_XTS] = ALG_CIPHER,
660 	[CRYPTO_AES_ICM] = ALG_CIPHER,
661 	[CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST,
662 	[CRYPTO_AES_NIST_GCM_16] = ALG_AEAD,
663 	[CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST,
664 	[CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST,
665 	[CRYPTO_CHACHA20] = ALG_CIPHER,
666 	[CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST,
667 	[CRYPTO_RIPEMD160] = ALG_DIGEST,
668 	[CRYPTO_SHA2_224] = ALG_DIGEST,
669 	[CRYPTO_SHA2_256] = ALG_DIGEST,
670 	[CRYPTO_SHA2_384] = ALG_DIGEST,
671 	[CRYPTO_SHA2_512] = ALG_DIGEST,
672 	[CRYPTO_POLY1305] = ALG_KEYED_DIGEST,
673 	[CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST,
674 	[CRYPTO_AES_CCM_16] = ALG_AEAD,
675 	[CRYPTO_CHACHA20_POLY1305] = ALG_AEAD,
676 	[CRYPTO_XCHACHA20_POLY1305] = ALG_AEAD,
677 };
678 
679 static enum alg_type
680 alg_type(int alg)
681 {
682 
683 	if (alg < nitems(alg_types))
684 		return (alg_types[alg]);
685 	return (ALG_NONE);
686 }
687 
688 static bool
689 alg_is_compression(int alg)
690 {
691 
692 	return (alg_type(alg) == ALG_COMPRESSION);
693 }
694 
695 static bool
696 alg_is_cipher(int alg)
697 {
698 
699 	return (alg_type(alg) == ALG_CIPHER);
700 }
701 
702 static bool
703 alg_is_digest(int alg)
704 {
705 
706 	return (alg_type(alg) == ALG_DIGEST ||
707 	    alg_type(alg) == ALG_KEYED_DIGEST);
708 }
709 
710 static bool
711 alg_is_keyed_digest(int alg)
712 {
713 
714 	return (alg_type(alg) == ALG_KEYED_DIGEST);
715 }
716 
717 static bool
718 alg_is_aead(int alg)
719 {
720 
721 	return (alg_type(alg) == ALG_AEAD);
722 }
723 
724 static bool
725 ccm_tag_length_valid(int len)
726 {
727 	/* RFC 3610 */
728 	switch (len) {
729 	case 4:
730 	case 6:
731 	case 8:
732 	case 10:
733 	case 12:
734 	case 14:
735 	case 16:
736 		return (true);
737 	default:
738 		return (false);
739 	}
740 }
741 
742 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
743 
744 /* Various sanity checks on crypto session parameters. */
745 static bool
746 check_csp(const struct crypto_session_params *csp)
747 {
748 	const struct auth_hash *axf;
749 
750 	/* Mode-independent checks. */
751 	if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
752 		return (false);
753 	if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
754 	    csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
755 		return (false);
756 	if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
757 		return (false);
758 	if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
759 		return (false);
760 
761 	switch (csp->csp_mode) {
762 	case CSP_MODE_COMPRESS:
763 		if (!alg_is_compression(csp->csp_cipher_alg))
764 			return (false);
765 		if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT)
766 			return (false);
767 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
768 			return (false);
769 		if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
770 		    csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
771 		    csp->csp_auth_mlen != 0)
772 			return (false);
773 		break;
774 	case CSP_MODE_CIPHER:
775 		if (!alg_is_cipher(csp->csp_cipher_alg))
776 			return (false);
777 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
778 			return (false);
779 		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
780 			if (csp->csp_cipher_klen == 0)
781 				return (false);
782 			if (csp->csp_ivlen == 0)
783 				return (false);
784 		}
785 		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
786 			return (false);
787 		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
788 		    csp->csp_auth_mlen != 0)
789 			return (false);
790 		break;
791 	case CSP_MODE_DIGEST:
792 		if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
793 			return (false);
794 
795 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
796 			return (false);
797 
798 		/* IV is optional for digests (e.g. GMAC). */
799 		switch (csp->csp_auth_alg) {
800 		case CRYPTO_AES_CCM_CBC_MAC:
801 			if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
802 				return (false);
803 			break;
804 		case CRYPTO_AES_NIST_GMAC:
805 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
806 				return (false);
807 			break;
808 		default:
809 			if (csp->csp_ivlen != 0)
810 				return (false);
811 			break;
812 		}
813 
814 		if (!alg_is_digest(csp->csp_auth_alg))
815 			return (false);
816 
817 		/* Key is optional for BLAKE2 digests. */
818 		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
819 		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
820 			;
821 		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
822 			if (csp->csp_auth_klen == 0)
823 				return (false);
824 		} else {
825 			if (csp->csp_auth_klen != 0)
826 				return (false);
827 		}
828 		if (csp->csp_auth_mlen != 0) {
829 			axf = crypto_auth_hash(csp);
830 			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
831 				return (false);
832 
833 			if (csp->csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC &&
834 			    !ccm_tag_length_valid(csp->csp_auth_mlen))
835 				return (false);
836 		}
837 		break;
838 	case CSP_MODE_AEAD:
839 		if (!alg_is_aead(csp->csp_cipher_alg))
840 			return (false);
841 		if (csp->csp_cipher_klen == 0)
842 			return (false);
843 		if (csp->csp_ivlen == 0 ||
844 		    csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
845 			return (false);
846 		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
847 			return (false);
848 
849 		switch (csp->csp_cipher_alg) {
850 		case CRYPTO_AES_CCM_16:
851 			if (csp->csp_auth_mlen != 0 &&
852 			    !ccm_tag_length_valid(csp->csp_auth_mlen))
853 				return (false);
854 
855 			if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
856 				return (false);
857 			break;
858 		case CRYPTO_AES_NIST_GCM_16:
859 			if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
860 				return (false);
861 
862 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
863 				return (false);
864 			break;
865 		case CRYPTO_CHACHA20_POLY1305:
866 			if (csp->csp_ivlen != 8 && csp->csp_ivlen != 12)
867 				return (false);
868 			if (csp->csp_auth_mlen > POLY1305_HASH_LEN)
869 				return (false);
870 			break;
871 		case CRYPTO_XCHACHA20_POLY1305:
872 			if (csp->csp_ivlen != XCHACHA20_POLY1305_IV_LEN)
873 				return (false);
874 			if (csp->csp_auth_mlen > POLY1305_HASH_LEN)
875 				return (false);
876 			break;
877 		}
878 		break;
879 	case CSP_MODE_ETA:
880 		if (!alg_is_cipher(csp->csp_cipher_alg))
881 			return (false);
882 		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
883 			if (csp->csp_cipher_klen == 0)
884 				return (false);
885 			if (csp->csp_ivlen == 0)
886 				return (false);
887 		}
888 		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
889 			return (false);
890 		if (!alg_is_digest(csp->csp_auth_alg))
891 			return (false);
892 
893 		/* Key is optional for BLAKE2 digests. */
894 		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
895 		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
896 			;
897 		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
898 			if (csp->csp_auth_klen == 0)
899 				return (false);
900 		} else {
901 			if (csp->csp_auth_klen != 0)
902 				return (false);
903 		}
904 		if (csp->csp_auth_mlen != 0) {
905 			axf = crypto_auth_hash(csp);
906 			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
907 				return (false);
908 		}
909 		break;
910 	default:
911 		return (false);
912 	}
913 
914 	return (true);
915 }
916 
917 /*
918  * Delete a session after it has been detached from its driver.
919  */
920 static void
921 crypto_deletesession(crypto_session_t cses)
922 {
923 	struct cryptocap *cap;
924 
925 	cap = cses->cap;
926 
927 	zfree(cses, M_CRYPTO_DATA);
928 
929 	CRYPTO_DRIVER_LOCK();
930 	cap->cc_sessions--;
931 	if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
932 		wakeup(cap);
933 	CRYPTO_DRIVER_UNLOCK();
934 	cap_rele(cap);
935 }
936 
937 /*
938  * Create a new session.  The crid argument specifies a crypto
939  * driver to use or constraints on a driver to select (hardware
940  * only, software only, either).  Whatever driver is selected
941  * must be capable of the requested crypto algorithms.
942  */
943 int
944 crypto_newsession(crypto_session_t *cses,
945     const struct crypto_session_params *csp, int crid)
946 {
947 	static uint64_t sessid = 0;
948 	crypto_session_t res;
949 	struct cryptocap *cap;
950 	int err;
951 
952 	if (!check_csp(csp))
953 		return (EINVAL);
954 
955 	res = NULL;
956 
957 	CRYPTO_DRIVER_LOCK();
958 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
959 		/*
960 		 * Use specified driver; verify it is capable.
961 		 */
962 		cap = crypto_checkdriver(crid);
963 		if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
964 			cap = NULL;
965 	} else {
966 		/*
967 		 * No requested driver; select based on crid flags.
968 		 */
969 		cap = crypto_select_driver(csp, crid);
970 	}
971 	if (cap == NULL) {
972 		CRYPTO_DRIVER_UNLOCK();
973 		CRYPTDEB("no driver");
974 		return (EOPNOTSUPP);
975 	}
976 	cap_ref(cap);
977 	cap->cc_sessions++;
978 	CRYPTO_DRIVER_UNLOCK();
979 
980 	/* Allocate a single block for the generic session and driver softc. */
981 	res = malloc(sizeof(*res) + cap->cc_session_size, M_CRYPTO_DATA,
982 	    M_WAITOK | M_ZERO);
983 	res->cap = cap;
984 	res->csp = *csp;
985 	res->id = atomic_fetchadd_64(&sessid, 1);
986 
987 	/* Call the driver initialization routine. */
988 	err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
989 	if (err != 0) {
990 		CRYPTDEB("dev newsession failed: %d", err);
991 		crypto_deletesession(res);
992 		return (err);
993 	}
994 
995 	*cses = res;
996 	return (0);
997 }
998 
999 /*
1000  * Delete an existing session (or a reserved session on an unregistered
1001  * driver).
1002  */
1003 void
1004 crypto_freesession(crypto_session_t cses)
1005 {
1006 	struct cryptocap *cap;
1007 
1008 	if (cses == NULL)
1009 		return;
1010 
1011 	cap = cses->cap;
1012 
1013 	/* Call the driver cleanup routine, if available. */
1014 	CRYPTODEV_FREESESSION(cap->cc_dev, cses);
1015 
1016 	crypto_deletesession(cses);
1017 }
1018 
1019 /*
1020  * Return a new driver id.  Registers a driver with the system so that
1021  * it can be probed by subsequent sessions.
1022  */
1023 int32_t
1024 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
1025 {
1026 	struct cryptocap *cap, **newdrv;
1027 	int i;
1028 
1029 	if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1030 		device_printf(dev,
1031 		    "no flags specified when registering driver\n");
1032 		return -1;
1033 	}
1034 
1035 	cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1036 	cap->cc_dev = dev;
1037 	cap->cc_session_size = sessionsize;
1038 	cap->cc_flags = flags;
1039 	refcount_init(&cap->cc_refs, 1);
1040 
1041 	CRYPTO_DRIVER_LOCK();
1042 	for (;;) {
1043 		for (i = 0; i < crypto_drivers_size; i++) {
1044 			if (crypto_drivers[i] == NULL)
1045 				break;
1046 		}
1047 
1048 		if (i < crypto_drivers_size)
1049 			break;
1050 
1051 		/* Out of entries, allocate some more. */
1052 
1053 		if (2 * crypto_drivers_size <= crypto_drivers_size) {
1054 			CRYPTO_DRIVER_UNLOCK();
1055 			printf("crypto: driver count wraparound!\n");
1056 			cap_rele(cap);
1057 			return (-1);
1058 		}
1059 		CRYPTO_DRIVER_UNLOCK();
1060 
1061 		newdrv = malloc(2 * crypto_drivers_size *
1062 		    sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1063 
1064 		CRYPTO_DRIVER_LOCK();
1065 		memcpy(newdrv, crypto_drivers,
1066 		    crypto_drivers_size * sizeof(*crypto_drivers));
1067 
1068 		crypto_drivers_size *= 2;
1069 
1070 		free(crypto_drivers, M_CRYPTO_DATA);
1071 		crypto_drivers = newdrv;
1072 	}
1073 
1074 	cap->cc_hid = i;
1075 	crypto_drivers[i] = cap;
1076 	CRYPTO_DRIVER_UNLOCK();
1077 
1078 	if (bootverbose)
1079 		printf("crypto: assign %s driver id %u, flags 0x%x\n",
1080 		    device_get_nameunit(dev), i, flags);
1081 
1082 	return i;
1083 }
1084 
1085 /*
1086  * Lookup a driver by name.  We match against the full device
1087  * name and unit, and against just the name.  The latter gives
1088  * us a simple widlcarding by device name.  On success return the
1089  * driver/hardware identifier; otherwise return -1.
1090  */
1091 int
1092 crypto_find_driver(const char *match)
1093 {
1094 	struct cryptocap *cap;
1095 	int i, len = strlen(match);
1096 
1097 	CRYPTO_DRIVER_LOCK();
1098 	for (i = 0; i < crypto_drivers_size; i++) {
1099 		if (crypto_drivers[i] == NULL)
1100 			continue;
1101 		cap = crypto_drivers[i];
1102 		if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
1103 		    strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
1104 			CRYPTO_DRIVER_UNLOCK();
1105 			return (i);
1106 		}
1107 	}
1108 	CRYPTO_DRIVER_UNLOCK();
1109 	return (-1);
1110 }
1111 
1112 /*
1113  * Return the device_t for the specified driver or NULL
1114  * if the driver identifier is invalid.
1115  */
1116 device_t
1117 crypto_find_device_byhid(int hid)
1118 {
1119 	struct cryptocap *cap;
1120 	device_t dev;
1121 
1122 	dev = NULL;
1123 	CRYPTO_DRIVER_LOCK();
1124 	cap = crypto_checkdriver(hid);
1125 	if (cap != NULL)
1126 		dev = cap->cc_dev;
1127 	CRYPTO_DRIVER_UNLOCK();
1128 	return (dev);
1129 }
1130 
1131 /*
1132  * Return the device/driver capabilities.
1133  */
1134 int
1135 crypto_getcaps(int hid)
1136 {
1137 	struct cryptocap *cap;
1138 	int flags;
1139 
1140 	flags = 0;
1141 	CRYPTO_DRIVER_LOCK();
1142 	cap = crypto_checkdriver(hid);
1143 	if (cap != NULL)
1144 		flags = cap->cc_flags;
1145 	CRYPTO_DRIVER_UNLOCK();
1146 	return (flags);
1147 }
1148 
1149 /*
1150  * Unregister all algorithms associated with a crypto driver.
1151  * If there are pending sessions using it, leave enough information
1152  * around so that subsequent calls using those sessions will
1153  * correctly detect the driver has been unregistered and reroute
1154  * requests.
1155  */
1156 int
1157 crypto_unregister_all(uint32_t driverid)
1158 {
1159 	struct cryptocap *cap;
1160 
1161 	CRYPTO_DRIVER_LOCK();
1162 	cap = crypto_checkdriver(driverid);
1163 	if (cap == NULL) {
1164 		CRYPTO_DRIVER_UNLOCK();
1165 		return (EINVAL);
1166 	}
1167 
1168 	cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
1169 	crypto_drivers[driverid] = NULL;
1170 
1171 	/*
1172 	 * XXX: This doesn't do anything to kick sessions that
1173 	 * have no pending operations.
1174 	 */
1175 	while (cap->cc_sessions != 0)
1176 		mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
1177 	CRYPTO_DRIVER_UNLOCK();
1178 	cap_rele(cap);
1179 
1180 	return (0);
1181 }
1182 
1183 /*
1184  * Clear blockage on a driver.  The what parameter indicates whether
1185  * the driver is now ready for cryptop's and/or cryptokop's.
1186  */
1187 int
1188 crypto_unblock(uint32_t driverid, int what)
1189 {
1190 	struct cryptocap *cap;
1191 	int err;
1192 
1193 	CRYPTO_Q_LOCK();
1194 	cap = crypto_checkdriver(driverid);
1195 	if (cap != NULL) {
1196 		if (what & CRYPTO_SYMQ)
1197 			cap->cc_qblocked = 0;
1198 		if (crp_sleep)
1199 			wakeup_one(&crp_q);
1200 		err = 0;
1201 	} else
1202 		err = EINVAL;
1203 	CRYPTO_Q_UNLOCK();
1204 
1205 	return err;
1206 }
1207 
1208 size_t
1209 crypto_buffer_len(struct crypto_buffer *cb)
1210 {
1211 	switch (cb->cb_type) {
1212 	case CRYPTO_BUF_CONTIG:
1213 		return (cb->cb_buf_len);
1214 	case CRYPTO_BUF_MBUF:
1215 		if (cb->cb_mbuf->m_flags & M_PKTHDR)
1216 			return (cb->cb_mbuf->m_pkthdr.len);
1217 		return (m_length(cb->cb_mbuf, NULL));
1218 	case CRYPTO_BUF_SINGLE_MBUF:
1219 		return (cb->cb_mbuf->m_len);
1220 	case CRYPTO_BUF_VMPAGE:
1221 		return (cb->cb_vm_page_len);
1222 	case CRYPTO_BUF_UIO:
1223 		return (cb->cb_uio->uio_resid);
1224 	default:
1225 		return (0);
1226 	}
1227 }
1228 
1229 #ifdef INVARIANTS
1230 /* Various sanity checks on crypto requests. */
1231 static void
1232 cb_sanity(struct crypto_buffer *cb, const char *name)
1233 {
1234 	KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
1235 	    ("incoming crp with invalid %s buffer type", name));
1236 	switch (cb->cb_type) {
1237 	case CRYPTO_BUF_CONTIG:
1238 		KASSERT(cb->cb_buf_len >= 0,
1239 		    ("incoming crp with -ve %s buffer length", name));
1240 		break;
1241 	case CRYPTO_BUF_VMPAGE:
1242 		KASSERT(CRYPTO_HAS_VMPAGE,
1243 		    ("incoming crp uses dmap on supported arch"));
1244 		KASSERT(cb->cb_vm_page_len >= 0,
1245 		    ("incoming crp with -ve %s buffer length", name));
1246 		KASSERT(cb->cb_vm_page_offset >= 0,
1247 		    ("incoming crp with -ve %s buffer offset", name));
1248 		KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
1249 		    ("incoming crp with %s buffer offset greater than page size"
1250 		     , name));
1251 		break;
1252 	default:
1253 		break;
1254 	}
1255 }
1256 
1257 static void
1258 crp_sanity(struct cryptop *crp)
1259 {
1260 	struct crypto_session_params *csp;
1261 	struct crypto_buffer *out;
1262 	size_t ilen, len, olen;
1263 
1264 	KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
1265 	KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
1266 	    crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
1267 	    ("incoming crp with invalid output buffer type"));
1268 	KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
1269 	KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
1270 	    ("incoming crp already done"));
1271 
1272 	csp = &crp->crp_session->csp;
1273 	cb_sanity(&crp->crp_buf, "input");
1274 	ilen = crypto_buffer_len(&crp->crp_buf);
1275 	olen = ilen;
1276 	out = NULL;
1277 	if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) {
1278 		if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
1279 			cb_sanity(&crp->crp_obuf, "output");
1280 			out = &crp->crp_obuf;
1281 			olen = crypto_buffer_len(out);
1282 		}
1283 	} else
1284 		KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
1285 		    ("incoming crp with separate output buffer "
1286 		    "but no session support"));
1287 
1288 	switch (csp->csp_mode) {
1289 	case CSP_MODE_COMPRESS:
1290 		KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
1291 		    crp->crp_op == CRYPTO_OP_DECOMPRESS,
1292 		    ("invalid compression op %x", crp->crp_op));
1293 		break;
1294 	case CSP_MODE_CIPHER:
1295 		KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
1296 		    crp->crp_op == CRYPTO_OP_DECRYPT,
1297 		    ("invalid cipher op %x", crp->crp_op));
1298 		break;
1299 	case CSP_MODE_DIGEST:
1300 		KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
1301 		    crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
1302 		    ("invalid digest op %x", crp->crp_op));
1303 		break;
1304 	case CSP_MODE_AEAD:
1305 		KASSERT(crp->crp_op ==
1306 		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1307 		    crp->crp_op ==
1308 		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1309 		    ("invalid AEAD op %x", crp->crp_op));
1310 		KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1311 		    ("AEAD without a separate IV"));
1312 		break;
1313 	case CSP_MODE_ETA:
1314 		KASSERT(crp->crp_op ==
1315 		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1316 		    crp->crp_op ==
1317 		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1318 		    ("invalid ETA op %x", crp->crp_op));
1319 		break;
1320 	}
1321 	if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1322 		if (crp->crp_aad == NULL) {
1323 			KASSERT(crp->crp_aad_start == 0 ||
1324 			    crp->crp_aad_start < ilen,
1325 			    ("invalid AAD start"));
1326 			KASSERT(crp->crp_aad_length != 0 ||
1327 			    crp->crp_aad_start == 0,
1328 			    ("AAD with zero length and non-zero start"));
1329 			KASSERT(crp->crp_aad_length == 0 ||
1330 			    crp->crp_aad_start + crp->crp_aad_length <= ilen,
1331 			    ("AAD outside input length"));
1332 		} else {
1333 			KASSERT(csp->csp_flags & CSP_F_SEPARATE_AAD,
1334 			    ("session doesn't support separate AAD buffer"));
1335 			KASSERT(crp->crp_aad_start == 0,
1336 			    ("separate AAD buffer with non-zero AAD start"));
1337 			KASSERT(crp->crp_aad_length != 0,
1338 			    ("separate AAD buffer with zero length"));
1339 		}
1340 	} else {
1341 		KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 &&
1342 		    crp->crp_aad_length == 0,
1343 		    ("AAD region in request not supporting AAD"));
1344 	}
1345 	if (csp->csp_ivlen == 0) {
1346 		KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0,
1347 		    ("IV_SEPARATE set when IV isn't used"));
1348 		KASSERT(crp->crp_iv_start == 0,
1349 		    ("crp_iv_start set when IV isn't used"));
1350 	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
1351 		KASSERT(crp->crp_iv_start == 0,
1352 		    ("IV_SEPARATE used with non-zero IV start"));
1353 	} else {
1354 		KASSERT(crp->crp_iv_start < ilen,
1355 		    ("invalid IV start"));
1356 		KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
1357 		    ("IV outside buffer length"));
1358 	}
1359 	/* XXX: payload_start of 0 should always be < ilen? */
1360 	KASSERT(crp->crp_payload_start == 0 ||
1361 	    crp->crp_payload_start < ilen,
1362 	    ("invalid payload start"));
1363 	KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
1364 	    ilen, ("payload outside input buffer"));
1365 	if (out == NULL) {
1366 		KASSERT(crp->crp_payload_output_start == 0,
1367 		    ("payload output start non-zero without output buffer"));
1368 	} else {
1369 		KASSERT(crp->crp_payload_output_start == 0 ||
1370 		    crp->crp_payload_output_start < olen,
1371 		    ("invalid payload output start"));
1372 		KASSERT(crp->crp_payload_output_start +
1373 		    crp->crp_payload_length <= olen,
1374 		    ("payload outside output buffer"));
1375 	}
1376 	if (csp->csp_mode == CSP_MODE_DIGEST ||
1377 	    csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1378 		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
1379 			len = ilen;
1380 		else
1381 			len = olen;
1382 		KASSERT(crp->crp_digest_start == 0 ||
1383 		    crp->crp_digest_start < len,
1384 		    ("invalid digest start"));
1385 		/* XXX: For the mlen == 0 case this check isn't perfect. */
1386 		KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
1387 		    ("digest outside buffer"));
1388 	} else {
1389 		KASSERT(crp->crp_digest_start == 0,
1390 		    ("non-zero digest start for request without a digest"));
1391 	}
1392 	if (csp->csp_cipher_klen != 0)
1393 		KASSERT(csp->csp_cipher_key != NULL ||
1394 		    crp->crp_cipher_key != NULL,
1395 		    ("cipher request without a key"));
1396 	if (csp->csp_auth_klen != 0)
1397 		KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
1398 		    ("auth request without a key"));
1399 	KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
1400 }
1401 #endif
1402 
1403 static int
1404 crypto_dispatch_one(struct cryptop *crp, int hint)
1405 {
1406 	struct cryptocap *cap;
1407 	int result;
1408 
1409 #ifdef INVARIANTS
1410 	crp_sanity(crp);
1411 #endif
1412 	CRYPTOSTAT_INC(cs_ops);
1413 
1414 	crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1415 
1416 	/*
1417 	 * Caller marked the request to be processed immediately; dispatch it
1418 	 * directly to the driver unless the driver is currently blocked, in
1419 	 * which case it is queued for deferred dispatch.
1420 	 */
1421 	cap = crp->crp_session->cap;
1422 	if (!atomic_load_int(&cap->cc_qblocked)) {
1423 		result = crypto_invoke(cap, crp, hint);
1424 		if (result != ERESTART)
1425 			return (result);
1426 
1427 		/*
1428 		 * The driver ran out of resources, put the request on the
1429 		 * queue.
1430 		 */
1431 	}
1432 	crypto_batch_enqueue(crp);
1433 	return (0);
1434 }
1435 
1436 int
1437 crypto_dispatch(struct cryptop *crp)
1438 {
1439 	return (crypto_dispatch_one(crp, 0));
1440 }
1441 
1442 int
1443 crypto_dispatch_async(struct cryptop *crp, int flags)
1444 {
1445 	struct crypto_ret_worker *ret_worker;
1446 
1447 	if (!CRYPTO_SESS_SYNC(crp->crp_session)) {
1448 		/*
1449 		 * The driver issues completions asynchonously, don't bother
1450 		 * deferring dispatch to a worker thread.
1451 		 */
1452 		return (crypto_dispatch(crp));
1453 	}
1454 
1455 #ifdef INVARIANTS
1456 	crp_sanity(crp);
1457 #endif
1458 	CRYPTOSTAT_INC(cs_ops);
1459 
1460 	crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1461 	if ((flags & CRYPTO_ASYNC_ORDERED) != 0) {
1462 		crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED;
1463 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1464 		CRYPTO_RETW_LOCK(ret_worker);
1465 		crp->crp_seq = ret_worker->reorder_ops++;
1466 		CRYPTO_RETW_UNLOCK(ret_worker);
1467 	}
1468 	TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
1469 	taskqueue_enqueue(crypto_tq, &crp->crp_task);
1470 	return (0);
1471 }
1472 
1473 void
1474 crypto_dispatch_batch(struct cryptopq *crpq, int flags)
1475 {
1476 	struct cryptop *crp;
1477 	int hint;
1478 
1479 	while ((crp = TAILQ_FIRST(crpq)) != NULL) {
1480 		hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0;
1481 		TAILQ_REMOVE(crpq, crp, crp_next);
1482 		if (crypto_dispatch_one(crp, hint) != 0)
1483 			crypto_batch_enqueue(crp);
1484 	}
1485 }
1486 
1487 static void
1488 crypto_batch_enqueue(struct cryptop *crp)
1489 {
1490 
1491 	CRYPTO_Q_LOCK();
1492 	TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
1493 	if (crp_sleep)
1494 		wakeup_one(&crp_q);
1495 	CRYPTO_Q_UNLOCK();
1496 }
1497 
1498 static void
1499 crypto_task_invoke(void *ctx, int pending)
1500 {
1501 	struct cryptocap *cap;
1502 	struct cryptop *crp;
1503 	int result;
1504 
1505 	crp = (struct cryptop *)ctx;
1506 	cap = crp->crp_session->cap;
1507 	result = crypto_invoke(cap, crp, 0);
1508 	if (result == ERESTART)
1509 		crypto_batch_enqueue(crp);
1510 }
1511 
1512 /*
1513  * Dispatch a crypto request to the appropriate crypto devices.
1514  */
1515 static int
1516 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1517 {
1518 
1519 	KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1520 	KASSERT(crp->crp_callback != NULL,
1521 	    ("%s: crp->crp_callback == NULL", __func__));
1522 	KASSERT(crp->crp_session != NULL,
1523 	    ("%s: crp->crp_session == NULL", __func__));
1524 
1525 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1526 		struct crypto_session_params csp;
1527 		crypto_session_t nses;
1528 
1529 		/*
1530 		 * Driver has unregistered; migrate the session and return
1531 		 * an error to the caller so they'll resubmit the op.
1532 		 *
1533 		 * XXX: What if there are more already queued requests for this
1534 		 *      session?
1535 		 *
1536 		 * XXX: Real solution is to make sessions refcounted
1537 		 * and force callers to hold a reference when
1538 		 * assigning to crp_session.  Could maybe change
1539 		 * crypto_getreq to accept a session pointer to make
1540 		 * that work.  Alternatively, we could abandon the
1541 		 * notion of rewriting crp_session in requests forcing
1542 		 * the caller to deal with allocating a new session.
1543 		 * Perhaps provide a method to allow a crp's session to
1544 		 * be swapped that callers could use.
1545 		 */
1546 		csp = crp->crp_session->csp;
1547 		crypto_freesession(crp->crp_session);
1548 
1549 		/*
1550 		 * XXX: Key pointers may no longer be valid.  If we
1551 		 * really want to support this we need to define the
1552 		 * KPI such that 'csp' is required to be valid for the
1553 		 * duration of a session by the caller perhaps.
1554 		 *
1555 		 * XXX: If the keys have been changed this will reuse
1556 		 * the old keys.  This probably suggests making
1557 		 * rekeying more explicit and updating the key
1558 		 * pointers in 'csp' when the keys change.
1559 		 */
1560 		if (crypto_newsession(&nses, &csp,
1561 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1562 			crp->crp_session = nses;
1563 
1564 		crp->crp_etype = EAGAIN;
1565 		crypto_done(crp);
1566 		return 0;
1567 	} else {
1568 		/*
1569 		 * Invoke the driver to process the request.
1570 		 */
1571 		return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1572 	}
1573 }
1574 
1575 void
1576 crypto_destroyreq(struct cryptop *crp)
1577 {
1578 #ifdef DIAGNOSTIC
1579 	{
1580 		struct cryptop *crp2;
1581 		struct crypto_ret_worker *ret_worker;
1582 
1583 		CRYPTO_Q_LOCK();
1584 		TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1585 			KASSERT(crp2 != crp,
1586 			    ("Freeing cryptop from the crypto queue (%p).",
1587 			    crp));
1588 		}
1589 		CRYPTO_Q_UNLOCK();
1590 
1591 		FOREACH_CRYPTO_RETW(ret_worker) {
1592 			CRYPTO_RETW_LOCK(ret_worker);
1593 			TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
1594 				KASSERT(crp2 != crp,
1595 				    ("Freeing cryptop from the return queue (%p).",
1596 				    crp));
1597 			}
1598 			CRYPTO_RETW_UNLOCK(ret_worker);
1599 		}
1600 	}
1601 #endif
1602 }
1603 
1604 void
1605 crypto_freereq(struct cryptop *crp)
1606 {
1607 	if (crp == NULL)
1608 		return;
1609 
1610 	crypto_destroyreq(crp);
1611 	uma_zfree(cryptop_zone, crp);
1612 }
1613 
1614 static void
1615 _crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1616 {
1617 	crp->crp_session = cses;
1618 }
1619 
1620 void
1621 crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1622 {
1623 	memset(crp, 0, sizeof(*crp));
1624 	_crypto_initreq(crp, cses);
1625 }
1626 
1627 struct cryptop *
1628 crypto_getreq(crypto_session_t cses, int how)
1629 {
1630 	struct cryptop *crp;
1631 
1632 	MPASS(how == M_WAITOK || how == M_NOWAIT);
1633 	crp = uma_zalloc(cryptop_zone, how | M_ZERO);
1634 	if (crp != NULL)
1635 		_crypto_initreq(crp, cses);
1636 	return (crp);
1637 }
1638 
1639 /*
1640  * Clone a crypto request, but associate it with the specified session
1641  * rather than inheriting the session from the original request.  The
1642  * fields describing the request buffers are copied, but not the
1643  * opaque field or callback function.
1644  */
1645 struct cryptop *
1646 crypto_clonereq(struct cryptop *crp, crypto_session_t cses, int how)
1647 {
1648 	struct cryptop *new;
1649 
1650 	MPASS((crp->crp_flags & CRYPTO_F_DONE) == 0);
1651 	new = crypto_getreq(cses, how);
1652 	if (new == NULL)
1653 		return (NULL);
1654 
1655 	memcpy(&new->crp_startcopy, &crp->crp_startcopy,
1656 	    __rangeof(struct cryptop, crp_startcopy, crp_endcopy));
1657 	return (new);
1658 }
1659 
1660 /*
1661  * Invoke the callback on behalf of the driver.
1662  */
1663 void
1664 crypto_done(struct cryptop *crp)
1665 {
1666 	KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1667 		("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1668 	crp->crp_flags |= CRYPTO_F_DONE;
1669 	if (crp->crp_etype != 0)
1670 		CRYPTOSTAT_INC(cs_errs);
1671 
1672 	/*
1673 	 * CBIMM means unconditionally do the callback immediately;
1674 	 * CBIFSYNC means do the callback immediately only if the
1675 	 * operation was done synchronously.  Both are used to avoid
1676 	 * doing extraneous context switches; the latter is mostly
1677 	 * used with the software crypto driver.
1678 	 */
1679 	if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 &&
1680 	    ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 ||
1681 	    ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 &&
1682 	    CRYPTO_SESS_SYNC(crp->crp_session)))) {
1683 		/*
1684 		 * Do the callback directly.  This is ok when the
1685 		 * callback routine does very little (e.g. the
1686 		 * /dev/crypto callback method just does a wakeup).
1687 		 */
1688 		crp->crp_callback(crp);
1689 	} else {
1690 		struct crypto_ret_worker *ret_worker;
1691 		bool wake;
1692 
1693 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1694 
1695 		/*
1696 		 * Normal case; queue the callback for the thread.
1697 		 */
1698 		CRYPTO_RETW_LOCK(ret_worker);
1699 		if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) {
1700 			struct cryptop *tmp;
1701 
1702 			TAILQ_FOREACH_REVERSE(tmp,
1703 			    &ret_worker->crp_ordered_ret_q, cryptop_q,
1704 			    crp_next) {
1705 				if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1706 					TAILQ_INSERT_AFTER(
1707 					    &ret_worker->crp_ordered_ret_q, tmp,
1708 					    crp, crp_next);
1709 					break;
1710 				}
1711 			}
1712 			if (tmp == NULL) {
1713 				TAILQ_INSERT_HEAD(
1714 				    &ret_worker->crp_ordered_ret_q, crp,
1715 				    crp_next);
1716 			}
1717 
1718 			wake = crp->crp_seq == ret_worker->reorder_cur_seq;
1719 		} else {
1720 			wake = TAILQ_EMPTY(&ret_worker->crp_ret_q);
1721 			TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp,
1722 			    crp_next);
1723 		}
1724 
1725 		if (wake)
1726 			wakeup_one(&ret_worker->crp_ret_q);	/* shared wait channel */
1727 		CRYPTO_RETW_UNLOCK(ret_worker);
1728 	}
1729 }
1730 
1731 /*
1732  * Terminate a thread at module unload.  The process that
1733  * initiated this is waiting for us to signal that we're gone;
1734  * wake it up and exit.  We use the driver table lock to insure
1735  * we don't do the wakeup before they're waiting.  There is no
1736  * race here because the waiter sleeps on the proc lock for the
1737  * thread so it gets notified at the right time because of an
1738  * extra wakeup that's done in exit1().
1739  */
1740 static void
1741 crypto_finis(void *chan)
1742 {
1743 	CRYPTO_DRIVER_LOCK();
1744 	wakeup_one(chan);
1745 	CRYPTO_DRIVER_UNLOCK();
1746 	kthread_exit();
1747 }
1748 
1749 /*
1750  * Crypto thread, dispatches crypto requests.
1751  */
1752 static void
1753 crypto_dispatch_thread(void *arg __unused)
1754 {
1755 	struct cryptop *crp, *submit;
1756 	struct cryptocap *cap;
1757 	int result, hint;
1758 
1759 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1760 	fpu_kern_thread(FPU_KERN_NORMAL);
1761 #endif
1762 
1763 	CRYPTO_Q_LOCK();
1764 	for (;;) {
1765 		/*
1766 		 * Find the first element in the queue that can be
1767 		 * processed and look-ahead to see if multiple ops
1768 		 * are ready for the same driver.
1769 		 */
1770 		submit = NULL;
1771 		hint = 0;
1772 		TAILQ_FOREACH(crp, &crp_q, crp_next) {
1773 			cap = crp->crp_session->cap;
1774 			/*
1775 			 * Driver cannot disappeared when there is an active
1776 			 * session.
1777 			 */
1778 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1779 			    __func__, __LINE__));
1780 			if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1781 				/* Op needs to be migrated, process it. */
1782 				if (submit == NULL)
1783 					submit = crp;
1784 				break;
1785 			}
1786 			if (!cap->cc_qblocked) {
1787 				if (submit != NULL) {
1788 					/*
1789 					 * We stop on finding another op,
1790 					 * regardless whether its for the same
1791 					 * driver or not.  We could keep
1792 					 * searching the queue but it might be
1793 					 * better to just use a per-driver
1794 					 * queue instead.
1795 					 */
1796 					if (submit->crp_session->cap == cap)
1797 						hint = CRYPTO_HINT_MORE;
1798 				} else {
1799 					submit = crp;
1800 				}
1801 				break;
1802 			}
1803 		}
1804 		if (submit != NULL) {
1805 			TAILQ_REMOVE(&crp_q, submit, crp_next);
1806 			cap = submit->crp_session->cap;
1807 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1808 			    __func__, __LINE__));
1809 			CRYPTO_Q_UNLOCK();
1810 			result = crypto_invoke(cap, submit, hint);
1811 			CRYPTO_Q_LOCK();
1812 			if (result == ERESTART) {
1813 				/*
1814 				 * The driver ran out of resources, mark the
1815 				 * driver ``blocked'' for cryptop's and put
1816 				 * the request back in the queue.  It would
1817 				 * best to put the request back where we got
1818 				 * it but that's hard so for now we put it
1819 				 * at the front.  This should be ok; putting
1820 				 * it at the end does not work.
1821 				 */
1822 				cap->cc_qblocked = 1;
1823 				TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1824 				CRYPTOSTAT_INC(cs_blocks);
1825 			}
1826 		} else {
1827 			/*
1828 			 * Nothing more to be processed.  Sleep until we're
1829 			 * woken because there are more ops to process.
1830 			 * This happens either by submission or by a driver
1831 			 * becoming unblocked and notifying us through
1832 			 * crypto_unblock.  Note that when we wakeup we
1833 			 * start processing each queue again from the
1834 			 * front. It's not clear that it's important to
1835 			 * preserve this ordering since ops may finish
1836 			 * out of order if dispatched to different devices
1837 			 * and some become blocked while others do not.
1838 			 */
1839 			crp_sleep = 1;
1840 			msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
1841 			crp_sleep = 0;
1842 			if (cryptotd == NULL)
1843 				break;
1844 			CRYPTOSTAT_INC(cs_intrs);
1845 		}
1846 	}
1847 	CRYPTO_Q_UNLOCK();
1848 
1849 	crypto_finis(&crp_q);
1850 }
1851 
1852 /*
1853  * Crypto returns thread, does callbacks for processed crypto requests.
1854  * Callbacks are done here, rather than in the crypto drivers, because
1855  * callbacks typically are expensive and would slow interrupt handling.
1856  */
1857 static void
1858 crypto_ret_thread(void *arg)
1859 {
1860 	struct crypto_ret_worker *ret_worker = arg;
1861 	struct cryptop *crpt;
1862 
1863 	CRYPTO_RETW_LOCK(ret_worker);
1864 	for (;;) {
1865 		/* Harvest return q's for completed ops */
1866 		crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
1867 		if (crpt != NULL) {
1868 			if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
1869 				TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
1870 				ret_worker->reorder_cur_seq++;
1871 			} else {
1872 				crpt = NULL;
1873 			}
1874 		}
1875 
1876 		if (crpt == NULL) {
1877 			crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
1878 			if (crpt != NULL)
1879 				TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
1880 		}
1881 
1882 		if (crpt != NULL) {
1883 			CRYPTO_RETW_UNLOCK(ret_worker);
1884 			/*
1885 			 * Run callbacks unlocked.
1886 			 */
1887 			if (crpt != NULL)
1888 				crpt->crp_callback(crpt);
1889 			CRYPTO_RETW_LOCK(ret_worker);
1890 		} else {
1891 			/*
1892 			 * Nothing more to be processed.  Sleep until we're
1893 			 * woken because there are more returns to process.
1894 			 */
1895 			msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
1896 				"crypto_ret_wait", 0);
1897 			if (ret_worker->td == NULL)
1898 				break;
1899 			CRYPTOSTAT_INC(cs_rets);
1900 		}
1901 	}
1902 	CRYPTO_RETW_UNLOCK(ret_worker);
1903 
1904 	crypto_finis(&ret_worker->crp_ret_q);
1905 }
1906 
1907 #ifdef DDB
1908 static void
1909 db_show_drivers(void)
1910 {
1911 	int hid;
1912 
1913 	db_printf("%12s %4s %8s %2s\n"
1914 		, "Device"
1915 		, "Ses"
1916 		, "Flags"
1917 		, "QB"
1918 	);
1919 	for (hid = 0; hid < crypto_drivers_size; hid++) {
1920 		const struct cryptocap *cap = crypto_drivers[hid];
1921 		if (cap == NULL)
1922 			continue;
1923 		db_printf("%-12s %4u %08x %2u\n"
1924 		    , device_get_nameunit(cap->cc_dev)
1925 		    , cap->cc_sessions
1926 		    , cap->cc_flags
1927 		    , cap->cc_qblocked
1928 		);
1929 	}
1930 }
1931 
1932 DB_SHOW_COMMAND(crypto, db_show_crypto)
1933 {
1934 	struct cryptop *crp;
1935 	struct crypto_ret_worker *ret_worker;
1936 
1937 	db_show_drivers();
1938 	db_printf("\n");
1939 
1940 	db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
1941 	    "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
1942 	    "Device", "Callback");
1943 	TAILQ_FOREACH(crp, &crp_q, crp_next) {
1944 		db_printf("%4u %08x %4u %4u %04x %8p %8p\n"
1945 		    , crp->crp_session->cap->cc_hid
1946 		    , (int) crypto_ses2caps(crp->crp_session)
1947 		    , crp->crp_olen
1948 		    , crp->crp_etype
1949 		    , crp->crp_flags
1950 		    , device_get_nameunit(crp->crp_session->cap->cc_dev)
1951 		    , crp->crp_callback
1952 		);
1953 	}
1954 	FOREACH_CRYPTO_RETW(ret_worker) {
1955 		db_printf("\n%8s %4s %4s %4s %8s\n",
1956 		    "ret_worker", "HID", "Etype", "Flags", "Callback");
1957 		if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
1958 			TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
1959 				db_printf("%8td %4u %4u %04x %8p\n"
1960 				    , CRYPTO_RETW_ID(ret_worker)
1961 				    , crp->crp_session->cap->cc_hid
1962 				    , crp->crp_etype
1963 				    , crp->crp_flags
1964 				    , crp->crp_callback
1965 				);
1966 			}
1967 		}
1968 	}
1969 }
1970 #endif
1971 
1972 int crypto_modevent(module_t mod, int type, void *unused);
1973 
1974 /*
1975  * Initialization code, both for static and dynamic loading.
1976  * Note this is not invoked with the usual MODULE_DECLARE
1977  * mechanism but instead is listed as a dependency by the
1978  * cryptosoft driver.  This guarantees proper ordering of
1979  * calls on module load/unload.
1980  */
1981 int
1982 crypto_modevent(module_t mod, int type, void *unused)
1983 {
1984 	int error = EINVAL;
1985 
1986 	switch (type) {
1987 	case MOD_LOAD:
1988 		error = crypto_init();
1989 		if (error == 0 && bootverbose)
1990 			printf("crypto: <crypto core>\n");
1991 		break;
1992 	case MOD_UNLOAD:
1993 		/*XXX disallow if active sessions */
1994 		error = 0;
1995 		crypto_destroy();
1996 		return 0;
1997 	}
1998 	return error;
1999 }
2000 MODULE_VERSION(crypto, 1);
2001 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
2002