xref: /freebsd/sys/opencrypto/crypto.c (revision acb1f1269c6f4ff89a0d28ba742f6687e9ef779d)
1 /*-
2  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  */
24 
25 #include <sys/cdefs.h>
26 __FBSDID("$FreeBSD$");
27 
28 /*
29  * Cryptographic Subsystem.
30  *
31  * This code is derived from the Openbsd Cryptographic Framework (OCF)
32  * that has the copyright shown below.  Very little of the original
33  * code remains.
34  */
35 
36 /*-
37  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
38  *
39  * This code was written by Angelos D. Keromytis in Athens, Greece, in
40  * February 2000. Network Security Technologies Inc. (NSTI) kindly
41  * supported the development of this code.
42  *
43  * Copyright (c) 2000, 2001 Angelos D. Keromytis
44  *
45  * Permission to use, copy, and modify this software with or without fee
46  * is hereby granted, provided that this entire notice is included in
47  * all source code copies of any software which is or includes a copy or
48  * modification of this software.
49  *
50  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
51  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
52  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
53  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
54  * PURPOSE.
55  */
56 
57 #include "opt_compat.h"
58 #include "opt_ddb.h"
59 
60 #include <sys/param.h>
61 #include <sys/systm.h>
62 #include <sys/counter.h>
63 #include <sys/kernel.h>
64 #include <sys/kthread.h>
65 #include <sys/linker.h>
66 #include <sys/lock.h>
67 #include <sys/module.h>
68 #include <sys/mutex.h>
69 #include <sys/malloc.h>
70 #include <sys/mbuf.h>
71 #include <sys/proc.h>
72 #include <sys/refcount.h>
73 #include <sys/sdt.h>
74 #include <sys/smp.h>
75 #include <sys/sysctl.h>
76 #include <sys/taskqueue.h>
77 #include <sys/uio.h>
78 
79 #include <ddb/ddb.h>
80 
81 #include <machine/vmparam.h>
82 #include <vm/uma.h>
83 
84 #include <crypto/intake.h>
85 #include <opencrypto/cryptodev.h>
86 #include <opencrypto/xform_auth.h>
87 #include <opencrypto/xform_enc.h>
88 
89 #include <sys/kobj.h>
90 #include <sys/bus.h>
91 #include "cryptodev_if.h"
92 
93 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
94 #include <machine/pcb.h>
95 #endif
96 
97 SDT_PROVIDER_DEFINE(opencrypto);
98 
99 /*
100  * Crypto drivers register themselves by allocating a slot in the
101  * crypto_drivers table with crypto_get_driverid().
102  */
103 static	struct mtx crypto_drivers_mtx;		/* lock on driver table */
104 #define	CRYPTO_DRIVER_LOCK()	mtx_lock(&crypto_drivers_mtx)
105 #define	CRYPTO_DRIVER_UNLOCK()	mtx_unlock(&crypto_drivers_mtx)
106 #define	CRYPTO_DRIVER_ASSERT()	mtx_assert(&crypto_drivers_mtx, MA_OWNED)
107 
108 /*
109  * Crypto device/driver capabilities structure.
110  *
111  * Synchronization:
112  * (d) - protected by CRYPTO_DRIVER_LOCK()
113  * (q) - protected by CRYPTO_Q_LOCK()
114  * Not tagged fields are read-only.
115  */
116 struct cryptocap {
117 	device_t	cc_dev;
118 	uint32_t	cc_hid;
119 	uint32_t	cc_sessions;		/* (d) # of sessions */
120 
121 	int		cc_flags;		/* (d) flags */
122 #define CRYPTOCAP_F_CLEANUP	0x80000000	/* needs resource cleanup */
123 	int		cc_qblocked;		/* (q) symmetric q blocked */
124 	size_t		cc_session_size;
125 	volatile int	cc_refs;
126 };
127 
128 static	struct cryptocap **crypto_drivers = NULL;
129 static	int crypto_drivers_size = 0;
130 
131 struct crypto_session {
132 	struct cryptocap *cap;
133 	struct crypto_session_params csp;
134 	uint64_t id;
135 	/* Driver softc follows. */
136 };
137 
138 static	int crp_sleep = 0;
139 static	TAILQ_HEAD(cryptop_q ,cryptop) crp_q;		/* request queues */
140 static	struct mtx crypto_q_mtx;
141 #define	CRYPTO_Q_LOCK()		mtx_lock(&crypto_q_mtx)
142 #define	CRYPTO_Q_UNLOCK()	mtx_unlock(&crypto_q_mtx)
143 
144 SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
145     "In-kernel cryptography");
146 
147 /*
148  * Taskqueue used to dispatch the crypto requests
149  * that have the CRYPTO_F_ASYNC flag
150  */
151 static struct taskqueue *crypto_tq;
152 
153 /*
154  * Crypto seq numbers are operated on with modular arithmetic
155  */
156 #define	CRYPTO_SEQ_GT(a,b)	((int)((a)-(b)) > 0)
157 
158 struct crypto_ret_worker {
159 	struct mtx crypto_ret_mtx;
160 
161 	TAILQ_HEAD(,cryptop) crp_ordered_ret_q;	/* ordered callback queue for symetric jobs */
162 	TAILQ_HEAD(,cryptop) crp_ret_q;		/* callback queue for symetric jobs */
163 
164 	uint32_t reorder_ops;		/* total ordered sym jobs received */
165 	uint32_t reorder_cur_seq;	/* current sym job dispatched */
166 
167 	struct proc *cryptoretproc;
168 };
169 static struct crypto_ret_worker *crypto_ret_workers = NULL;
170 
171 #define CRYPTO_RETW(i)		(&crypto_ret_workers[i])
172 #define CRYPTO_RETW_ID(w)	((w) - crypto_ret_workers)
173 #define FOREACH_CRYPTO_RETW(w) \
174 	for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
175 
176 #define	CRYPTO_RETW_LOCK(w)	mtx_lock(&w->crypto_ret_mtx)
177 #define	CRYPTO_RETW_UNLOCK(w)	mtx_unlock(&w->crypto_ret_mtx)
178 
179 static int crypto_workers_num = 0;
180 SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
181 	   &crypto_workers_num, 0,
182 	   "Number of crypto workers used to dispatch crypto jobs");
183 #ifdef COMPAT_FREEBSD12
184 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
185 	   &crypto_workers_num, 0,
186 	   "Number of crypto workers used to dispatch crypto jobs");
187 #endif
188 
189 static	uma_zone_t cryptop_zone;
190 
191 int	crypto_devallowsoft = 0;
192 SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RW,
193 	   &crypto_devallowsoft, 0,
194 	   "Enable use of software crypto by /dev/crypto");
195 #ifdef COMPAT_FREEBSD12
196 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
197 	   &crypto_devallowsoft, 0,
198 	   "Enable/disable use of software crypto by /dev/crypto");
199 #endif
200 
201 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
202 
203 static	void crypto_proc(void);
204 static	struct proc *cryptoproc;
205 static	void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
206 static	void crypto_destroy(void);
207 static	int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
208 static	void crypto_task_invoke(void *ctx, int pending);
209 static void crypto_batch_enqueue(struct cryptop *crp);
210 
211 static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)];
212 SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW,
213     cryptostats, nitems(cryptostats),
214     "Crypto system statistics");
215 
216 #define	CRYPTOSTAT_INC(stat) do {					\
217 	counter_u64_add(						\
218 	    cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\
219 	    1);								\
220 } while (0)
221 
222 static void
223 cryptostats_init(void *arg __unused)
224 {
225 	COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK);
226 }
227 SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL);
228 
229 static void
230 cryptostats_fini(void *arg __unused)
231 {
232 	COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats));
233 }
234 SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini,
235     NULL);
236 
237 /* Try to avoid directly exposing the key buffer as a symbol */
238 static struct keybuf *keybuf;
239 
240 static struct keybuf empty_keybuf = {
241         .kb_nents = 0
242 };
243 
244 /* Obtain the key buffer from boot metadata */
245 static void
246 keybuf_init(void)
247 {
248 	caddr_t kmdp;
249 
250 	kmdp = preload_search_by_type("elf kernel");
251 
252 	if (kmdp == NULL)
253 		kmdp = preload_search_by_type("elf64 kernel");
254 
255 	keybuf = (struct keybuf *)preload_search_info(kmdp,
256 	    MODINFO_METADATA | MODINFOMD_KEYBUF);
257 
258         if (keybuf == NULL)
259                 keybuf = &empty_keybuf;
260 }
261 
262 /* It'd be nice if we could store these in some kind of secure memory... */
263 struct keybuf *
264 get_keybuf(void)
265 {
266 
267         return (keybuf);
268 }
269 
270 static struct cryptocap *
271 cap_ref(struct cryptocap *cap)
272 {
273 
274 	refcount_acquire(&cap->cc_refs);
275 	return (cap);
276 }
277 
278 static void
279 cap_rele(struct cryptocap *cap)
280 {
281 
282 	if (refcount_release(&cap->cc_refs) == 0)
283 		return;
284 
285 	KASSERT(cap->cc_sessions == 0,
286 	    ("freeing crypto driver with active sessions"));
287 
288 	free(cap, M_CRYPTO_DATA);
289 }
290 
291 static int
292 crypto_init(void)
293 {
294 	struct crypto_ret_worker *ret_worker;
295 	int error;
296 
297 	mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
298 		MTX_DEF|MTX_QUIET);
299 
300 	TAILQ_INIT(&crp_q);
301 	mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
302 
303 	cryptop_zone = uma_zcreate("cryptop",
304 	    sizeof(struct cryptop), NULL, NULL, NULL, NULL,
305 	    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
306 
307 	crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
308 	crypto_drivers = malloc(crypto_drivers_size *
309 	    sizeof(struct cryptocap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
310 
311 	if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
312 		crypto_workers_num = mp_ncpus;
313 
314 	crypto_tq = taskqueue_create("crypto", M_WAITOK | M_ZERO,
315 	    taskqueue_thread_enqueue, &crypto_tq);
316 
317 	taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
318 	    "crypto");
319 
320 	error = kproc_create((void (*)(void *)) crypto_proc, NULL,
321 		    &cryptoproc, 0, 0, "crypto");
322 	if (error) {
323 		printf("crypto_init: cannot start crypto thread; error %d",
324 			error);
325 		goto bad;
326 	}
327 
328 	crypto_ret_workers = mallocarray(crypto_workers_num,
329 	    sizeof(struct crypto_ret_worker), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
330 
331 	FOREACH_CRYPTO_RETW(ret_worker) {
332 		TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
333 		TAILQ_INIT(&ret_worker->crp_ret_q);
334 
335 		ret_worker->reorder_ops = 0;
336 		ret_worker->reorder_cur_seq = 0;
337 
338 		mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
339 
340 		error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
341 				&ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
342 		if (error) {
343 			printf("crypto_init: cannot start cryptoret thread; error %d",
344 				error);
345 			goto bad;
346 		}
347 	}
348 
349 	keybuf_init();
350 
351 	return 0;
352 bad:
353 	crypto_destroy();
354 	return error;
355 }
356 
357 /*
358  * Signal a crypto thread to terminate.  We use the driver
359  * table lock to synchronize the sleep/wakeups so that we
360  * are sure the threads have terminated before we release
361  * the data structures they use.  See crypto_finis below
362  * for the other half of this song-and-dance.
363  */
364 static void
365 crypto_terminate(struct proc **pp, void *q)
366 {
367 	struct proc *p;
368 
369 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
370 	p = *pp;
371 	*pp = NULL;
372 	if (p) {
373 		wakeup_one(q);
374 		PROC_LOCK(p);		/* NB: insure we don't miss wakeup */
375 		CRYPTO_DRIVER_UNLOCK();	/* let crypto_finis progress */
376 		msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
377 		PROC_UNLOCK(p);
378 		CRYPTO_DRIVER_LOCK();
379 	}
380 }
381 
382 static void
383 hmac_init_pad(const struct auth_hash *axf, const char *key, int klen,
384     void *auth_ctx, uint8_t padval)
385 {
386 	uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
387 	u_int i;
388 
389 	KASSERT(axf->blocksize <= sizeof(hmac_key),
390 	    ("Invalid HMAC block size %d", axf->blocksize));
391 
392 	/*
393 	 * If the key is larger than the block size, use the digest of
394 	 * the key as the key instead.
395 	 */
396 	memset(hmac_key, 0, sizeof(hmac_key));
397 	if (klen > axf->blocksize) {
398 		axf->Init(auth_ctx);
399 		axf->Update(auth_ctx, key, klen);
400 		axf->Final(hmac_key, auth_ctx);
401 		klen = axf->hashsize;
402 	} else
403 		memcpy(hmac_key, key, klen);
404 
405 	for (i = 0; i < axf->blocksize; i++)
406 		hmac_key[i] ^= padval;
407 
408 	axf->Init(auth_ctx);
409 	axf->Update(auth_ctx, hmac_key, axf->blocksize);
410 	explicit_bzero(hmac_key, sizeof(hmac_key));
411 }
412 
413 void
414 hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen,
415     void *auth_ctx)
416 {
417 
418 	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
419 }
420 
421 void
422 hmac_init_opad(const struct auth_hash *axf, const char *key, int klen,
423     void *auth_ctx)
424 {
425 
426 	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
427 }
428 
429 static void
430 crypto_destroy(void)
431 {
432 	struct crypto_ret_worker *ret_worker;
433 	int i;
434 
435 	/*
436 	 * Terminate any crypto threads.
437 	 */
438 	if (crypto_tq != NULL)
439 		taskqueue_drain_all(crypto_tq);
440 	CRYPTO_DRIVER_LOCK();
441 	crypto_terminate(&cryptoproc, &crp_q);
442 	FOREACH_CRYPTO_RETW(ret_worker)
443 		crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
444 	CRYPTO_DRIVER_UNLOCK();
445 
446 	/* XXX flush queues??? */
447 
448 	/*
449 	 * Reclaim dynamically allocated resources.
450 	 */
451 	for (i = 0; i < crypto_drivers_size; i++) {
452 		if (crypto_drivers[i] != NULL)
453 			cap_rele(crypto_drivers[i]);
454 	}
455 	free(crypto_drivers, M_CRYPTO_DATA);
456 
457 	if (cryptop_zone != NULL)
458 		uma_zdestroy(cryptop_zone);
459 	mtx_destroy(&crypto_q_mtx);
460 	FOREACH_CRYPTO_RETW(ret_worker)
461 		mtx_destroy(&ret_worker->crypto_ret_mtx);
462 	free(crypto_ret_workers, M_CRYPTO_DATA);
463 	if (crypto_tq != NULL)
464 		taskqueue_free(crypto_tq);
465 	mtx_destroy(&crypto_drivers_mtx);
466 }
467 
468 uint32_t
469 crypto_ses2hid(crypto_session_t crypto_session)
470 {
471 	return (crypto_session->cap->cc_hid);
472 }
473 
474 uint32_t
475 crypto_ses2caps(crypto_session_t crypto_session)
476 {
477 	return (crypto_session->cap->cc_flags & 0xff000000);
478 }
479 
480 void *
481 crypto_get_driver_session(crypto_session_t crypto_session)
482 {
483 	return (crypto_session + 1);
484 }
485 
486 const struct crypto_session_params *
487 crypto_get_params(crypto_session_t crypto_session)
488 {
489 	return (&crypto_session->csp);
490 }
491 
492 struct auth_hash *
493 crypto_auth_hash(const struct crypto_session_params *csp)
494 {
495 
496 	switch (csp->csp_auth_alg) {
497 	case CRYPTO_SHA1_HMAC:
498 		return (&auth_hash_hmac_sha1);
499 	case CRYPTO_SHA2_224_HMAC:
500 		return (&auth_hash_hmac_sha2_224);
501 	case CRYPTO_SHA2_256_HMAC:
502 		return (&auth_hash_hmac_sha2_256);
503 	case CRYPTO_SHA2_384_HMAC:
504 		return (&auth_hash_hmac_sha2_384);
505 	case CRYPTO_SHA2_512_HMAC:
506 		return (&auth_hash_hmac_sha2_512);
507 	case CRYPTO_NULL_HMAC:
508 		return (&auth_hash_null);
509 	case CRYPTO_RIPEMD160_HMAC:
510 		return (&auth_hash_hmac_ripemd_160);
511 	case CRYPTO_SHA1:
512 		return (&auth_hash_sha1);
513 	case CRYPTO_SHA2_224:
514 		return (&auth_hash_sha2_224);
515 	case CRYPTO_SHA2_256:
516 		return (&auth_hash_sha2_256);
517 	case CRYPTO_SHA2_384:
518 		return (&auth_hash_sha2_384);
519 	case CRYPTO_SHA2_512:
520 		return (&auth_hash_sha2_512);
521 	case CRYPTO_AES_NIST_GMAC:
522 		switch (csp->csp_auth_klen) {
523 		case 128 / 8:
524 			return (&auth_hash_nist_gmac_aes_128);
525 		case 192 / 8:
526 			return (&auth_hash_nist_gmac_aes_192);
527 		case 256 / 8:
528 			return (&auth_hash_nist_gmac_aes_256);
529 		default:
530 			return (NULL);
531 		}
532 	case CRYPTO_BLAKE2B:
533 		return (&auth_hash_blake2b);
534 	case CRYPTO_BLAKE2S:
535 		return (&auth_hash_blake2s);
536 	case CRYPTO_POLY1305:
537 		return (&auth_hash_poly1305);
538 	case CRYPTO_AES_CCM_CBC_MAC:
539 		switch (csp->csp_auth_klen) {
540 		case 128 / 8:
541 			return (&auth_hash_ccm_cbc_mac_128);
542 		case 192 / 8:
543 			return (&auth_hash_ccm_cbc_mac_192);
544 		case 256 / 8:
545 			return (&auth_hash_ccm_cbc_mac_256);
546 		default:
547 			return (NULL);
548 		}
549 	default:
550 		return (NULL);
551 	}
552 }
553 
554 struct enc_xform *
555 crypto_cipher(const struct crypto_session_params *csp)
556 {
557 
558 	switch (csp->csp_cipher_alg) {
559 	case CRYPTO_RIJNDAEL128_CBC:
560 		return (&enc_xform_rijndael128);
561 	case CRYPTO_AES_XTS:
562 		return (&enc_xform_aes_xts);
563 	case CRYPTO_AES_ICM:
564 		return (&enc_xform_aes_icm);
565 	case CRYPTO_AES_NIST_GCM_16:
566 		return (&enc_xform_aes_nist_gcm);
567 	case CRYPTO_CAMELLIA_CBC:
568 		return (&enc_xform_camellia);
569 	case CRYPTO_NULL_CBC:
570 		return (&enc_xform_null);
571 	case CRYPTO_CHACHA20:
572 		return (&enc_xform_chacha20);
573 	case CRYPTO_AES_CCM_16:
574 		return (&enc_xform_ccm);
575 	case CRYPTO_CHACHA20_POLY1305:
576 		return (&enc_xform_chacha20_poly1305);
577 	default:
578 		return (NULL);
579 	}
580 }
581 
582 static struct cryptocap *
583 crypto_checkdriver(uint32_t hid)
584 {
585 
586 	return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
587 }
588 
589 /*
590  * Select a driver for a new session that supports the specified
591  * algorithms and, optionally, is constrained according to the flags.
592  */
593 static struct cryptocap *
594 crypto_select_driver(const struct crypto_session_params *csp, int flags)
595 {
596 	struct cryptocap *cap, *best;
597 	int best_match, error, hid;
598 
599 	CRYPTO_DRIVER_ASSERT();
600 
601 	best = NULL;
602 	for (hid = 0; hid < crypto_drivers_size; hid++) {
603 		/*
604 		 * If there is no driver for this slot, or the driver
605 		 * is not appropriate (hardware or software based on
606 		 * match), then skip.
607 		 */
608 		cap = crypto_drivers[hid];
609 		if (cap == NULL ||
610 		    (cap->cc_flags & flags) == 0)
611 			continue;
612 
613 		error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
614 		if (error >= 0)
615 			continue;
616 
617 		/*
618 		 * Use the driver with the highest probe value.
619 		 * Hardware drivers use a higher probe value than
620 		 * software.  In case of a tie, prefer the driver with
621 		 * the fewest active sessions.
622 		 */
623 		if (best == NULL || error > best_match ||
624 		    (error == best_match &&
625 		    cap->cc_sessions < best->cc_sessions)) {
626 			best = cap;
627 			best_match = error;
628 		}
629 	}
630 	return best;
631 }
632 
633 static enum alg_type {
634 	ALG_NONE = 0,
635 	ALG_CIPHER,
636 	ALG_DIGEST,
637 	ALG_KEYED_DIGEST,
638 	ALG_COMPRESSION,
639 	ALG_AEAD
640 } alg_types[] = {
641 	[CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST,
642 	[CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST,
643 	[CRYPTO_AES_CBC] = ALG_CIPHER,
644 	[CRYPTO_SHA1] = ALG_DIGEST,
645 	[CRYPTO_NULL_HMAC] = ALG_DIGEST,
646 	[CRYPTO_NULL_CBC] = ALG_CIPHER,
647 	[CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION,
648 	[CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST,
649 	[CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST,
650 	[CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST,
651 	[CRYPTO_CAMELLIA_CBC] = ALG_CIPHER,
652 	[CRYPTO_AES_XTS] = ALG_CIPHER,
653 	[CRYPTO_AES_ICM] = ALG_CIPHER,
654 	[CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST,
655 	[CRYPTO_AES_NIST_GCM_16] = ALG_AEAD,
656 	[CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST,
657 	[CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST,
658 	[CRYPTO_CHACHA20] = ALG_CIPHER,
659 	[CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST,
660 	[CRYPTO_RIPEMD160] = ALG_DIGEST,
661 	[CRYPTO_SHA2_224] = ALG_DIGEST,
662 	[CRYPTO_SHA2_256] = ALG_DIGEST,
663 	[CRYPTO_SHA2_384] = ALG_DIGEST,
664 	[CRYPTO_SHA2_512] = ALG_DIGEST,
665 	[CRYPTO_POLY1305] = ALG_KEYED_DIGEST,
666 	[CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST,
667 	[CRYPTO_AES_CCM_16] = ALG_AEAD,
668 	[CRYPTO_CHACHA20_POLY1305] = ALG_AEAD,
669 };
670 
671 static enum alg_type
672 alg_type(int alg)
673 {
674 
675 	if (alg < nitems(alg_types))
676 		return (alg_types[alg]);
677 	return (ALG_NONE);
678 }
679 
680 static bool
681 alg_is_compression(int alg)
682 {
683 
684 	return (alg_type(alg) == ALG_COMPRESSION);
685 }
686 
687 static bool
688 alg_is_cipher(int alg)
689 {
690 
691 	return (alg_type(alg) == ALG_CIPHER);
692 }
693 
694 static bool
695 alg_is_digest(int alg)
696 {
697 
698 	return (alg_type(alg) == ALG_DIGEST ||
699 	    alg_type(alg) == ALG_KEYED_DIGEST);
700 }
701 
702 static bool
703 alg_is_keyed_digest(int alg)
704 {
705 
706 	return (alg_type(alg) == ALG_KEYED_DIGEST);
707 }
708 
709 static bool
710 alg_is_aead(int alg)
711 {
712 
713 	return (alg_type(alg) == ALG_AEAD);
714 }
715 
716 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
717 
718 /* Various sanity checks on crypto session parameters. */
719 static bool
720 check_csp(const struct crypto_session_params *csp)
721 {
722 	struct auth_hash *axf;
723 
724 	/* Mode-independent checks. */
725 	if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
726 		return (false);
727 	if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
728 	    csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
729 		return (false);
730 	if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
731 		return (false);
732 	if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
733 		return (false);
734 
735 	switch (csp->csp_mode) {
736 	case CSP_MODE_COMPRESS:
737 		if (!alg_is_compression(csp->csp_cipher_alg))
738 			return (false);
739 		if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT)
740 			return (false);
741 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
742 			return (false);
743 		if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
744 		    csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
745 		    csp->csp_auth_mlen != 0)
746 			return (false);
747 		break;
748 	case CSP_MODE_CIPHER:
749 		if (!alg_is_cipher(csp->csp_cipher_alg))
750 			return (false);
751 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
752 			return (false);
753 		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
754 			if (csp->csp_cipher_klen == 0)
755 				return (false);
756 			if (csp->csp_ivlen == 0)
757 				return (false);
758 		}
759 		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
760 			return (false);
761 		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
762 		    csp->csp_auth_mlen != 0)
763 			return (false);
764 		break;
765 	case CSP_MODE_DIGEST:
766 		if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
767 			return (false);
768 
769 		if (csp->csp_flags & CSP_F_SEPARATE_AAD)
770 			return (false);
771 
772 		/* IV is optional for digests (e.g. GMAC). */
773 		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
774 			return (false);
775 		if (!alg_is_digest(csp->csp_auth_alg))
776 			return (false);
777 
778 		/* Key is optional for BLAKE2 digests. */
779 		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
780 		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
781 			;
782 		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
783 			if (csp->csp_auth_klen == 0)
784 				return (false);
785 		} else {
786 			if (csp->csp_auth_klen != 0)
787 				return (false);
788 		}
789 		if (csp->csp_auth_mlen != 0) {
790 			axf = crypto_auth_hash(csp);
791 			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
792 				return (false);
793 		}
794 		break;
795 	case CSP_MODE_AEAD:
796 		if (!alg_is_aead(csp->csp_cipher_alg))
797 			return (false);
798 		if (csp->csp_cipher_klen == 0)
799 			return (false);
800 		if (csp->csp_ivlen == 0 ||
801 		    csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
802 			return (false);
803 		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
804 			return (false);
805 
806 		/*
807 		 * XXX: Would be nice to have a better way to get this
808 		 * value.
809 		 */
810 		switch (csp->csp_cipher_alg) {
811 		case CRYPTO_AES_NIST_GCM_16:
812 		case CRYPTO_AES_CCM_16:
813 		case CRYPTO_CHACHA20_POLY1305:
814 			if (csp->csp_auth_mlen > 16)
815 				return (false);
816 			break;
817 		}
818 		break;
819 	case CSP_MODE_ETA:
820 		if (!alg_is_cipher(csp->csp_cipher_alg))
821 			return (false);
822 		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
823 			if (csp->csp_cipher_klen == 0)
824 				return (false);
825 			if (csp->csp_ivlen == 0)
826 				return (false);
827 		}
828 		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
829 			return (false);
830 		if (!alg_is_digest(csp->csp_auth_alg))
831 			return (false);
832 
833 		/* Key is optional for BLAKE2 digests. */
834 		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
835 		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
836 			;
837 		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
838 			if (csp->csp_auth_klen == 0)
839 				return (false);
840 		} else {
841 			if (csp->csp_auth_klen != 0)
842 				return (false);
843 		}
844 		if (csp->csp_auth_mlen != 0) {
845 			axf = crypto_auth_hash(csp);
846 			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
847 				return (false);
848 		}
849 		break;
850 	default:
851 		return (false);
852 	}
853 
854 	return (true);
855 }
856 
857 /*
858  * Delete a session after it has been detached from its driver.
859  */
860 static void
861 crypto_deletesession(crypto_session_t cses)
862 {
863 	struct cryptocap *cap;
864 
865 	cap = cses->cap;
866 
867 	zfree(cses, M_CRYPTO_DATA);
868 
869 	CRYPTO_DRIVER_LOCK();
870 	cap->cc_sessions--;
871 	if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
872 		wakeup(cap);
873 	CRYPTO_DRIVER_UNLOCK();
874 	cap_rele(cap);
875 }
876 
877 /*
878  * Create a new session.  The crid argument specifies a crypto
879  * driver to use or constraints on a driver to select (hardware
880  * only, software only, either).  Whatever driver is selected
881  * must be capable of the requested crypto algorithms.
882  */
883 int
884 crypto_newsession(crypto_session_t *cses,
885     const struct crypto_session_params *csp, int crid)
886 {
887 	static uint64_t sessid = 0;
888 	crypto_session_t res;
889 	struct cryptocap *cap;
890 	int err;
891 
892 	if (!check_csp(csp))
893 		return (EINVAL);
894 
895 	res = NULL;
896 
897 	CRYPTO_DRIVER_LOCK();
898 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
899 		/*
900 		 * Use specified driver; verify it is capable.
901 		 */
902 		cap = crypto_checkdriver(crid);
903 		if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
904 			cap = NULL;
905 	} else {
906 		/*
907 		 * No requested driver; select based on crid flags.
908 		 */
909 		cap = crypto_select_driver(csp, crid);
910 	}
911 	if (cap == NULL) {
912 		CRYPTO_DRIVER_UNLOCK();
913 		CRYPTDEB("no driver");
914 		return (EOPNOTSUPP);
915 	}
916 	cap_ref(cap);
917 	cap->cc_sessions++;
918 	CRYPTO_DRIVER_UNLOCK();
919 
920 	/* Allocate a single block for the generic session and driver softc. */
921 	res = malloc(sizeof(*res) + cap->cc_session_size, M_CRYPTO_DATA,
922 	    M_WAITOK | M_ZERO);
923 	res->cap = cap;
924 	res->csp = *csp;
925 	res->id = atomic_fetchadd_64(&sessid, 1);
926 
927 	/* Call the driver initialization routine. */
928 	err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
929 	if (err != 0) {
930 		CRYPTDEB("dev newsession failed: %d", err);
931 		crypto_deletesession(res);
932 		return (err);
933 	}
934 
935 	*cses = res;
936 	return (0);
937 }
938 
939 /*
940  * Delete an existing session (or a reserved session on an unregistered
941  * driver).
942  */
943 void
944 crypto_freesession(crypto_session_t cses)
945 {
946 	struct cryptocap *cap;
947 
948 	if (cses == NULL)
949 		return;
950 
951 	cap = cses->cap;
952 
953 	/* Call the driver cleanup routine, if available. */
954 	CRYPTODEV_FREESESSION(cap->cc_dev, cses);
955 
956 	crypto_deletesession(cses);
957 }
958 
959 /*
960  * Return a new driver id.  Registers a driver with the system so that
961  * it can be probed by subsequent sessions.
962  */
963 int32_t
964 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
965 {
966 	struct cryptocap *cap, **newdrv;
967 	int i;
968 
969 	if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
970 		device_printf(dev,
971 		    "no flags specified when registering driver\n");
972 		return -1;
973 	}
974 
975 	cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
976 	cap->cc_dev = dev;
977 	cap->cc_session_size = sessionsize;
978 	cap->cc_flags = flags;
979 	refcount_init(&cap->cc_refs, 1);
980 
981 	CRYPTO_DRIVER_LOCK();
982 	for (;;) {
983 		for (i = 0; i < crypto_drivers_size; i++) {
984 			if (crypto_drivers[i] == NULL)
985 				break;
986 		}
987 
988 		if (i < crypto_drivers_size)
989 			break;
990 
991 		/* Out of entries, allocate some more. */
992 
993 		if (2 * crypto_drivers_size <= crypto_drivers_size) {
994 			CRYPTO_DRIVER_UNLOCK();
995 			printf("crypto: driver count wraparound!\n");
996 			cap_rele(cap);
997 			return (-1);
998 		}
999 		CRYPTO_DRIVER_UNLOCK();
1000 
1001 		newdrv = malloc(2 * crypto_drivers_size *
1002 		    sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1003 
1004 		CRYPTO_DRIVER_LOCK();
1005 		memcpy(newdrv, crypto_drivers,
1006 		    crypto_drivers_size * sizeof(*crypto_drivers));
1007 
1008 		crypto_drivers_size *= 2;
1009 
1010 		free(crypto_drivers, M_CRYPTO_DATA);
1011 		crypto_drivers = newdrv;
1012 	}
1013 
1014 	cap->cc_hid = i;
1015 	crypto_drivers[i] = cap;
1016 	CRYPTO_DRIVER_UNLOCK();
1017 
1018 	if (bootverbose)
1019 		printf("crypto: assign %s driver id %u, flags 0x%x\n",
1020 		    device_get_nameunit(dev), i, flags);
1021 
1022 	return i;
1023 }
1024 
1025 /*
1026  * Lookup a driver by name.  We match against the full device
1027  * name and unit, and against just the name.  The latter gives
1028  * us a simple widlcarding by device name.  On success return the
1029  * driver/hardware identifier; otherwise return -1.
1030  */
1031 int
1032 crypto_find_driver(const char *match)
1033 {
1034 	struct cryptocap *cap;
1035 	int i, len = strlen(match);
1036 
1037 	CRYPTO_DRIVER_LOCK();
1038 	for (i = 0; i < crypto_drivers_size; i++) {
1039 		if (crypto_drivers[i] == NULL)
1040 			continue;
1041 		cap = crypto_drivers[i];
1042 		if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
1043 		    strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
1044 			CRYPTO_DRIVER_UNLOCK();
1045 			return (i);
1046 		}
1047 	}
1048 	CRYPTO_DRIVER_UNLOCK();
1049 	return (-1);
1050 }
1051 
1052 /*
1053  * Return the device_t for the specified driver or NULL
1054  * if the driver identifier is invalid.
1055  */
1056 device_t
1057 crypto_find_device_byhid(int hid)
1058 {
1059 	struct cryptocap *cap;
1060 	device_t dev;
1061 
1062 	dev = NULL;
1063 	CRYPTO_DRIVER_LOCK();
1064 	cap = crypto_checkdriver(hid);
1065 	if (cap != NULL)
1066 		dev = cap->cc_dev;
1067 	CRYPTO_DRIVER_UNLOCK();
1068 	return (dev);
1069 }
1070 
1071 /*
1072  * Return the device/driver capabilities.
1073  */
1074 int
1075 crypto_getcaps(int hid)
1076 {
1077 	struct cryptocap *cap;
1078 	int flags;
1079 
1080 	flags = 0;
1081 	CRYPTO_DRIVER_LOCK();
1082 	cap = crypto_checkdriver(hid);
1083 	if (cap != NULL)
1084 		flags = cap->cc_flags;
1085 	CRYPTO_DRIVER_UNLOCK();
1086 	return (flags);
1087 }
1088 
1089 /*
1090  * Unregister all algorithms associated with a crypto driver.
1091  * If there are pending sessions using it, leave enough information
1092  * around so that subsequent calls using those sessions will
1093  * correctly detect the driver has been unregistered and reroute
1094  * requests.
1095  */
1096 int
1097 crypto_unregister_all(uint32_t driverid)
1098 {
1099 	struct cryptocap *cap;
1100 
1101 	CRYPTO_DRIVER_LOCK();
1102 	cap = crypto_checkdriver(driverid);
1103 	if (cap == NULL) {
1104 		CRYPTO_DRIVER_UNLOCK();
1105 		return (EINVAL);
1106 	}
1107 
1108 	cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
1109 	crypto_drivers[driverid] = NULL;
1110 
1111 	/*
1112 	 * XXX: This doesn't do anything to kick sessions that
1113 	 * have no pending operations.
1114 	 */
1115 	while (cap->cc_sessions != 0)
1116 		mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
1117 	CRYPTO_DRIVER_UNLOCK();
1118 	cap_rele(cap);
1119 
1120 	return (0);
1121 }
1122 
1123 /*
1124  * Clear blockage on a driver.  The what parameter indicates whether
1125  * the driver is now ready for cryptop's and/or cryptokop's.
1126  */
1127 int
1128 crypto_unblock(uint32_t driverid, int what)
1129 {
1130 	struct cryptocap *cap;
1131 	int err;
1132 
1133 	CRYPTO_Q_LOCK();
1134 	cap = crypto_checkdriver(driverid);
1135 	if (cap != NULL) {
1136 		if (what & CRYPTO_SYMQ)
1137 			cap->cc_qblocked = 0;
1138 		if (crp_sleep)
1139 			wakeup_one(&crp_q);
1140 		err = 0;
1141 	} else
1142 		err = EINVAL;
1143 	CRYPTO_Q_UNLOCK();
1144 
1145 	return err;
1146 }
1147 
1148 size_t
1149 crypto_buffer_len(struct crypto_buffer *cb)
1150 {
1151 	switch (cb->cb_type) {
1152 	case CRYPTO_BUF_CONTIG:
1153 		return (cb->cb_buf_len);
1154 	case CRYPTO_BUF_MBUF:
1155 		if (cb->cb_mbuf->m_flags & M_PKTHDR)
1156 			return (cb->cb_mbuf->m_pkthdr.len);
1157 		return (m_length(cb->cb_mbuf, NULL));
1158 	case CRYPTO_BUF_SINGLE_MBUF:
1159 		return (cb->cb_mbuf->m_len);
1160 	case CRYPTO_BUF_VMPAGE:
1161 		return (cb->cb_vm_page_len);
1162 	case CRYPTO_BUF_UIO:
1163 		return (cb->cb_uio->uio_resid);
1164 	default:
1165 		return (0);
1166 	}
1167 }
1168 
1169 #ifdef INVARIANTS
1170 /* Various sanity checks on crypto requests. */
1171 static void
1172 cb_sanity(struct crypto_buffer *cb, const char *name)
1173 {
1174 	KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
1175 	    ("incoming crp with invalid %s buffer type", name));
1176 	switch (cb->cb_type) {
1177 	case CRYPTO_BUF_CONTIG:
1178 		KASSERT(cb->cb_buf_len >= 0,
1179 		    ("incoming crp with -ve %s buffer length", name));
1180 		break;
1181 	case CRYPTO_BUF_VMPAGE:
1182 		KASSERT(CRYPTO_HAS_VMPAGE,
1183 		    ("incoming crp uses dmap on supported arch"));
1184 		KASSERT(cb->cb_vm_page_len >= 0,
1185 		    ("incoming crp with -ve %s buffer length", name));
1186 		KASSERT(cb->cb_vm_page_offset >= 0,
1187 		    ("incoming crp with -ve %s buffer offset", name));
1188 		KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
1189 		    ("incoming crp with %s buffer offset greater than page size"
1190 		     , name));
1191 		break;
1192 	default:
1193 		break;
1194 	}
1195 }
1196 
1197 static void
1198 crp_sanity(struct cryptop *crp)
1199 {
1200 	struct crypto_session_params *csp;
1201 	struct crypto_buffer *out;
1202 	size_t ilen, len, olen;
1203 
1204 	KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
1205 	KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
1206 	    crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
1207 	    ("incoming crp with invalid output buffer type"));
1208 	KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
1209 	KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
1210 	    ("incoming crp already done"));
1211 
1212 	csp = &crp->crp_session->csp;
1213 	cb_sanity(&crp->crp_buf, "input");
1214 	ilen = crypto_buffer_len(&crp->crp_buf);
1215 	olen = ilen;
1216 	out = NULL;
1217 	if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) {
1218 		if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
1219 			cb_sanity(&crp->crp_obuf, "output");
1220 			out = &crp->crp_obuf;
1221 			olen = crypto_buffer_len(out);
1222 		}
1223 	} else
1224 		KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
1225 		    ("incoming crp with separate output buffer "
1226 		    "but no session support"));
1227 
1228 	switch (csp->csp_mode) {
1229 	case CSP_MODE_COMPRESS:
1230 		KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
1231 		    crp->crp_op == CRYPTO_OP_DECOMPRESS,
1232 		    ("invalid compression op %x", crp->crp_op));
1233 		break;
1234 	case CSP_MODE_CIPHER:
1235 		KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
1236 		    crp->crp_op == CRYPTO_OP_DECRYPT,
1237 		    ("invalid cipher op %x", crp->crp_op));
1238 		break;
1239 	case CSP_MODE_DIGEST:
1240 		KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
1241 		    crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
1242 		    ("invalid digest op %x", crp->crp_op));
1243 		break;
1244 	case CSP_MODE_AEAD:
1245 		KASSERT(crp->crp_op ==
1246 		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1247 		    crp->crp_op ==
1248 		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1249 		    ("invalid AEAD op %x", crp->crp_op));
1250 		KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1251 		    ("AEAD without a separate IV"));
1252 		break;
1253 	case CSP_MODE_ETA:
1254 		KASSERT(crp->crp_op ==
1255 		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1256 		    crp->crp_op ==
1257 		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1258 		    ("invalid ETA op %x", crp->crp_op));
1259 		break;
1260 	}
1261 	if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1262 		if (crp->crp_aad == NULL) {
1263 			KASSERT(crp->crp_aad_start == 0 ||
1264 			    crp->crp_aad_start < ilen,
1265 			    ("invalid AAD start"));
1266 			KASSERT(crp->crp_aad_length != 0 ||
1267 			    crp->crp_aad_start == 0,
1268 			    ("AAD with zero length and non-zero start"));
1269 			KASSERT(crp->crp_aad_length == 0 ||
1270 			    crp->crp_aad_start + crp->crp_aad_length <= ilen,
1271 			    ("AAD outside input length"));
1272 		} else {
1273 			KASSERT(csp->csp_flags & CSP_F_SEPARATE_AAD,
1274 			    ("session doesn't support separate AAD buffer"));
1275 			KASSERT(crp->crp_aad_start == 0,
1276 			    ("separate AAD buffer with non-zero AAD start"));
1277 			KASSERT(crp->crp_aad_length != 0,
1278 			    ("separate AAD buffer with zero length"));
1279 		}
1280 	} else {
1281 		KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 &&
1282 		    crp->crp_aad_length == 0,
1283 		    ("AAD region in request not supporting AAD"));
1284 	}
1285 	if (csp->csp_ivlen == 0) {
1286 		KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0,
1287 		    ("IV_SEPARATE set when IV isn't used"));
1288 		KASSERT(crp->crp_iv_start == 0,
1289 		    ("crp_iv_start set when IV isn't used"));
1290 	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
1291 		KASSERT(crp->crp_iv_start == 0,
1292 		    ("IV_SEPARATE used with non-zero IV start"));
1293 	} else {
1294 		KASSERT(crp->crp_iv_start < ilen,
1295 		    ("invalid IV start"));
1296 		KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
1297 		    ("IV outside buffer length"));
1298 	}
1299 	/* XXX: payload_start of 0 should always be < ilen? */
1300 	KASSERT(crp->crp_payload_start == 0 ||
1301 	    crp->crp_payload_start < ilen,
1302 	    ("invalid payload start"));
1303 	KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
1304 	    ilen, ("payload outside input buffer"));
1305 	if (out == NULL) {
1306 		KASSERT(crp->crp_payload_output_start == 0,
1307 		    ("payload output start non-zero without output buffer"));
1308 	} else {
1309 		KASSERT(crp->crp_payload_output_start < olen,
1310 		    ("invalid payload output start"));
1311 		KASSERT(crp->crp_payload_output_start +
1312 		    crp->crp_payload_length <= olen,
1313 		    ("payload outside output buffer"));
1314 	}
1315 	if (csp->csp_mode == CSP_MODE_DIGEST ||
1316 	    csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1317 		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
1318 			len = ilen;
1319 		else
1320 			len = olen;
1321 		KASSERT(crp->crp_digest_start == 0 ||
1322 		    crp->crp_digest_start < len,
1323 		    ("invalid digest start"));
1324 		/* XXX: For the mlen == 0 case this check isn't perfect. */
1325 		KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
1326 		    ("digest outside buffer"));
1327 	} else {
1328 		KASSERT(crp->crp_digest_start == 0,
1329 		    ("non-zero digest start for request without a digest"));
1330 	}
1331 	if (csp->csp_cipher_klen != 0)
1332 		KASSERT(csp->csp_cipher_key != NULL ||
1333 		    crp->crp_cipher_key != NULL,
1334 		    ("cipher request without a key"));
1335 	if (csp->csp_auth_klen != 0)
1336 		KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
1337 		    ("auth request without a key"));
1338 	KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
1339 }
1340 #endif
1341 
1342 static int
1343 crypto_dispatch_one(struct cryptop *crp, int hint)
1344 {
1345 	struct cryptocap *cap;
1346 	int result;
1347 
1348 #ifdef INVARIANTS
1349 	crp_sanity(crp);
1350 #endif
1351 	CRYPTOSTAT_INC(cs_ops);
1352 
1353 	crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1354 
1355 	/*
1356 	 * Caller marked the request to be processed immediately; dispatch it
1357 	 * directly to the driver unless the driver is currently blocked, in
1358 	 * which case it is queued for deferred dispatch.
1359 	 */
1360 	cap = crp->crp_session->cap;
1361 	if (!atomic_load_int(&cap->cc_qblocked)) {
1362 		result = crypto_invoke(cap, crp, hint);
1363 		if (result != ERESTART)
1364 			return (result);
1365 
1366 		/*
1367 		 * The driver ran out of resources, put the request on the
1368 		 * queue.
1369 		 */
1370 	}
1371 	crypto_batch_enqueue(crp);
1372 	return (0);
1373 }
1374 
1375 int
1376 crypto_dispatch(struct cryptop *crp)
1377 {
1378 	return (crypto_dispatch_one(crp, 0));
1379 }
1380 
1381 int
1382 crypto_dispatch_async(struct cryptop *crp, int flags)
1383 {
1384 	struct crypto_ret_worker *ret_worker;
1385 
1386 	if (!CRYPTO_SESS_SYNC(crp->crp_session)) {
1387 		/*
1388 		 * The driver issues completions asynchonously, don't bother
1389 		 * deferring dispatch to a worker thread.
1390 		 */
1391 		return (crypto_dispatch(crp));
1392 	}
1393 
1394 #ifdef INVARIANTS
1395 	crp_sanity(crp);
1396 #endif
1397 	CRYPTOSTAT_INC(cs_ops);
1398 
1399 	crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1400 	if ((flags & CRYPTO_ASYNC_ORDERED) != 0) {
1401 		crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED;
1402 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1403 		CRYPTO_RETW_LOCK(ret_worker);
1404 		crp->crp_seq = ret_worker->reorder_ops++;
1405 		CRYPTO_RETW_UNLOCK(ret_worker);
1406 	}
1407 	TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
1408 	taskqueue_enqueue(crypto_tq, &crp->crp_task);
1409 	return (0);
1410 }
1411 
1412 void
1413 crypto_dispatch_batch(struct cryptopq *crpq, int flags)
1414 {
1415 	struct cryptop *crp;
1416 	int hint;
1417 
1418 	while ((crp = TAILQ_FIRST(crpq)) != NULL) {
1419 		hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0;
1420 		TAILQ_REMOVE(crpq, crp, crp_next);
1421 		if (crypto_dispatch_one(crp, hint) != 0)
1422 			crypto_batch_enqueue(crp);
1423 	}
1424 }
1425 
1426 static void
1427 crypto_batch_enqueue(struct cryptop *crp)
1428 {
1429 
1430 	CRYPTO_Q_LOCK();
1431 	TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
1432 	if (crp_sleep)
1433 		wakeup_one(&crp_q);
1434 	CRYPTO_Q_UNLOCK();
1435 }
1436 
1437 static void
1438 crypto_task_invoke(void *ctx, int pending)
1439 {
1440 	struct cryptocap *cap;
1441 	struct cryptop *crp;
1442 	int result;
1443 
1444 	crp = (struct cryptop *)ctx;
1445 	cap = crp->crp_session->cap;
1446 	result = crypto_invoke(cap, crp, 0);
1447 	if (result == ERESTART)
1448 		crypto_batch_enqueue(crp);
1449 }
1450 
1451 /*
1452  * Dispatch a crypto request to the appropriate crypto devices.
1453  */
1454 static int
1455 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1456 {
1457 
1458 	KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1459 	KASSERT(crp->crp_callback != NULL,
1460 	    ("%s: crp->crp_callback == NULL", __func__));
1461 	KASSERT(crp->crp_session != NULL,
1462 	    ("%s: crp->crp_session == NULL", __func__));
1463 
1464 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1465 		struct crypto_session_params csp;
1466 		crypto_session_t nses;
1467 
1468 		/*
1469 		 * Driver has unregistered; migrate the session and return
1470 		 * an error to the caller so they'll resubmit the op.
1471 		 *
1472 		 * XXX: What if there are more already queued requests for this
1473 		 *      session?
1474 		 *
1475 		 * XXX: Real solution is to make sessions refcounted
1476 		 * and force callers to hold a reference when
1477 		 * assigning to crp_session.  Could maybe change
1478 		 * crypto_getreq to accept a session pointer to make
1479 		 * that work.  Alternatively, we could abandon the
1480 		 * notion of rewriting crp_session in requests forcing
1481 		 * the caller to deal with allocating a new session.
1482 		 * Perhaps provide a method to allow a crp's session to
1483 		 * be swapped that callers could use.
1484 		 */
1485 		csp = crp->crp_session->csp;
1486 		crypto_freesession(crp->crp_session);
1487 
1488 		/*
1489 		 * XXX: Key pointers may no longer be valid.  If we
1490 		 * really want to support this we need to define the
1491 		 * KPI such that 'csp' is required to be valid for the
1492 		 * duration of a session by the caller perhaps.
1493 		 *
1494 		 * XXX: If the keys have been changed this will reuse
1495 		 * the old keys.  This probably suggests making
1496 		 * rekeying more explicit and updating the key
1497 		 * pointers in 'csp' when the keys change.
1498 		 */
1499 		if (crypto_newsession(&nses, &csp,
1500 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1501 			crp->crp_session = nses;
1502 
1503 		crp->crp_etype = EAGAIN;
1504 		crypto_done(crp);
1505 		return 0;
1506 	} else {
1507 		/*
1508 		 * Invoke the driver to process the request.
1509 		 */
1510 		return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1511 	}
1512 }
1513 
1514 void
1515 crypto_destroyreq(struct cryptop *crp)
1516 {
1517 #ifdef DIAGNOSTIC
1518 	{
1519 		struct cryptop *crp2;
1520 		struct crypto_ret_worker *ret_worker;
1521 
1522 		CRYPTO_Q_LOCK();
1523 		TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1524 			KASSERT(crp2 != crp,
1525 			    ("Freeing cryptop from the crypto queue (%p).",
1526 			    crp));
1527 		}
1528 		CRYPTO_Q_UNLOCK();
1529 
1530 		FOREACH_CRYPTO_RETW(ret_worker) {
1531 			CRYPTO_RETW_LOCK(ret_worker);
1532 			TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
1533 				KASSERT(crp2 != crp,
1534 				    ("Freeing cryptop from the return queue (%p).",
1535 				    crp));
1536 			}
1537 			CRYPTO_RETW_UNLOCK(ret_worker);
1538 		}
1539 	}
1540 #endif
1541 }
1542 
1543 void
1544 crypto_freereq(struct cryptop *crp)
1545 {
1546 	if (crp == NULL)
1547 		return;
1548 
1549 	crypto_destroyreq(crp);
1550 	uma_zfree(cryptop_zone, crp);
1551 }
1552 
1553 static void
1554 _crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1555 {
1556 	crp->crp_session = cses;
1557 }
1558 
1559 void
1560 crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1561 {
1562 	memset(crp, 0, sizeof(*crp));
1563 	_crypto_initreq(crp, cses);
1564 }
1565 
1566 struct cryptop *
1567 crypto_getreq(crypto_session_t cses, int how)
1568 {
1569 	struct cryptop *crp;
1570 
1571 	MPASS(how == M_WAITOK || how == M_NOWAIT);
1572 	crp = uma_zalloc(cryptop_zone, how | M_ZERO);
1573 	if (crp != NULL)
1574 		_crypto_initreq(crp, cses);
1575 	return (crp);
1576 }
1577 
1578 /*
1579  * Invoke the callback on behalf of the driver.
1580  */
1581 void
1582 crypto_done(struct cryptop *crp)
1583 {
1584 	KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1585 		("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1586 	crp->crp_flags |= CRYPTO_F_DONE;
1587 	if (crp->crp_etype != 0)
1588 		CRYPTOSTAT_INC(cs_errs);
1589 
1590 	/*
1591 	 * CBIMM means unconditionally do the callback immediately;
1592 	 * CBIFSYNC means do the callback immediately only if the
1593 	 * operation was done synchronously.  Both are used to avoid
1594 	 * doing extraneous context switches; the latter is mostly
1595 	 * used with the software crypto driver.
1596 	 */
1597 	if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 &&
1598 	    ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 ||
1599 	    ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 &&
1600 	    CRYPTO_SESS_SYNC(crp->crp_session)))) {
1601 		/*
1602 		 * Do the callback directly.  This is ok when the
1603 		 * callback routine does very little (e.g. the
1604 		 * /dev/crypto callback method just does a wakeup).
1605 		 */
1606 		crp->crp_callback(crp);
1607 	} else {
1608 		struct crypto_ret_worker *ret_worker;
1609 		bool wake;
1610 
1611 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1612 
1613 		/*
1614 		 * Normal case; queue the callback for the thread.
1615 		 */
1616 		CRYPTO_RETW_LOCK(ret_worker);
1617 		if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) {
1618 			struct cryptop *tmp;
1619 
1620 			TAILQ_FOREACH_REVERSE(tmp,
1621 			    &ret_worker->crp_ordered_ret_q, cryptop_q,
1622 			    crp_next) {
1623 				if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1624 					TAILQ_INSERT_AFTER(
1625 					    &ret_worker->crp_ordered_ret_q, tmp,
1626 					    crp, crp_next);
1627 					break;
1628 				}
1629 			}
1630 			if (tmp == NULL) {
1631 				TAILQ_INSERT_HEAD(
1632 				    &ret_worker->crp_ordered_ret_q, crp,
1633 				    crp_next);
1634 			}
1635 
1636 			wake = crp->crp_seq == ret_worker->reorder_cur_seq;
1637 		} else {
1638 			wake = TAILQ_EMPTY(&ret_worker->crp_ret_q);
1639 			TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp,
1640 			    crp_next);
1641 		}
1642 
1643 		if (wake)
1644 			wakeup_one(&ret_worker->crp_ret_q);	/* shared wait channel */
1645 		CRYPTO_RETW_UNLOCK(ret_worker);
1646 	}
1647 }
1648 
1649 /*
1650  * Terminate a thread at module unload.  The process that
1651  * initiated this is waiting for us to signal that we're gone;
1652  * wake it up and exit.  We use the driver table lock to insure
1653  * we don't do the wakeup before they're waiting.  There is no
1654  * race here because the waiter sleeps on the proc lock for the
1655  * thread so it gets notified at the right time because of an
1656  * extra wakeup that's done in exit1().
1657  */
1658 static void
1659 crypto_finis(void *chan)
1660 {
1661 	CRYPTO_DRIVER_LOCK();
1662 	wakeup_one(chan);
1663 	CRYPTO_DRIVER_UNLOCK();
1664 	kproc_exit(0);
1665 }
1666 
1667 /*
1668  * Crypto thread, dispatches crypto requests.
1669  */
1670 static void
1671 crypto_proc(void)
1672 {
1673 	struct cryptop *crp, *submit;
1674 	struct cryptocap *cap;
1675 	int result, hint;
1676 
1677 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1678 	fpu_kern_thread(FPU_KERN_NORMAL);
1679 #endif
1680 
1681 	CRYPTO_Q_LOCK();
1682 	for (;;) {
1683 		/*
1684 		 * Find the first element in the queue that can be
1685 		 * processed and look-ahead to see if multiple ops
1686 		 * are ready for the same driver.
1687 		 */
1688 		submit = NULL;
1689 		hint = 0;
1690 		TAILQ_FOREACH(crp, &crp_q, crp_next) {
1691 			cap = crp->crp_session->cap;
1692 			/*
1693 			 * Driver cannot disappeared when there is an active
1694 			 * session.
1695 			 */
1696 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1697 			    __func__, __LINE__));
1698 			if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1699 				/* Op needs to be migrated, process it. */
1700 				if (submit == NULL)
1701 					submit = crp;
1702 				break;
1703 			}
1704 			if (!cap->cc_qblocked) {
1705 				if (submit != NULL) {
1706 					/*
1707 					 * We stop on finding another op,
1708 					 * regardless whether its for the same
1709 					 * driver or not.  We could keep
1710 					 * searching the queue but it might be
1711 					 * better to just use a per-driver
1712 					 * queue instead.
1713 					 */
1714 					if (submit->crp_session->cap == cap)
1715 						hint = CRYPTO_HINT_MORE;
1716 				} else {
1717 					submit = crp;
1718 				}
1719 				break;
1720 			}
1721 		}
1722 		if (submit != NULL) {
1723 			TAILQ_REMOVE(&crp_q, submit, crp_next);
1724 			cap = submit->crp_session->cap;
1725 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1726 			    __func__, __LINE__));
1727 			CRYPTO_Q_UNLOCK();
1728 			result = crypto_invoke(cap, submit, hint);
1729 			CRYPTO_Q_LOCK();
1730 			if (result == ERESTART) {
1731 				/*
1732 				 * The driver ran out of resources, mark the
1733 				 * driver ``blocked'' for cryptop's and put
1734 				 * the request back in the queue.  It would
1735 				 * best to put the request back where we got
1736 				 * it but that's hard so for now we put it
1737 				 * at the front.  This should be ok; putting
1738 				 * it at the end does not work.
1739 				 */
1740 				cap->cc_qblocked = 1;
1741 				TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1742 				CRYPTOSTAT_INC(cs_blocks);
1743 			}
1744 		} else {
1745 			/*
1746 			 * Nothing more to be processed.  Sleep until we're
1747 			 * woken because there are more ops to process.
1748 			 * This happens either by submission or by a driver
1749 			 * becoming unblocked and notifying us through
1750 			 * crypto_unblock.  Note that when we wakeup we
1751 			 * start processing each queue again from the
1752 			 * front. It's not clear that it's important to
1753 			 * preserve this ordering since ops may finish
1754 			 * out of order if dispatched to different devices
1755 			 * and some become blocked while others do not.
1756 			 */
1757 			crp_sleep = 1;
1758 			msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
1759 			crp_sleep = 0;
1760 			if (cryptoproc == NULL)
1761 				break;
1762 			CRYPTOSTAT_INC(cs_intrs);
1763 		}
1764 	}
1765 	CRYPTO_Q_UNLOCK();
1766 
1767 	crypto_finis(&crp_q);
1768 }
1769 
1770 /*
1771  * Crypto returns thread, does callbacks for processed crypto requests.
1772  * Callbacks are done here, rather than in the crypto drivers, because
1773  * callbacks typically are expensive and would slow interrupt handling.
1774  */
1775 static void
1776 crypto_ret_proc(struct crypto_ret_worker *ret_worker)
1777 {
1778 	struct cryptop *crpt;
1779 
1780 	CRYPTO_RETW_LOCK(ret_worker);
1781 	for (;;) {
1782 		/* Harvest return q's for completed ops */
1783 		crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
1784 		if (crpt != NULL) {
1785 			if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
1786 				TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
1787 				ret_worker->reorder_cur_seq++;
1788 			} else {
1789 				crpt = NULL;
1790 			}
1791 		}
1792 
1793 		if (crpt == NULL) {
1794 			crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
1795 			if (crpt != NULL)
1796 				TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
1797 		}
1798 
1799 		if (crpt != NULL) {
1800 			CRYPTO_RETW_UNLOCK(ret_worker);
1801 			/*
1802 			 * Run callbacks unlocked.
1803 			 */
1804 			if (crpt != NULL)
1805 				crpt->crp_callback(crpt);
1806 			CRYPTO_RETW_LOCK(ret_worker);
1807 		} else {
1808 			/*
1809 			 * Nothing more to be processed.  Sleep until we're
1810 			 * woken because there are more returns to process.
1811 			 */
1812 			msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
1813 				"crypto_ret_wait", 0);
1814 			if (ret_worker->cryptoretproc == NULL)
1815 				break;
1816 			CRYPTOSTAT_INC(cs_rets);
1817 		}
1818 	}
1819 	CRYPTO_RETW_UNLOCK(ret_worker);
1820 
1821 	crypto_finis(&ret_worker->crp_ret_q);
1822 }
1823 
1824 #ifdef DDB
1825 static void
1826 db_show_drivers(void)
1827 {
1828 	int hid;
1829 
1830 	db_printf("%12s %4s %8s %2s\n"
1831 		, "Device"
1832 		, "Ses"
1833 		, "Flags"
1834 		, "QB"
1835 	);
1836 	for (hid = 0; hid < crypto_drivers_size; hid++) {
1837 		const struct cryptocap *cap = crypto_drivers[hid];
1838 		if (cap == NULL)
1839 			continue;
1840 		db_printf("%-12s %4u %08x %2u\n"
1841 		    , device_get_nameunit(cap->cc_dev)
1842 		    , cap->cc_sessions
1843 		    , cap->cc_flags
1844 		    , cap->cc_qblocked
1845 		);
1846 	}
1847 }
1848 
1849 DB_SHOW_COMMAND(crypto, db_show_crypto)
1850 {
1851 	struct cryptop *crp;
1852 	struct crypto_ret_worker *ret_worker;
1853 
1854 	db_show_drivers();
1855 	db_printf("\n");
1856 
1857 	db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
1858 	    "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
1859 	    "Device", "Callback");
1860 	TAILQ_FOREACH(crp, &crp_q, crp_next) {
1861 		db_printf("%4u %08x %4u %4u %04x %8p %8p\n"
1862 		    , crp->crp_session->cap->cc_hid
1863 		    , (int) crypto_ses2caps(crp->crp_session)
1864 		    , crp->crp_olen
1865 		    , crp->crp_etype
1866 		    , crp->crp_flags
1867 		    , device_get_nameunit(crp->crp_session->cap->cc_dev)
1868 		    , crp->crp_callback
1869 		);
1870 	}
1871 	FOREACH_CRYPTO_RETW(ret_worker) {
1872 		db_printf("\n%8s %4s %4s %4s %8s\n",
1873 		    "ret_worker", "HID", "Etype", "Flags", "Callback");
1874 		if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
1875 			TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
1876 				db_printf("%8td %4u %4u %04x %8p\n"
1877 				    , CRYPTO_RETW_ID(ret_worker)
1878 				    , crp->crp_session->cap->cc_hid
1879 				    , crp->crp_etype
1880 				    , crp->crp_flags
1881 				    , crp->crp_callback
1882 				);
1883 			}
1884 		}
1885 	}
1886 }
1887 #endif
1888 
1889 int crypto_modevent(module_t mod, int type, void *unused);
1890 
1891 /*
1892  * Initialization code, both for static and dynamic loading.
1893  * Note this is not invoked with the usual MODULE_DECLARE
1894  * mechanism but instead is listed as a dependency by the
1895  * cryptosoft driver.  This guarantees proper ordering of
1896  * calls on module load/unload.
1897  */
1898 int
1899 crypto_modevent(module_t mod, int type, void *unused)
1900 {
1901 	int error = EINVAL;
1902 
1903 	switch (type) {
1904 	case MOD_LOAD:
1905 		error = crypto_init();
1906 		if (error == 0 && bootverbose)
1907 			printf("crypto: <crypto core>\n");
1908 		break;
1909 	case MOD_UNLOAD:
1910 		/*XXX disallow if active sessions */
1911 		error = 0;
1912 		crypto_destroy();
1913 		return 0;
1914 	}
1915 	return error;
1916 }
1917 MODULE_VERSION(crypto, 1);
1918 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
1919