xref: /freebsd/sys/opencrypto/crypto.c (revision 1e413cf93298b5b97441a21d9a50fdcd0ee9945e)
1 /*-
2  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  */
24 
25 #include <sys/cdefs.h>
26 __FBSDID("$FreeBSD$");
27 
28 /*
29  * Cryptographic Subsystem.
30  *
31  * This code is derived from the Openbsd Cryptographic Framework (OCF)
32  * that has the copyright shown below.  Very little of the original
33  * code remains.
34  */
35 
36 /*-
37  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
38  *
39  * This code was written by Angelos D. Keromytis in Athens, Greece, in
40  * February 2000. Network Security Technologies Inc. (NSTI) kindly
41  * supported the development of this code.
42  *
43  * Copyright (c) 2000, 2001 Angelos D. Keromytis
44  *
45  * Permission to use, copy, and modify this software with or without fee
46  * is hereby granted, provided that this entire notice is included in
47  * all source code copies of any software which is or includes a copy or
48  * modification of this software.
49  *
50  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
51  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
52  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
53  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
54  * PURPOSE.
55  */
56 
57 #define	CRYPTO_TIMING				/* enable timing support */
58 
59 #include "opt_ddb.h"
60 
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/eventhandler.h>
64 #include <sys/kernel.h>
65 #include <sys/kthread.h>
66 #include <sys/lock.h>
67 #include <sys/module.h>
68 #include <sys/mutex.h>
69 #include <sys/malloc.h>
70 #include <sys/proc.h>
71 #include <sys/sysctl.h>
72 
73 #include <ddb/ddb.h>
74 
75 #include <vm/uma.h>
76 #include <opencrypto/cryptodev.h>
77 #include <opencrypto/xform.h>			/* XXX for M_XDATA */
78 
79 #include <sys/kobj.h>
80 #include <sys/bus.h>
81 #include "cryptodev_if.h"
82 
83 /*
84  * Crypto drivers register themselves by allocating a slot in the
85  * crypto_drivers table with crypto_get_driverid() and then registering
86  * each algorithm they support with crypto_register() and crypto_kregister().
87  */
88 static	struct mtx crypto_drivers_mtx;		/* lock on driver table */
89 #define	CRYPTO_DRIVER_LOCK()	mtx_lock(&crypto_drivers_mtx)
90 #define	CRYPTO_DRIVER_UNLOCK()	mtx_unlock(&crypto_drivers_mtx)
91 #define	CRYPTO_DRIVER_ASSERT()	mtx_assert(&crypto_drivers_mtx, MA_OWNED)
92 
93 /*
94  * Crypto device/driver capabilities structure.
95  *
96  * Synchronization:
97  * (d) - protected by CRYPTO_DRIVER_LOCK()
98  * (q) - protected by CRYPTO_Q_LOCK()
99  * Not tagged fields are read-only.
100  */
101 struct cryptocap {
102 	device_t	cc_dev;			/* (d) device/driver */
103 	u_int32_t	cc_sessions;		/* (d) # of sessions */
104 	u_int32_t	cc_koperations;		/* (d) # os asym operations */
105 	/*
106 	 * Largest possible operator length (in bits) for each type of
107 	 * encryption algorithm. XXX not used
108 	 */
109 	u_int16_t	cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1];
110 	u_int8_t	cc_alg[CRYPTO_ALGORITHM_MAX + 1];
111 	u_int8_t	cc_kalg[CRK_ALGORITHM_MAX + 1];
112 
113 	int		cc_flags;		/* (d) flags */
114 #define CRYPTOCAP_F_CLEANUP	0x80000000	/* needs resource cleanup */
115 	int		cc_qblocked;		/* (q) symmetric q blocked */
116 	int		cc_kqblocked;		/* (q) asymmetric q blocked */
117 };
118 static	struct cryptocap *crypto_drivers = NULL;
119 static	int crypto_drivers_num = 0;
120 
121 /*
122  * There are two queues for crypto requests; one for symmetric (e.g.
123  * cipher) operations and one for asymmetric (e.g. MOD)operations.
124  * A single mutex is used to lock access to both queues.  We could
125  * have one per-queue but having one simplifies handling of block/unblock
126  * operations.
127  */
128 static	int crp_sleep = 0;
129 static	TAILQ_HEAD(,cryptop) crp_q;		/* request queues */
130 static	TAILQ_HEAD(,cryptkop) crp_kq;
131 static	struct mtx crypto_q_mtx;
132 #define	CRYPTO_Q_LOCK()		mtx_lock(&crypto_q_mtx)
133 #define	CRYPTO_Q_UNLOCK()	mtx_unlock(&crypto_q_mtx)
134 
135 /*
136  * There are two queues for processing completed crypto requests; one
137  * for the symmetric and one for the asymmetric ops.  We only need one
138  * but have two to avoid type futzing (cryptop vs. cryptkop).  A single
139  * mutex is used to lock access to both queues.  Note that this lock
140  * must be separate from the lock on request queues to insure driver
141  * callbacks don't generate lock order reversals.
142  */
143 static	TAILQ_HEAD(,cryptop) crp_ret_q;		/* callback queues */
144 static	TAILQ_HEAD(,cryptkop) crp_ret_kq;
145 static	struct mtx crypto_ret_q_mtx;
146 #define	CRYPTO_RETQ_LOCK()	mtx_lock(&crypto_ret_q_mtx)
147 #define	CRYPTO_RETQ_UNLOCK()	mtx_unlock(&crypto_ret_q_mtx)
148 #define	CRYPTO_RETQ_EMPTY()	(TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq))
149 
150 static	uma_zone_t cryptop_zone;
151 static	uma_zone_t cryptodesc_zone;
152 
153 int	crypto_userasymcrypto = 1;	/* userland may do asym crypto reqs */
154 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
155 	   &crypto_userasymcrypto, 0,
156 	   "Enable/disable user-mode access to asymmetric crypto support");
157 int	crypto_devallowsoft = 0;	/* only use hardware crypto for asym */
158 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
159 	   &crypto_devallowsoft, 0,
160 	   "Enable/disable use of software asym crypto support");
161 
162 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
163 
164 static	void crypto_proc(void);
165 static	struct proc *cryptoproc;
166 static	void crypto_ret_proc(void);
167 static	struct proc *cryptoretproc;
168 static	void crypto_destroy(void);
169 static	int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
170 static	int crypto_kinvoke(struct cryptkop *krp, int flags);
171 
172 static	struct cryptostats cryptostats;
173 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
174 	    cryptostats, "Crypto system statistics");
175 
176 #ifdef CRYPTO_TIMING
177 static	int crypto_timing = 0;
178 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
179 	   &crypto_timing, 0, "Enable/disable crypto timing support");
180 #endif
181 
182 static int
183 crypto_init(void)
184 {
185 	int error;
186 
187 	mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
188 		MTX_DEF|MTX_QUIET);
189 
190 	TAILQ_INIT(&crp_q);
191 	TAILQ_INIT(&crp_kq);
192 	mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
193 
194 	TAILQ_INIT(&crp_ret_q);
195 	TAILQ_INIT(&crp_ret_kq);
196 	mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF);
197 
198 	cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
199 				    0, 0, 0, 0,
200 				    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
201 	cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
202 				    0, 0, 0, 0,
203 				    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
204 	if (cryptodesc_zone == NULL || cryptop_zone == NULL) {
205 		printf("crypto_init: cannot setup crypto zones\n");
206 		error = ENOMEM;
207 		goto bad;
208 	}
209 
210 	crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
211 	crypto_drivers = malloc(crypto_drivers_num *
212 	    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
213 	if (crypto_drivers == NULL) {
214 		printf("crypto_init: cannot setup crypto drivers\n");
215 		error = ENOMEM;
216 		goto bad;
217 	}
218 
219 	error = kproc_create((void (*)(void *)) crypto_proc, NULL,
220 		    &cryptoproc, 0, 0, "crypto");
221 	if (error) {
222 		printf("crypto_init: cannot start crypto thread; error %d",
223 			error);
224 		goto bad;
225 	}
226 
227 	error = kproc_create((void (*)(void *)) crypto_ret_proc, NULL,
228 		    &cryptoretproc, 0, 0, "crypto returns");
229 	if (error) {
230 		printf("crypto_init: cannot start cryptoret thread; error %d",
231 			error);
232 		goto bad;
233 	}
234 	return 0;
235 bad:
236 	crypto_destroy();
237 	return error;
238 }
239 
240 /*
241  * Signal a crypto thread to terminate.  We use the driver
242  * table lock to synchronize the sleep/wakeups so that we
243  * are sure the threads have terminated before we release
244  * the data structures they use.  See crypto_finis below
245  * for the other half of this song-and-dance.
246  */
247 static void
248 crypto_terminate(struct proc **pp, void *q)
249 {
250 	struct proc *p;
251 
252 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
253 	p = *pp;
254 	*pp = NULL;
255 	if (p) {
256 		wakeup_one(q);
257 		PROC_LOCK(p);		/* NB: insure we don't miss wakeup */
258 		CRYPTO_DRIVER_UNLOCK();	/* let crypto_finis progress */
259 		msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
260 		PROC_UNLOCK(p);
261 		CRYPTO_DRIVER_LOCK();
262 	}
263 }
264 
265 static void
266 crypto_destroy(void)
267 {
268 	/*
269 	 * Terminate any crypto threads.
270 	 */
271 	CRYPTO_DRIVER_LOCK();
272 	crypto_terminate(&cryptoproc, &crp_q);
273 	crypto_terminate(&cryptoretproc, &crp_ret_q);
274 	CRYPTO_DRIVER_UNLOCK();
275 
276 	/* XXX flush queues??? */
277 
278 	/*
279 	 * Reclaim dynamically allocated resources.
280 	 */
281 	if (crypto_drivers != NULL)
282 		free(crypto_drivers, M_CRYPTO_DATA);
283 
284 	if (cryptodesc_zone != NULL)
285 		uma_zdestroy(cryptodesc_zone);
286 	if (cryptop_zone != NULL)
287 		uma_zdestroy(cryptop_zone);
288 	mtx_destroy(&crypto_q_mtx);
289 	mtx_destroy(&crypto_ret_q_mtx);
290 	mtx_destroy(&crypto_drivers_mtx);
291 }
292 
293 static struct cryptocap *
294 crypto_checkdriver(u_int32_t hid)
295 {
296 	if (crypto_drivers == NULL)
297 		return NULL;
298 	return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
299 }
300 
301 /*
302  * Compare a driver's list of supported algorithms against another
303  * list; return non-zero if all algorithms are supported.
304  */
305 static int
306 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri)
307 {
308 	const struct cryptoini *cr;
309 
310 	/* See if all the algorithms are supported. */
311 	for (cr = cri; cr; cr = cr->cri_next)
312 		if (cap->cc_alg[cr->cri_alg] == 0)
313 			return 0;
314 	return 1;
315 }
316 
317 /*
318  * Select a driver for a new session that supports the specified
319  * algorithms and, optionally, is constrained according to the flags.
320  * The algorithm we use here is pretty stupid; just use the
321  * first driver that supports all the algorithms we need. If there
322  * are multiple drivers we choose the driver with the fewest active
323  * sessions.  We prefer hardware-backed drivers to software ones.
324  *
325  * XXX We need more smarts here (in real life too, but that's
326  * XXX another story altogether).
327  */
328 static struct cryptocap *
329 crypto_select_driver(const struct cryptoini *cri, int flags)
330 {
331 	struct cryptocap *cap, *best;
332 	int match, hid;
333 
334 	CRYPTO_DRIVER_ASSERT();
335 
336 	/*
337 	 * Look first for hardware crypto devices if permitted.
338 	 */
339 	if (flags & CRYPTOCAP_F_HARDWARE)
340 		match = CRYPTOCAP_F_HARDWARE;
341 	else
342 		match = CRYPTOCAP_F_SOFTWARE;
343 	best = NULL;
344 again:
345 	for (hid = 0; hid < crypto_drivers_num; hid++) {
346 		cap = &crypto_drivers[hid];
347 		/*
348 		 * If it's not initialized, is in the process of
349 		 * going away, or is not appropriate (hardware
350 		 * or software based on match), then skip.
351 		 */
352 		if (cap->cc_dev == NULL ||
353 		    (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
354 		    (cap->cc_flags & match) == 0)
355 			continue;
356 
357 		/* verify all the algorithms are supported. */
358 		if (driver_suitable(cap, cri)) {
359 			if (best == NULL ||
360 			    cap->cc_sessions < best->cc_sessions)
361 				best = cap;
362 		}
363 	}
364 	if (best != NULL)
365 		return best;
366 	if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
367 		/* sort of an Algol 68-style for loop */
368 		match = CRYPTOCAP_F_SOFTWARE;
369 		goto again;
370 	}
371 	return best;
372 }
373 
374 /*
375  * Create a new session.  The crid argument specifies a crypto
376  * driver to use or constraints on a driver to select (hardware
377  * only, software only, either).  Whatever driver is selected
378  * must be capable of the requested crypto algorithms.
379  */
380 int
381 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid)
382 {
383 	struct cryptocap *cap;
384 	u_int32_t hid, lid;
385 	int err;
386 
387 	CRYPTO_DRIVER_LOCK();
388 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
389 		/*
390 		 * Use specified driver; verify it is capable.
391 		 */
392 		cap = crypto_checkdriver(crid);
393 		if (cap != NULL && !driver_suitable(cap, cri))
394 			cap = NULL;
395 	} else {
396 		/*
397 		 * No requested driver; select based on crid flags.
398 		 */
399 		cap = crypto_select_driver(cri, crid);
400 		/*
401 		 * if NULL then can't do everything in one session.
402 		 * XXX Fix this. We need to inject a "virtual" session
403 		 * XXX layer right about here.
404 		 */
405 	}
406 	if (cap != NULL) {
407 		/* Call the driver initialization routine. */
408 		hid = cap - crypto_drivers;
409 		lid = hid;		/* Pass the driver ID. */
410 		err = CRYPTODEV_NEWSESSION(cap->cc_dev, &lid, cri);
411 		if (err == 0) {
412 			(*sid) = (cap->cc_flags & 0xff000000)
413 			       | (hid & 0x00ffffff);
414 			(*sid) <<= 32;
415 			(*sid) |= (lid & 0xffffffff);
416 			cap->cc_sessions++;
417 		}
418 	} else
419 		err = EINVAL;
420 	CRYPTO_DRIVER_UNLOCK();
421 	return err;
422 }
423 
424 static void
425 crypto_remove(struct cryptocap *cap)
426 {
427 
428 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
429 	if (cap->cc_sessions == 0 && cap->cc_koperations == 0)
430 		bzero(cap, sizeof(*cap));
431 }
432 
433 /*
434  * Delete an existing session (or a reserved session on an unregistered
435  * driver).
436  */
437 int
438 crypto_freesession(u_int64_t sid)
439 {
440 	struct cryptocap *cap;
441 	u_int32_t hid;
442 	int err;
443 
444 	CRYPTO_DRIVER_LOCK();
445 
446 	if (crypto_drivers == NULL) {
447 		err = EINVAL;
448 		goto done;
449 	}
450 
451 	/* Determine two IDs. */
452 	hid = CRYPTO_SESID2HID(sid);
453 
454 	if (hid >= crypto_drivers_num) {
455 		err = ENOENT;
456 		goto done;
457 	}
458 	cap = &crypto_drivers[hid];
459 
460 	if (cap->cc_sessions)
461 		cap->cc_sessions--;
462 
463 	/* Call the driver cleanup routine, if available. */
464 	err = CRYPTODEV_FREESESSION(cap->cc_dev, sid);
465 
466 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
467 		crypto_remove(cap);
468 
469 done:
470 	CRYPTO_DRIVER_UNLOCK();
471 	return err;
472 }
473 
474 /*
475  * Return an unused driver id.  Used by drivers prior to registering
476  * support for the algorithms they handle.
477  */
478 int32_t
479 crypto_get_driverid(device_t dev, int flags)
480 {
481 	struct cryptocap *newdrv;
482 	int i;
483 
484 	if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
485 		printf("%s: no flags specified when registering driver\n",
486 		    device_get_nameunit(dev));
487 		return -1;
488 	}
489 
490 	CRYPTO_DRIVER_LOCK();
491 
492 	for (i = 0; i < crypto_drivers_num; i++) {
493 		if (crypto_drivers[i].cc_dev == NULL &&
494 		    (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) {
495 			break;
496 		}
497 	}
498 
499 	/* Out of entries, allocate some more. */
500 	if (i == crypto_drivers_num) {
501 		/* Be careful about wrap-around. */
502 		if (2 * crypto_drivers_num <= crypto_drivers_num) {
503 			CRYPTO_DRIVER_UNLOCK();
504 			printf("crypto: driver count wraparound!\n");
505 			return -1;
506 		}
507 
508 		newdrv = malloc(2 * crypto_drivers_num *
509 		    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
510 		if (newdrv == NULL) {
511 			CRYPTO_DRIVER_UNLOCK();
512 			printf("crypto: no space to expand driver table!\n");
513 			return -1;
514 		}
515 
516 		bcopy(crypto_drivers, newdrv,
517 		    crypto_drivers_num * sizeof(struct cryptocap));
518 
519 		crypto_drivers_num *= 2;
520 
521 		free(crypto_drivers, M_CRYPTO_DATA);
522 		crypto_drivers = newdrv;
523 	}
524 
525 	/* NB: state is zero'd on free */
526 	crypto_drivers[i].cc_sessions = 1;	/* Mark */
527 	crypto_drivers[i].cc_dev = dev;
528 	crypto_drivers[i].cc_flags = flags;
529 	if (bootverbose)
530 		printf("crypto: assign %s driver id %u, flags %u\n",
531 		    device_get_nameunit(dev), i, flags);
532 
533 	CRYPTO_DRIVER_UNLOCK();
534 
535 	return i;
536 }
537 
538 /*
539  * Lookup a driver by name.  We match against the full device
540  * name and unit, and against just the name.  The latter gives
541  * us a simple widlcarding by device name.  On success return the
542  * driver/hardware identifier; otherwise return -1.
543  */
544 int
545 crypto_find_driver(const char *match)
546 {
547 	int i, len = strlen(match);
548 
549 	CRYPTO_DRIVER_LOCK();
550 	for (i = 0; i < crypto_drivers_num; i++) {
551 		device_t dev = crypto_drivers[i].cc_dev;
552 		if (dev == NULL ||
553 		    (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP))
554 			continue;
555 		if (strncmp(match, device_get_nameunit(dev), len) == 0 ||
556 		    strncmp(match, device_get_name(dev), len) == 0)
557 			break;
558 	}
559 	CRYPTO_DRIVER_UNLOCK();
560 	return i < crypto_drivers_num ? i : -1;
561 }
562 
563 /*
564  * Return the device_t for the specified driver or NULL
565  * if the driver identifier is invalid.
566  */
567 device_t
568 crypto_find_device_byhid(int hid)
569 {
570 	struct cryptocap *cap = crypto_checkdriver(hid);
571 	return cap != NULL ? cap->cc_dev : NULL;
572 }
573 
574 /*
575  * Return the device/driver capabilities.
576  */
577 int
578 crypto_getcaps(int hid)
579 {
580 	struct cryptocap *cap = crypto_checkdriver(hid);
581 	return cap != NULL ? cap->cc_flags : 0;
582 }
583 
584 /*
585  * Register support for a key-related algorithm.  This routine
586  * is called once for each algorithm supported a driver.
587  */
588 int
589 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags)
590 {
591 	struct cryptocap *cap;
592 	int err;
593 
594 	CRYPTO_DRIVER_LOCK();
595 
596 	cap = crypto_checkdriver(driverid);
597 	if (cap != NULL &&
598 	    (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
599 		/*
600 		 * XXX Do some performance testing to determine placing.
601 		 * XXX We probably need an auxiliary data structure that
602 		 * XXX describes relative performances.
603 		 */
604 
605 		cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
606 		if (bootverbose)
607 			printf("crypto: %s registers key alg %u flags %u\n"
608 				, device_get_nameunit(cap->cc_dev)
609 				, kalg
610 				, flags
611 			);
612 		err = 0;
613 	} else
614 		err = EINVAL;
615 
616 	CRYPTO_DRIVER_UNLOCK();
617 	return err;
618 }
619 
620 /*
621  * Register support for a non-key-related algorithm.  This routine
622  * is called once for each such algorithm supported by a driver.
623  */
624 int
625 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
626     u_int32_t flags)
627 {
628 	struct cryptocap *cap;
629 	int err;
630 
631 	CRYPTO_DRIVER_LOCK();
632 
633 	cap = crypto_checkdriver(driverid);
634 	/* NB: algorithms are in the range [1..max] */
635 	if (cap != NULL &&
636 	    (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
637 		/*
638 		 * XXX Do some performance testing to determine placing.
639 		 * XXX We probably need an auxiliary data structure that
640 		 * XXX describes relative performances.
641 		 */
642 
643 		cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
644 		cap->cc_max_op_len[alg] = maxoplen;
645 		if (bootverbose)
646 			printf("crypto: %s registers alg %u flags %u maxoplen %u\n"
647 				, device_get_nameunit(cap->cc_dev)
648 				, alg
649 				, flags
650 				, maxoplen
651 			);
652 		cap->cc_sessions = 0;		/* Unmark */
653 		err = 0;
654 	} else
655 		err = EINVAL;
656 
657 	CRYPTO_DRIVER_UNLOCK();
658 	return err;
659 }
660 
661 static void
662 driver_finis(struct cryptocap *cap)
663 {
664 	u_int32_t ses, kops;
665 
666 	CRYPTO_DRIVER_ASSERT();
667 
668 	ses = cap->cc_sessions;
669 	kops = cap->cc_koperations;
670 	bzero(cap, sizeof(*cap));
671 	if (ses != 0 || kops != 0) {
672 		/*
673 		 * If there are pending sessions,
674 		 * just mark as invalid.
675 		 */
676 		cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
677 		cap->cc_sessions = ses;
678 		cap->cc_koperations = kops;
679 	}
680 }
681 
682 /*
683  * Unregister a crypto driver. If there are pending sessions using it,
684  * leave enough information around so that subsequent calls using those
685  * sessions will correctly detect the driver has been unregistered and
686  * reroute requests.
687  */
688 int
689 crypto_unregister(u_int32_t driverid, int alg)
690 {
691 	struct cryptocap *cap;
692 	int i, err;
693 
694 	CRYPTO_DRIVER_LOCK();
695 	cap = crypto_checkdriver(driverid);
696 	if (cap != NULL &&
697 	    (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
698 	    cap->cc_alg[alg] != 0) {
699 		cap->cc_alg[alg] = 0;
700 		cap->cc_max_op_len[alg] = 0;
701 
702 		/* Was this the last algorithm ? */
703 		for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
704 			if (cap->cc_alg[i] != 0)
705 				break;
706 
707 		if (i == CRYPTO_ALGORITHM_MAX + 1)
708 			driver_finis(cap);
709 		err = 0;
710 	} else
711 		err = EINVAL;
712 	CRYPTO_DRIVER_UNLOCK();
713 
714 	return err;
715 }
716 
717 /*
718  * Unregister all algorithms associated with a crypto driver.
719  * If there are pending sessions using it, leave enough information
720  * around so that subsequent calls using those sessions will
721  * correctly detect the driver has been unregistered and reroute
722  * requests.
723  */
724 int
725 crypto_unregister_all(u_int32_t driverid)
726 {
727 	struct cryptocap *cap;
728 	int err;
729 
730 	CRYPTO_DRIVER_LOCK();
731 	cap = crypto_checkdriver(driverid);
732 	if (cap != NULL) {
733 		driver_finis(cap);
734 		err = 0;
735 	} else
736 		err = EINVAL;
737 	CRYPTO_DRIVER_UNLOCK();
738 
739 	return err;
740 }
741 
742 /*
743  * Clear blockage on a driver.  The what parameter indicates whether
744  * the driver is now ready for cryptop's and/or cryptokop's.
745  */
746 int
747 crypto_unblock(u_int32_t driverid, int what)
748 {
749 	struct cryptocap *cap;
750 	int err;
751 
752 	CRYPTO_Q_LOCK();
753 	cap = crypto_checkdriver(driverid);
754 	if (cap != NULL) {
755 		if (what & CRYPTO_SYMQ)
756 			cap->cc_qblocked = 0;
757 		if (what & CRYPTO_ASYMQ)
758 			cap->cc_kqblocked = 0;
759 		if (crp_sleep)
760 			wakeup_one(&crp_q);
761 		err = 0;
762 	} else
763 		err = EINVAL;
764 	CRYPTO_Q_UNLOCK();
765 
766 	return err;
767 }
768 
769 /*
770  * Add a crypto request to a queue, to be processed by the kernel thread.
771  */
772 int
773 crypto_dispatch(struct cryptop *crp)
774 {
775 	struct cryptocap *cap;
776 	u_int32_t hid;
777 	int result;
778 
779 	cryptostats.cs_ops++;
780 
781 #ifdef CRYPTO_TIMING
782 	if (crypto_timing)
783 		binuptime(&crp->crp_tstamp);
784 #endif
785 
786 	hid = CRYPTO_SESID2HID(crp->crp_sid);
787 
788 	if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
789 		/*
790 		 * Caller marked the request to be processed
791 		 * immediately; dispatch it directly to the
792 		 * driver unless the driver is currently blocked.
793 		 */
794 		cap = crypto_checkdriver(hid);
795 		/* Driver cannot disappeared when there is an active session. */
796 		KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
797 		if (!cap->cc_qblocked) {
798 			result = crypto_invoke(cap, crp, 0);
799 			if (result != ERESTART)
800 				return (result);
801 			/*
802 			 * The driver ran out of resources, put the request on
803 			 * the queue.
804 			 */
805 		}
806 	}
807 	CRYPTO_Q_LOCK();
808 	TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
809 	if (crp_sleep)
810 		wakeup_one(&crp_q);
811 	CRYPTO_Q_UNLOCK();
812 	return 0;
813 }
814 
815 /*
816  * Add an asymetric crypto request to a queue,
817  * to be processed by the kernel thread.
818  */
819 int
820 crypto_kdispatch(struct cryptkop *krp)
821 {
822 	int error;
823 
824 	cryptostats.cs_kops++;
825 
826 	error = crypto_kinvoke(krp, krp->krp_crid);
827 	if (error == ERESTART) {
828 		CRYPTO_Q_LOCK();
829 		TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
830 		if (crp_sleep)
831 			wakeup_one(&crp_q);
832 		CRYPTO_Q_UNLOCK();
833 		error = 0;
834 	}
835 	return error;
836 }
837 
838 /*
839  * Verify a driver is suitable for the specified operation.
840  */
841 static __inline int
842 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp)
843 {
844 	return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0;
845 }
846 
847 /*
848  * Select a driver for an asym operation.  The driver must
849  * support the necessary algorithm.  The caller can constrain
850  * which device is selected with the flags parameter.  The
851  * algorithm we use here is pretty stupid; just use the first
852  * driver that supports the algorithms we need. If there are
853  * multiple suitable drivers we choose the driver with the
854  * fewest active operations.  We prefer hardware-backed
855  * drivers to software ones when either may be used.
856  */
857 static struct cryptocap *
858 crypto_select_kdriver(const struct cryptkop *krp, int flags)
859 {
860 	struct cryptocap *cap, *best, *blocked;
861 	int match, hid;
862 
863 	CRYPTO_DRIVER_ASSERT();
864 
865 	/*
866 	 * Look first for hardware crypto devices if permitted.
867 	 */
868 	if (flags & CRYPTOCAP_F_HARDWARE)
869 		match = CRYPTOCAP_F_HARDWARE;
870 	else
871 		match = CRYPTOCAP_F_SOFTWARE;
872 	best = NULL;
873 	blocked = NULL;
874 again:
875 	for (hid = 0; hid < crypto_drivers_num; hid++) {
876 		cap = &crypto_drivers[hid];
877 		/*
878 		 * If it's not initialized, is in the process of
879 		 * going away, or is not appropriate (hardware
880 		 * or software based on match), then skip.
881 		 */
882 		if (cap->cc_dev == NULL ||
883 		    (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
884 		    (cap->cc_flags & match) == 0)
885 			continue;
886 
887 		/* verify all the algorithms are supported. */
888 		if (kdriver_suitable(cap, krp)) {
889 			if (best == NULL ||
890 			    cap->cc_koperations < best->cc_koperations)
891 				best = cap;
892 		}
893 	}
894 	if (best != NULL)
895 		return best;
896 	if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
897 		/* sort of an Algol 68-style for loop */
898 		match = CRYPTOCAP_F_SOFTWARE;
899 		goto again;
900 	}
901 	return best;
902 }
903 
904 /*
905  * Dispatch an assymetric crypto request.
906  */
907 static int
908 crypto_kinvoke(struct cryptkop *krp, int crid)
909 {
910 	struct cryptocap *cap = NULL;
911 	int error;
912 
913 	KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
914 	KASSERT(krp->krp_callback != NULL,
915 	    ("%s: krp->crp_callback == NULL", __func__));
916 
917 	CRYPTO_DRIVER_LOCK();
918 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
919 		cap = crypto_checkdriver(crid);
920 		if (cap != NULL) {
921 			/*
922 			 * Driver present, it must support the necessary
923 			 * algorithm and, if s/w drivers are excluded,
924 			 * it must be registered as hardware-backed.
925 			 */
926 			if (!kdriver_suitable(cap, krp) ||
927 			    (!crypto_devallowsoft &&
928 			     (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
929 				cap = NULL;
930 		}
931 	} else {
932 		/*
933 		 * No requested driver; select based on crid flags.
934 		 */
935 		if (!crypto_devallowsoft)	/* NB: disallow s/w drivers */
936 			crid &= ~CRYPTOCAP_F_SOFTWARE;
937 		cap = crypto_select_kdriver(krp, crid);
938 	}
939 	if (cap != NULL && !cap->cc_kqblocked) {
940 		krp->krp_hid = cap - crypto_drivers;
941 		cap->cc_koperations++;
942 		CRYPTO_DRIVER_UNLOCK();
943 		error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
944 		CRYPTO_DRIVER_LOCK();
945 		if (error == ERESTART) {
946 			cap->cc_koperations--;
947 			CRYPTO_DRIVER_UNLOCK();
948 			return (error);
949 		}
950 	} else {
951 		/*
952 		 * NB: cap is !NULL if device is blocked; in
953 		 *     that case return ERESTART so the operation
954 		 *     is resubmitted if possible.
955 		 */
956 		error = (cap == NULL) ? ENODEV : ERESTART;
957 	}
958 	CRYPTO_DRIVER_UNLOCK();
959 
960 	if (error) {
961 		krp->krp_status = error;
962 		crypto_kdone(krp);
963 	}
964 	return 0;
965 }
966 
967 #ifdef CRYPTO_TIMING
968 static void
969 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
970 {
971 	struct bintime now, delta;
972 	struct timespec t;
973 	uint64_t u;
974 
975 	binuptime(&now);
976 	u = now.frac;
977 	delta.frac = now.frac - bt->frac;
978 	delta.sec = now.sec - bt->sec;
979 	if (u < delta.frac)
980 		delta.sec--;
981 	bintime2timespec(&delta, &t);
982 	timespecadd(&ts->acc, &t);
983 	if (timespeccmp(&t, &ts->min, <))
984 		ts->min = t;
985 	if (timespeccmp(&t, &ts->max, >))
986 		ts->max = t;
987 	ts->count++;
988 
989 	*bt = now;
990 }
991 #endif
992 
993 /*
994  * Dispatch a crypto request to the appropriate crypto devices.
995  */
996 static int
997 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
998 {
999 
1000 	KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1001 	KASSERT(crp->crp_callback != NULL,
1002 	    ("%s: crp->crp_callback == NULL", __func__));
1003 	KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__));
1004 
1005 #ifdef CRYPTO_TIMING
1006 	if (crypto_timing)
1007 		crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
1008 #endif
1009 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1010 		struct cryptodesc *crd;
1011 		u_int64_t nid;
1012 
1013 		/*
1014 		 * Driver has unregistered; migrate the session and return
1015 		 * an error to the caller so they'll resubmit the op.
1016 		 *
1017 		 * XXX: What if there are more already queued requests for this
1018 		 *      session?
1019 		 */
1020 		crypto_freesession(crp->crp_sid);
1021 
1022 		for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
1023 			crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
1024 
1025 		/* XXX propagate flags from initial session? */
1026 		if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI),
1027 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1028 			crp->crp_sid = nid;
1029 
1030 		crp->crp_etype = EAGAIN;
1031 		crypto_done(crp);
1032 		return 0;
1033 	} else {
1034 		/*
1035 		 * Invoke the driver to process the request.
1036 		 */
1037 		return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1038 	}
1039 }
1040 
1041 /*
1042  * Release a set of crypto descriptors.
1043  */
1044 void
1045 crypto_freereq(struct cryptop *crp)
1046 {
1047 	struct cryptodesc *crd;
1048 
1049 	if (crp == NULL)
1050 		return;
1051 
1052 #ifdef DIAGNOSTIC
1053 	{
1054 		struct cryptop *crp2;
1055 
1056 		CRYPTO_Q_LOCK();
1057 		TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1058 			KASSERT(crp2 != crp,
1059 			    ("Freeing cryptop from the crypto queue (%p).",
1060 			    crp));
1061 		}
1062 		CRYPTO_Q_UNLOCK();
1063 		CRYPTO_RETQ_LOCK();
1064 		TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
1065 			KASSERT(crp2 != crp,
1066 			    ("Freeing cryptop from the return queue (%p).",
1067 			    crp));
1068 		}
1069 		CRYPTO_RETQ_UNLOCK();
1070 	}
1071 #endif
1072 
1073 	while ((crd = crp->crp_desc) != NULL) {
1074 		crp->crp_desc = crd->crd_next;
1075 		uma_zfree(cryptodesc_zone, crd);
1076 	}
1077 	uma_zfree(cryptop_zone, crp);
1078 }
1079 
1080 /*
1081  * Acquire a set of crypto descriptors.
1082  */
1083 struct cryptop *
1084 crypto_getreq(int num)
1085 {
1086 	struct cryptodesc *crd;
1087 	struct cryptop *crp;
1088 
1089 	crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
1090 	if (crp != NULL) {
1091 		while (num--) {
1092 			crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
1093 			if (crd == NULL) {
1094 				crypto_freereq(crp);
1095 				return NULL;
1096 			}
1097 
1098 			crd->crd_next = crp->crp_desc;
1099 			crp->crp_desc = crd;
1100 		}
1101 	}
1102 	return crp;
1103 }
1104 
1105 /*
1106  * Invoke the callback on behalf of the driver.
1107  */
1108 void
1109 crypto_done(struct cryptop *crp)
1110 {
1111 	KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1112 		("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1113 	crp->crp_flags |= CRYPTO_F_DONE;
1114 	if (crp->crp_etype != 0)
1115 		cryptostats.cs_errs++;
1116 #ifdef CRYPTO_TIMING
1117 	if (crypto_timing)
1118 		crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
1119 #endif
1120 	/*
1121 	 * CBIMM means unconditionally do the callback immediately;
1122 	 * CBIFSYNC means do the callback immediately only if the
1123 	 * operation was done synchronously.  Both are used to avoid
1124 	 * doing extraneous context switches; the latter is mostly
1125 	 * used with the software crypto driver.
1126 	 */
1127 	if ((crp->crp_flags & CRYPTO_F_CBIMM) ||
1128 	    ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
1129 	     (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) {
1130 		/*
1131 		 * Do the callback directly.  This is ok when the
1132 		 * callback routine does very little (e.g. the
1133 		 * /dev/crypto callback method just does a wakeup).
1134 		 */
1135 #ifdef CRYPTO_TIMING
1136 		if (crypto_timing) {
1137 			/*
1138 			 * NB: We must copy the timestamp before
1139 			 * doing the callback as the cryptop is
1140 			 * likely to be reclaimed.
1141 			 */
1142 			struct bintime t = crp->crp_tstamp;
1143 			crypto_tstat(&cryptostats.cs_cb, &t);
1144 			crp->crp_callback(crp);
1145 			crypto_tstat(&cryptostats.cs_finis, &t);
1146 		} else
1147 #endif
1148 			crp->crp_callback(crp);
1149 	} else {
1150 		/*
1151 		 * Normal case; queue the callback for the thread.
1152 		 */
1153 		CRYPTO_RETQ_LOCK();
1154 		if (CRYPTO_RETQ_EMPTY())
1155 			wakeup_one(&crp_ret_q);	/* shared wait channel */
1156 		TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
1157 		CRYPTO_RETQ_UNLOCK();
1158 	}
1159 }
1160 
1161 /*
1162  * Invoke the callback on behalf of the driver.
1163  */
1164 void
1165 crypto_kdone(struct cryptkop *krp)
1166 {
1167 	struct cryptocap *cap;
1168 
1169 	if (krp->krp_status != 0)
1170 		cryptostats.cs_kerrs++;
1171 	CRYPTO_DRIVER_LOCK();
1172 	/* XXX: What if driver is loaded in the meantime? */
1173 	if (krp->krp_hid < crypto_drivers_num) {
1174 		cap = &crypto_drivers[krp->krp_hid];
1175 		cap->cc_koperations--;
1176 		KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0"));
1177 		if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
1178 			crypto_remove(cap);
1179 	}
1180 	CRYPTO_DRIVER_UNLOCK();
1181 	CRYPTO_RETQ_LOCK();
1182 	if (CRYPTO_RETQ_EMPTY())
1183 		wakeup_one(&crp_ret_q);		/* shared wait channel */
1184 	TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
1185 	CRYPTO_RETQ_UNLOCK();
1186 }
1187 
1188 int
1189 crypto_getfeat(int *featp)
1190 {
1191 	int hid, kalg, feat = 0;
1192 
1193 	CRYPTO_DRIVER_LOCK();
1194 	for (hid = 0; hid < crypto_drivers_num; hid++) {
1195 		const struct cryptocap *cap = &crypto_drivers[hid];
1196 
1197 		if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1198 		    !crypto_devallowsoft) {
1199 			continue;
1200 		}
1201 		for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
1202 			if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED)
1203 				feat |=  1 << kalg;
1204 	}
1205 	CRYPTO_DRIVER_UNLOCK();
1206 	*featp = feat;
1207 	return (0);
1208 }
1209 
1210 /*
1211  * Terminate a thread at module unload.  The process that
1212  * initiated this is waiting for us to signal that we're gone;
1213  * wake it up and exit.  We use the driver table lock to insure
1214  * we don't do the wakeup before they're waiting.  There is no
1215  * race here because the waiter sleeps on the proc lock for the
1216  * thread so it gets notified at the right time because of an
1217  * extra wakeup that's done in exit1().
1218  */
1219 static void
1220 crypto_finis(void *chan)
1221 {
1222 	CRYPTO_DRIVER_LOCK();
1223 	wakeup_one(chan);
1224 	CRYPTO_DRIVER_UNLOCK();
1225 	kproc_exit(0);
1226 }
1227 
1228 /*
1229  * Crypto thread, dispatches crypto requests.
1230  */
1231 static void
1232 crypto_proc(void)
1233 {
1234 	struct cryptop *crp, *submit;
1235 	struct cryptkop *krp;
1236 	struct cryptocap *cap;
1237 	u_int32_t hid;
1238 	int result, hint;
1239 
1240 	CRYPTO_Q_LOCK();
1241 	for (;;) {
1242 		/*
1243 		 * Find the first element in the queue that can be
1244 		 * processed and look-ahead to see if multiple ops
1245 		 * are ready for the same driver.
1246 		 */
1247 		submit = NULL;
1248 		hint = 0;
1249 		TAILQ_FOREACH(crp, &crp_q, crp_next) {
1250 			hid = CRYPTO_SESID2HID(crp->crp_sid);
1251 			cap = crypto_checkdriver(hid);
1252 			/*
1253 			 * Driver cannot disappeared when there is an active
1254 			 * session.
1255 			 */
1256 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1257 			    __func__, __LINE__));
1258 			if (cap == NULL || cap->cc_dev == NULL) {
1259 				/* Op needs to be migrated, process it. */
1260 				if (submit == NULL)
1261 					submit = crp;
1262 				break;
1263 			}
1264 			if (!cap->cc_qblocked) {
1265 				if (submit != NULL) {
1266 					/*
1267 					 * We stop on finding another op,
1268 					 * regardless whether its for the same
1269 					 * driver or not.  We could keep
1270 					 * searching the queue but it might be
1271 					 * better to just use a per-driver
1272 					 * queue instead.
1273 					 */
1274 					if (CRYPTO_SESID2HID(submit->crp_sid) == hid)
1275 						hint = CRYPTO_HINT_MORE;
1276 					break;
1277 				} else {
1278 					submit = crp;
1279 					if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1280 						break;
1281 					/* keep scanning for more are q'd */
1282 				}
1283 			}
1284 		}
1285 		if (submit != NULL) {
1286 			TAILQ_REMOVE(&crp_q, submit, crp_next);
1287 			hid = CRYPTO_SESID2HID(submit->crp_sid);
1288 			cap = crypto_checkdriver(hid);
1289 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1290 			    __func__, __LINE__));
1291 			result = crypto_invoke(cap, submit, hint);
1292 			if (result == ERESTART) {
1293 				/*
1294 				 * The driver ran out of resources, mark the
1295 				 * driver ``blocked'' for cryptop's and put
1296 				 * the request back in the queue.  It would
1297 				 * best to put the request back where we got
1298 				 * it but that's hard so for now we put it
1299 				 * at the front.  This should be ok; putting
1300 				 * it at the end does not work.
1301 				 */
1302 				/* XXX validate sid again? */
1303 				crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
1304 				TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1305 				cryptostats.cs_blocks++;
1306 			}
1307 		}
1308 
1309 		/* As above, but for key ops */
1310 		TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1311 			cap = crypto_checkdriver(krp->krp_hid);
1312 			if (cap == NULL || cap->cc_dev == NULL) {
1313 				/*
1314 				 * Operation needs to be migrated, invalidate
1315 				 * the assigned device so it will reselect a
1316 				 * new one below.  Propagate the original
1317 				 * crid selection flags if supplied.
1318 				 */
1319 				krp->krp_hid = krp->krp_crid &
1320 				    (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE);
1321 				if (krp->krp_hid == 0)
1322 					krp->krp_hid =
1323 				    CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE;
1324 				break;
1325 			}
1326 			if (!cap->cc_kqblocked)
1327 				break;
1328 		}
1329 		if (krp != NULL) {
1330 			TAILQ_REMOVE(&crp_kq, krp, krp_next);
1331 			result = crypto_kinvoke(krp, krp->krp_hid);
1332 			if (result == ERESTART) {
1333 				/*
1334 				 * The driver ran out of resources, mark the
1335 				 * driver ``blocked'' for cryptkop's and put
1336 				 * the request back in the queue.  It would
1337 				 * best to put the request back where we got
1338 				 * it but that's hard so for now we put it
1339 				 * at the front.  This should be ok; putting
1340 				 * it at the end does not work.
1341 				 */
1342 				/* XXX validate sid again? */
1343 				crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1344 				TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1345 				cryptostats.cs_kblocks++;
1346 			}
1347 		}
1348 
1349 		if (submit == NULL && krp == NULL) {
1350 			/*
1351 			 * Nothing more to be processed.  Sleep until we're
1352 			 * woken because there are more ops to process.
1353 			 * This happens either by submission or by a driver
1354 			 * becoming unblocked and notifying us through
1355 			 * crypto_unblock.  Note that when we wakeup we
1356 			 * start processing each queue again from the
1357 			 * front. It's not clear that it's important to
1358 			 * preserve this ordering since ops may finish
1359 			 * out of order if dispatched to different devices
1360 			 * and some become blocked while others do not.
1361 			 */
1362 			crp_sleep = 1;
1363 			msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
1364 			crp_sleep = 0;
1365 			if (cryptoproc == NULL)
1366 				break;
1367 			cryptostats.cs_intrs++;
1368 		}
1369 	}
1370 	CRYPTO_Q_UNLOCK();
1371 
1372 	crypto_finis(&crp_q);
1373 }
1374 
1375 /*
1376  * Crypto returns thread, does callbacks for processed crypto requests.
1377  * Callbacks are done here, rather than in the crypto drivers, because
1378  * callbacks typically are expensive and would slow interrupt handling.
1379  */
1380 static void
1381 crypto_ret_proc(void)
1382 {
1383 	struct cryptop *crpt;
1384 	struct cryptkop *krpt;
1385 
1386 	CRYPTO_RETQ_LOCK();
1387 	for (;;) {
1388 		/* Harvest return q's for completed ops */
1389 		crpt = TAILQ_FIRST(&crp_ret_q);
1390 		if (crpt != NULL)
1391 			TAILQ_REMOVE(&crp_ret_q, crpt, crp_next);
1392 
1393 		krpt = TAILQ_FIRST(&crp_ret_kq);
1394 		if (krpt != NULL)
1395 			TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next);
1396 
1397 		if (crpt != NULL || krpt != NULL) {
1398 			CRYPTO_RETQ_UNLOCK();
1399 			/*
1400 			 * Run callbacks unlocked.
1401 			 */
1402 			if (crpt != NULL) {
1403 #ifdef CRYPTO_TIMING
1404 				if (crypto_timing) {
1405 					/*
1406 					 * NB: We must copy the timestamp before
1407 					 * doing the callback as the cryptop is
1408 					 * likely to be reclaimed.
1409 					 */
1410 					struct bintime t = crpt->crp_tstamp;
1411 					crypto_tstat(&cryptostats.cs_cb, &t);
1412 					crpt->crp_callback(crpt);
1413 					crypto_tstat(&cryptostats.cs_finis, &t);
1414 				} else
1415 #endif
1416 					crpt->crp_callback(crpt);
1417 			}
1418 			if (krpt != NULL)
1419 				krpt->krp_callback(krpt);
1420 			CRYPTO_RETQ_LOCK();
1421 		} else {
1422 			/*
1423 			 * Nothing more to be processed.  Sleep until we're
1424 			 * woken because there are more returns to process.
1425 			 */
1426 			msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT,
1427 				"crypto_ret_wait", 0);
1428 			if (cryptoretproc == NULL)
1429 				break;
1430 			cryptostats.cs_rets++;
1431 		}
1432 	}
1433 	CRYPTO_RETQ_UNLOCK();
1434 
1435 	crypto_finis(&crp_ret_q);
1436 }
1437 
1438 #ifdef DDB
1439 static void
1440 db_show_drivers(void)
1441 {
1442 	int hid;
1443 
1444 	db_printf("%12s %4s %4s %8s %2s %2s\n"
1445 		, "Device"
1446 		, "Ses"
1447 		, "Kops"
1448 		, "Flags"
1449 		, "QB"
1450 		, "KB"
1451 	);
1452 	for (hid = 0; hid < crypto_drivers_num; hid++) {
1453 		const struct cryptocap *cap = &crypto_drivers[hid];
1454 		if (cap->cc_dev == NULL)
1455 			continue;
1456 		db_printf("%-12s %4u %4u %08x %2u %2u\n"
1457 		    , device_get_nameunit(cap->cc_dev)
1458 		    , cap->cc_sessions
1459 		    , cap->cc_koperations
1460 		    , cap->cc_flags
1461 		    , cap->cc_qblocked
1462 		    , cap->cc_kqblocked
1463 		);
1464 	}
1465 }
1466 
1467 DB_SHOW_COMMAND(crypto, db_show_crypto)
1468 {
1469 	struct cryptop *crp;
1470 
1471 	db_show_drivers();
1472 	db_printf("\n");
1473 
1474 	db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
1475 	    "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
1476 	    "Desc", "Callback");
1477 	TAILQ_FOREACH(crp, &crp_q, crp_next) {
1478 		db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
1479 		    , (int) CRYPTO_SESID2HID(crp->crp_sid)
1480 		    , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
1481 		    , crp->crp_ilen, crp->crp_olen
1482 		    , crp->crp_etype
1483 		    , crp->crp_flags
1484 		    , crp->crp_desc
1485 		    , crp->crp_callback
1486 		);
1487 	}
1488 	if (!TAILQ_EMPTY(&crp_ret_q)) {
1489 		db_printf("\n%4s %4s %4s %8s\n",
1490 		    "HID", "Etype", "Flags", "Callback");
1491 		TAILQ_FOREACH(crp, &crp_ret_q, crp_next) {
1492 			db_printf("%4u %4u %04x %8p\n"
1493 			    , (int) CRYPTO_SESID2HID(crp->crp_sid)
1494 			    , crp->crp_etype
1495 			    , crp->crp_flags
1496 			    , crp->crp_callback
1497 			);
1498 		}
1499 	}
1500 }
1501 
1502 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
1503 {
1504 	struct cryptkop *krp;
1505 
1506 	db_show_drivers();
1507 	db_printf("\n");
1508 
1509 	db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
1510 	    "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
1511 	TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1512 		db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
1513 		    , krp->krp_op
1514 		    , krp->krp_status
1515 		    , krp->krp_iparams, krp->krp_oparams
1516 		    , krp->krp_crid, krp->krp_hid
1517 		    , krp->krp_callback
1518 		);
1519 	}
1520 	if (!TAILQ_EMPTY(&crp_ret_q)) {
1521 		db_printf("%4s %5s %8s %4s %8s\n",
1522 		    "Op", "Status", "CRID", "HID", "Callback");
1523 		TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) {
1524 			db_printf("%4u %5u %08x %4u %8p\n"
1525 			    , krp->krp_op
1526 			    , krp->krp_status
1527 			    , krp->krp_crid, krp->krp_hid
1528 			    , krp->krp_callback
1529 			);
1530 		}
1531 	}
1532 }
1533 #endif
1534 
1535 int crypto_modevent(module_t mod, int type, void *unused);
1536 
1537 /*
1538  * Initialization code, both for static and dynamic loading.
1539  * Note this is not invoked with the usual MODULE_DECLARE
1540  * mechanism but instead is listed as a dependency by the
1541  * cryptosoft driver.  This guarantees proper ordering of
1542  * calls on module load/unload.
1543  */
1544 int
1545 crypto_modevent(module_t mod, int type, void *unused)
1546 {
1547 	int error = EINVAL;
1548 
1549 	switch (type) {
1550 	case MOD_LOAD:
1551 		error = crypto_init();
1552 		if (error == 0 && bootverbose)
1553 			printf("crypto: <crypto core>\n");
1554 		break;
1555 	case MOD_UNLOAD:
1556 		/*XXX disallow if active sessions */
1557 		error = 0;
1558 		crypto_destroy();
1559 		return 0;
1560 	}
1561 	return error;
1562 }
1563 MODULE_VERSION(crypto, 1);
1564 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
1565