xref: /freebsd/sys/opencrypto/crypto.c (revision f0cfa1b168014f56c02b83e5f28412cc5f78d117)
1 /*-
2  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  */
24 
25 #include <sys/cdefs.h>
26 __FBSDID("$FreeBSD$");
27 
28 /*
29  * Cryptographic Subsystem.
30  *
31  * This code is derived from the Openbsd Cryptographic Framework (OCF)
32  * that has the copyright shown below.  Very little of the original
33  * code remains.
34  */
35 
36 /*-
37  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
38  *
39  * This code was written by Angelos D. Keromytis in Athens, Greece, in
40  * February 2000. Network Security Technologies Inc. (NSTI) kindly
41  * supported the development of this code.
42  *
43  * Copyright (c) 2000, 2001 Angelos D. Keromytis
44  *
45  * Permission to use, copy, and modify this software with or without fee
46  * is hereby granted, provided that this entire notice is included in
47  * all source code copies of any software which is or includes a copy or
48  * modification of this software.
49  *
50  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
51  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
52  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
53  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
54  * PURPOSE.
55  */
56 
57 #define	CRYPTO_TIMING				/* enable timing support */
58 
59 #include "opt_ddb.h"
60 
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/eventhandler.h>
64 #include <sys/kernel.h>
65 #include <sys/kthread.h>
66 #include <sys/linker.h>
67 #include <sys/lock.h>
68 #include <sys/module.h>
69 #include <sys/mutex.h>
70 #include <sys/malloc.h>
71 #include <sys/proc.h>
72 #include <sys/sdt.h>
73 #include <sys/smp.h>
74 #include <sys/sysctl.h>
75 #include <sys/taskqueue.h>
76 
77 #include <ddb/ddb.h>
78 
79 #include <vm/uma.h>
80 #include <crypto/intake.h>
81 #include <opencrypto/cryptodev.h>
82 #include <opencrypto/xform.h>			/* XXX for M_XDATA */
83 
84 #include <sys/kobj.h>
85 #include <sys/bus.h>
86 #include "cryptodev_if.h"
87 
88 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
89 #include <machine/pcb.h>
90 #endif
91 
92 SDT_PROVIDER_DEFINE(opencrypto);
93 
94 /*
95  * Crypto drivers register themselves by allocating a slot in the
96  * crypto_drivers table with crypto_get_driverid() and then registering
97  * each algorithm they support with crypto_register() and crypto_kregister().
98  */
99 static	struct mtx crypto_drivers_mtx;		/* lock on driver table */
100 #define	CRYPTO_DRIVER_LOCK()	mtx_lock(&crypto_drivers_mtx)
101 #define	CRYPTO_DRIVER_UNLOCK()	mtx_unlock(&crypto_drivers_mtx)
102 #define	CRYPTO_DRIVER_ASSERT()	mtx_assert(&crypto_drivers_mtx, MA_OWNED)
103 
104 /*
105  * Crypto device/driver capabilities structure.
106  *
107  * Synchronization:
108  * (d) - protected by CRYPTO_DRIVER_LOCK()
109  * (q) - protected by CRYPTO_Q_LOCK()
110  * Not tagged fields are read-only.
111  */
112 struct cryptocap {
113 	device_t	cc_dev;			/* (d) device/driver */
114 	u_int32_t	cc_sessions;		/* (d) # of sessions */
115 	u_int32_t	cc_koperations;		/* (d) # os asym operations */
116 	/*
117 	 * Largest possible operator length (in bits) for each type of
118 	 * encryption algorithm. XXX not used
119 	 */
120 	u_int16_t	cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1];
121 	u_int8_t	cc_alg[CRYPTO_ALGORITHM_MAX + 1];
122 	u_int8_t	cc_kalg[CRK_ALGORITHM_MAX + 1];
123 
124 	int		cc_flags;		/* (d) flags */
125 #define CRYPTOCAP_F_CLEANUP	0x80000000	/* needs resource cleanup */
126 	int		cc_qblocked;		/* (q) symmetric q blocked */
127 	int		cc_kqblocked;		/* (q) asymmetric q blocked */
128 };
129 static	struct cryptocap *crypto_drivers = NULL;
130 static	int crypto_drivers_num = 0;
131 
132 /*
133  * There are two queues for crypto requests; one for symmetric (e.g.
134  * cipher) operations and one for asymmetric (e.g. MOD)operations.
135  * A single mutex is used to lock access to both queues.  We could
136  * have one per-queue but having one simplifies handling of block/unblock
137  * operations.
138  */
139 static	int crp_sleep = 0;
140 static	TAILQ_HEAD(cryptop_q ,cryptop) crp_q;		/* request queues */
141 static	TAILQ_HEAD(,cryptkop) crp_kq;
142 static	struct mtx crypto_q_mtx;
143 #define	CRYPTO_Q_LOCK()		mtx_lock(&crypto_q_mtx)
144 #define	CRYPTO_Q_UNLOCK()	mtx_unlock(&crypto_q_mtx)
145 
146 /*
147  * Taskqueue used to dispatch the crypto requests
148  * that have the CRYPTO_F_ASYNC flag
149  */
150 static struct taskqueue *crypto_tq;
151 
152 /*
153  * Crypto seq numbers are operated on with modular arithmetic
154  */
155 #define	CRYPTO_SEQ_GT(a,b)	((int)((a)-(b)) > 0)
156 
157 struct crypto_ret_worker {
158 	struct mtx crypto_ret_mtx;
159 
160 	TAILQ_HEAD(,cryptop) crp_ordered_ret_q;	/* ordered callback queue for symetric jobs */
161 	TAILQ_HEAD(,cryptop) crp_ret_q;		/* callback queue for symetric jobs */
162 	TAILQ_HEAD(,cryptkop) crp_ret_kq;	/* callback queue for asym jobs */
163 
164 	u_int32_t reorder_ops;		/* total ordered sym jobs received */
165 	u_int32_t reorder_cur_seq;	/* current sym job dispatched */
166 
167 	struct proc *cryptoretproc;
168 };
169 static struct crypto_ret_worker *crypto_ret_workers = NULL;
170 
171 #define CRYPTO_RETW(i)		(&crypto_ret_workers[i])
172 #define CRYPTO_RETW_ID(w)	((w) - crypto_ret_workers)
173 #define FOREACH_CRYPTO_RETW(w) \
174 	for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
175 
176 #define	CRYPTO_RETW_LOCK(w)	mtx_lock(&w->crypto_ret_mtx)
177 #define	CRYPTO_RETW_UNLOCK(w)	mtx_unlock(&w->crypto_ret_mtx)
178 #define	CRYPTO_RETW_EMPTY(w) \
179 	(TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
180 
181 static int crypto_workers_num = 0;
182 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
183 	   &crypto_workers_num, 0,
184 	   "Number of crypto workers used to dispatch crypto jobs");
185 
186 static	uma_zone_t cryptop_zone;
187 static	uma_zone_t cryptodesc_zone;
188 
189 int	crypto_userasymcrypto = 1;	/* userland may do asym crypto reqs */
190 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
191 	   &crypto_userasymcrypto, 0,
192 	   "Enable/disable user-mode access to asymmetric crypto support");
193 int	crypto_devallowsoft = 0;	/* only use hardware crypto */
194 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
195 	   &crypto_devallowsoft, 0,
196 	   "Enable/disable use of software crypto by /dev/crypto");
197 
198 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
199 
200 static	void crypto_proc(void);
201 static	struct proc *cryptoproc;
202 static	void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
203 static	void crypto_destroy(void);
204 static	int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
205 static	int crypto_kinvoke(struct cryptkop *krp, int flags);
206 static	void crypto_task_invoke(void *ctx, int pending);
207 static void crypto_batch_enqueue(struct cryptop *crp);
208 
209 static	struct cryptostats cryptostats;
210 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
211 	    cryptostats, "Crypto system statistics");
212 
213 #ifdef CRYPTO_TIMING
214 static	int crypto_timing = 0;
215 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
216 	   &crypto_timing, 0, "Enable/disable crypto timing support");
217 #endif
218 
219 /* Try to avoid directly exposing the key buffer as a symbol */
220 static struct keybuf *keybuf;
221 
222 static struct keybuf empty_keybuf = {
223         .kb_nents = 0
224 };
225 
226 /* Obtain the key buffer from boot metadata */
227 static void
228 keybuf_init(void)
229 {
230 	caddr_t kmdp;
231 
232 	kmdp = preload_search_by_type("elf kernel");
233 
234 	if (kmdp == NULL)
235 		kmdp = preload_search_by_type("elf64 kernel");
236 
237 	keybuf = (struct keybuf *)preload_search_info(kmdp,
238 	    MODINFO_METADATA | MODINFOMD_KEYBUF);
239 
240         if (keybuf == NULL)
241                 keybuf = &empty_keybuf;
242 }
243 
244 /* It'd be nice if we could store these in some kind of secure memory... */
245 struct keybuf * get_keybuf(void) {
246 
247         return (keybuf);
248 }
249 
250 static int
251 crypto_init(void)
252 {
253 	struct crypto_ret_worker *ret_worker;
254 	int error;
255 
256 	mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
257 		MTX_DEF|MTX_QUIET);
258 
259 	TAILQ_INIT(&crp_q);
260 	TAILQ_INIT(&crp_kq);
261 	mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
262 
263 	cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
264 				    0, 0, 0, 0,
265 				    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
266 	cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
267 				    0, 0, 0, 0,
268 				    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
269 	if (cryptodesc_zone == NULL || cryptop_zone == NULL) {
270 		printf("crypto_init: cannot setup crypto zones\n");
271 		error = ENOMEM;
272 		goto bad;
273 	}
274 
275 	crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
276 	crypto_drivers = malloc(crypto_drivers_num *
277 	    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
278 	if (crypto_drivers == NULL) {
279 		printf("crypto_init: cannot setup crypto drivers\n");
280 		error = ENOMEM;
281 		goto bad;
282 	}
283 
284 	if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
285 		crypto_workers_num = mp_ncpus;
286 
287 	crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO,
288 				taskqueue_thread_enqueue, &crypto_tq);
289 	if (crypto_tq == NULL) {
290 		printf("crypto init: cannot setup crypto taskqueue\n");
291 		error = ENOMEM;
292 		goto bad;
293 	}
294 
295 	taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
296 		"crypto");
297 
298 	error = kproc_create((void (*)(void *)) crypto_proc, NULL,
299 		    &cryptoproc, 0, 0, "crypto");
300 	if (error) {
301 		printf("crypto_init: cannot start crypto thread; error %d",
302 			error);
303 		goto bad;
304 	}
305 
306 	crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
307 			M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
308 	if (crypto_ret_workers == NULL) {
309 		error = ENOMEM;
310 		printf("crypto_init: cannot allocate ret workers\n");
311 		goto bad;
312 	}
313 
314 
315 	FOREACH_CRYPTO_RETW(ret_worker) {
316 		TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
317 		TAILQ_INIT(&ret_worker->crp_ret_q);
318 		TAILQ_INIT(&ret_worker->crp_ret_kq);
319 
320 		ret_worker->reorder_ops = 0;
321 		ret_worker->reorder_cur_seq = 0;
322 
323 		mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
324 
325 		error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
326 				&ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
327 		if (error) {
328 			printf("crypto_init: cannot start cryptoret thread; error %d",
329 				error);
330 			goto bad;
331 		}
332 	}
333 
334 	keybuf_init();
335 
336 	return 0;
337 bad:
338 	crypto_destroy();
339 	return error;
340 }
341 
342 /*
343  * Signal a crypto thread to terminate.  We use the driver
344  * table lock to synchronize the sleep/wakeups so that we
345  * are sure the threads have terminated before we release
346  * the data structures they use.  See crypto_finis below
347  * for the other half of this song-and-dance.
348  */
349 static void
350 crypto_terminate(struct proc **pp, void *q)
351 {
352 	struct proc *p;
353 
354 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
355 	p = *pp;
356 	*pp = NULL;
357 	if (p) {
358 		wakeup_one(q);
359 		PROC_LOCK(p);		/* NB: insure we don't miss wakeup */
360 		CRYPTO_DRIVER_UNLOCK();	/* let crypto_finis progress */
361 		msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
362 		PROC_UNLOCK(p);
363 		CRYPTO_DRIVER_LOCK();
364 	}
365 }
366 
367 static void
368 crypto_destroy(void)
369 {
370 	struct crypto_ret_worker *ret_worker;
371 
372 	/*
373 	 * Terminate any crypto threads.
374 	 */
375 	if (crypto_tq != NULL)
376 		taskqueue_drain_all(crypto_tq);
377 	CRYPTO_DRIVER_LOCK();
378 	crypto_terminate(&cryptoproc, &crp_q);
379 	FOREACH_CRYPTO_RETW(ret_worker)
380 		crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
381 	CRYPTO_DRIVER_UNLOCK();
382 
383 	/* XXX flush queues??? */
384 
385 	/*
386 	 * Reclaim dynamically allocated resources.
387 	 */
388 	if (crypto_drivers != NULL)
389 		free(crypto_drivers, M_CRYPTO_DATA);
390 
391 	if (cryptodesc_zone != NULL)
392 		uma_zdestroy(cryptodesc_zone);
393 	if (cryptop_zone != NULL)
394 		uma_zdestroy(cryptop_zone);
395 	mtx_destroy(&crypto_q_mtx);
396 	FOREACH_CRYPTO_RETW(ret_worker)
397 		mtx_destroy(&ret_worker->crypto_ret_mtx);
398 	free(crypto_ret_workers, M_CRYPTO_DATA);
399 	if (crypto_tq != NULL)
400 		taskqueue_free(crypto_tq);
401 	mtx_destroy(&crypto_drivers_mtx);
402 }
403 
404 static struct cryptocap *
405 crypto_checkdriver(u_int32_t hid)
406 {
407 	if (crypto_drivers == NULL)
408 		return NULL;
409 	return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
410 }
411 
412 /*
413  * Compare a driver's list of supported algorithms against another
414  * list; return non-zero if all algorithms are supported.
415  */
416 static int
417 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri)
418 {
419 	const struct cryptoini *cr;
420 
421 	/* See if all the algorithms are supported. */
422 	for (cr = cri; cr; cr = cr->cri_next)
423 		if (cap->cc_alg[cr->cri_alg] == 0)
424 			return 0;
425 	return 1;
426 }
427 
428 /*
429  * Select a driver for a new session that supports the specified
430  * algorithms and, optionally, is constrained according to the flags.
431  * The algorithm we use here is pretty stupid; just use the
432  * first driver that supports all the algorithms we need. If there
433  * are multiple drivers we choose the driver with the fewest active
434  * sessions.  We prefer hardware-backed drivers to software ones.
435  *
436  * XXX We need more smarts here (in real life too, but that's
437  * XXX another story altogether).
438  */
439 static struct cryptocap *
440 crypto_select_driver(const struct cryptoini *cri, int flags)
441 {
442 	struct cryptocap *cap, *best;
443 	int match, hid;
444 
445 	CRYPTO_DRIVER_ASSERT();
446 
447 	/*
448 	 * Look first for hardware crypto devices if permitted.
449 	 */
450 	if (flags & CRYPTOCAP_F_HARDWARE)
451 		match = CRYPTOCAP_F_HARDWARE;
452 	else
453 		match = CRYPTOCAP_F_SOFTWARE;
454 	best = NULL;
455 again:
456 	for (hid = 0; hid < crypto_drivers_num; hid++) {
457 		cap = &crypto_drivers[hid];
458 		/*
459 		 * If it's not initialized, is in the process of
460 		 * going away, or is not appropriate (hardware
461 		 * or software based on match), then skip.
462 		 */
463 		if (cap->cc_dev == NULL ||
464 		    (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
465 		    (cap->cc_flags & match) == 0)
466 			continue;
467 
468 		/* verify all the algorithms are supported. */
469 		if (driver_suitable(cap, cri)) {
470 			if (best == NULL ||
471 			    cap->cc_sessions < best->cc_sessions)
472 				best = cap;
473 		}
474 	}
475 	if (best == NULL && match == CRYPTOCAP_F_HARDWARE &&
476 	    (flags & CRYPTOCAP_F_SOFTWARE)) {
477 		/* sort of an Algol 68-style for loop */
478 		match = CRYPTOCAP_F_SOFTWARE;
479 		goto again;
480 	}
481 	return best;
482 }
483 
484 /*
485  * Create a new session.  The crid argument specifies a crypto
486  * driver to use or constraints on a driver to select (hardware
487  * only, software only, either).  Whatever driver is selected
488  * must be capable of the requested crypto algorithms.
489  */
490 int
491 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid)
492 {
493 	struct cryptocap *cap;
494 	u_int32_t hid, lid;
495 	int err;
496 
497 	CRYPTO_DRIVER_LOCK();
498 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
499 		/*
500 		 * Use specified driver; verify it is capable.
501 		 */
502 		cap = crypto_checkdriver(crid);
503 		if (cap != NULL && !driver_suitable(cap, cri))
504 			cap = NULL;
505 	} else {
506 		/*
507 		 * No requested driver; select based on crid flags.
508 		 */
509 		cap = crypto_select_driver(cri, crid);
510 		/*
511 		 * if NULL then can't do everything in one session.
512 		 * XXX Fix this. We need to inject a "virtual" session
513 		 * XXX layer right about here.
514 		 */
515 	}
516 	if (cap != NULL) {
517 		/* Call the driver initialization routine. */
518 		hid = cap - crypto_drivers;
519 		lid = hid;		/* Pass the driver ID. */
520 		err = CRYPTODEV_NEWSESSION(cap->cc_dev, &lid, cri);
521 		if (err == 0) {
522 			(*sid) = (cap->cc_flags & 0xff000000)
523 			       | (hid & 0x00ffffff);
524 			(*sid) <<= 32;
525 			(*sid) |= (lid & 0xffffffff);
526 			cap->cc_sessions++;
527 		} else
528 			CRYPTDEB("dev newsession failed");
529 	} else {
530 		CRYPTDEB("no driver");
531 		err = EOPNOTSUPP;
532 	}
533 	CRYPTO_DRIVER_UNLOCK();
534 	return err;
535 }
536 
537 static void
538 crypto_remove(struct cryptocap *cap)
539 {
540 
541 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
542 	if (cap->cc_sessions == 0 && cap->cc_koperations == 0)
543 		bzero(cap, sizeof(*cap));
544 }
545 
546 /*
547  * Delete an existing session (or a reserved session on an unregistered
548  * driver).
549  */
550 int
551 crypto_freesession(u_int64_t sid)
552 {
553 	struct cryptocap *cap;
554 	u_int32_t hid;
555 	int err;
556 
557 	CRYPTO_DRIVER_LOCK();
558 
559 	if (crypto_drivers == NULL) {
560 		err = EINVAL;
561 		goto done;
562 	}
563 
564 	/* Determine two IDs. */
565 	hid = CRYPTO_SESID2HID(sid);
566 
567 	if (hid >= crypto_drivers_num) {
568 		err = ENOENT;
569 		goto done;
570 	}
571 	cap = &crypto_drivers[hid];
572 
573 	if (cap->cc_sessions)
574 		cap->cc_sessions--;
575 
576 	/* Call the driver cleanup routine, if available. */
577 	err = CRYPTODEV_FREESESSION(cap->cc_dev, sid);
578 
579 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
580 		crypto_remove(cap);
581 
582 done:
583 	CRYPTO_DRIVER_UNLOCK();
584 	return err;
585 }
586 
587 /*
588  * Return an unused driver id.  Used by drivers prior to registering
589  * support for the algorithms they handle.
590  */
591 int32_t
592 crypto_get_driverid(device_t dev, int flags)
593 {
594 	struct cryptocap *newdrv;
595 	int i;
596 
597 	if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
598 		printf("%s: no flags specified when registering driver\n",
599 		    device_get_nameunit(dev));
600 		return -1;
601 	}
602 
603 	CRYPTO_DRIVER_LOCK();
604 
605 	for (i = 0; i < crypto_drivers_num; i++) {
606 		if (crypto_drivers[i].cc_dev == NULL &&
607 		    (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) {
608 			break;
609 		}
610 	}
611 
612 	/* Out of entries, allocate some more. */
613 	if (i == crypto_drivers_num) {
614 		/* Be careful about wrap-around. */
615 		if (2 * crypto_drivers_num <= crypto_drivers_num) {
616 			CRYPTO_DRIVER_UNLOCK();
617 			printf("crypto: driver count wraparound!\n");
618 			return -1;
619 		}
620 
621 		newdrv = malloc(2 * crypto_drivers_num *
622 		    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
623 		if (newdrv == NULL) {
624 			CRYPTO_DRIVER_UNLOCK();
625 			printf("crypto: no space to expand driver table!\n");
626 			return -1;
627 		}
628 
629 		bcopy(crypto_drivers, newdrv,
630 		    crypto_drivers_num * sizeof(struct cryptocap));
631 
632 		crypto_drivers_num *= 2;
633 
634 		free(crypto_drivers, M_CRYPTO_DATA);
635 		crypto_drivers = newdrv;
636 	}
637 
638 	/* NB: state is zero'd on free */
639 	crypto_drivers[i].cc_sessions = 1;	/* Mark */
640 	crypto_drivers[i].cc_dev = dev;
641 	crypto_drivers[i].cc_flags = flags;
642 	if (bootverbose)
643 		printf("crypto: assign %s driver id %u, flags 0x%x\n",
644 		    device_get_nameunit(dev), i, flags);
645 
646 	CRYPTO_DRIVER_UNLOCK();
647 
648 	return i;
649 }
650 
651 /*
652  * Lookup a driver by name.  We match against the full device
653  * name and unit, and against just the name.  The latter gives
654  * us a simple widlcarding by device name.  On success return the
655  * driver/hardware identifier; otherwise return -1.
656  */
657 int
658 crypto_find_driver(const char *match)
659 {
660 	int i, len = strlen(match);
661 
662 	CRYPTO_DRIVER_LOCK();
663 	for (i = 0; i < crypto_drivers_num; i++) {
664 		device_t dev = crypto_drivers[i].cc_dev;
665 		if (dev == NULL ||
666 		    (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP))
667 			continue;
668 		if (strncmp(match, device_get_nameunit(dev), len) == 0 ||
669 		    strncmp(match, device_get_name(dev), len) == 0)
670 			break;
671 	}
672 	CRYPTO_DRIVER_UNLOCK();
673 	return i < crypto_drivers_num ? i : -1;
674 }
675 
676 /*
677  * Return the device_t for the specified driver or NULL
678  * if the driver identifier is invalid.
679  */
680 device_t
681 crypto_find_device_byhid(int hid)
682 {
683 	struct cryptocap *cap = crypto_checkdriver(hid);
684 	return cap != NULL ? cap->cc_dev : NULL;
685 }
686 
687 /*
688  * Return the device/driver capabilities.
689  */
690 int
691 crypto_getcaps(int hid)
692 {
693 	struct cryptocap *cap = crypto_checkdriver(hid);
694 	return cap != NULL ? cap->cc_flags : 0;
695 }
696 
697 /*
698  * Register support for a key-related algorithm.  This routine
699  * is called once for each algorithm supported a driver.
700  */
701 int
702 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags)
703 {
704 	struct cryptocap *cap;
705 	int err;
706 
707 	CRYPTO_DRIVER_LOCK();
708 
709 	cap = crypto_checkdriver(driverid);
710 	if (cap != NULL &&
711 	    (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
712 		/*
713 		 * XXX Do some performance testing to determine placing.
714 		 * XXX We probably need an auxiliary data structure that
715 		 * XXX describes relative performances.
716 		 */
717 
718 		cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
719 		if (bootverbose)
720 			printf("crypto: %s registers key alg %u flags %u\n"
721 				, device_get_nameunit(cap->cc_dev)
722 				, kalg
723 				, flags
724 			);
725 		err = 0;
726 	} else
727 		err = EINVAL;
728 
729 	CRYPTO_DRIVER_UNLOCK();
730 	return err;
731 }
732 
733 /*
734  * Register support for a non-key-related algorithm.  This routine
735  * is called once for each such algorithm supported by a driver.
736  */
737 int
738 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
739     u_int32_t flags)
740 {
741 	struct cryptocap *cap;
742 	int err;
743 
744 	CRYPTO_DRIVER_LOCK();
745 
746 	cap = crypto_checkdriver(driverid);
747 	/* NB: algorithms are in the range [1..max] */
748 	if (cap != NULL &&
749 	    (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
750 		/*
751 		 * XXX Do some performance testing to determine placing.
752 		 * XXX We probably need an auxiliary data structure that
753 		 * XXX describes relative performances.
754 		 */
755 
756 		cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
757 		cap->cc_max_op_len[alg] = maxoplen;
758 		if (bootverbose)
759 			printf("crypto: %s registers alg %u flags %u maxoplen %u\n"
760 				, device_get_nameunit(cap->cc_dev)
761 				, alg
762 				, flags
763 				, maxoplen
764 			);
765 		cap->cc_sessions = 0;		/* Unmark */
766 		err = 0;
767 	} else
768 		err = EINVAL;
769 
770 	CRYPTO_DRIVER_UNLOCK();
771 	return err;
772 }
773 
774 static void
775 driver_finis(struct cryptocap *cap)
776 {
777 	u_int32_t ses, kops;
778 
779 	CRYPTO_DRIVER_ASSERT();
780 
781 	ses = cap->cc_sessions;
782 	kops = cap->cc_koperations;
783 	bzero(cap, sizeof(*cap));
784 	if (ses != 0 || kops != 0) {
785 		/*
786 		 * If there are pending sessions,
787 		 * just mark as invalid.
788 		 */
789 		cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
790 		cap->cc_sessions = ses;
791 		cap->cc_koperations = kops;
792 	}
793 }
794 
795 /*
796  * Unregister a crypto driver. If there are pending sessions using it,
797  * leave enough information around so that subsequent calls using those
798  * sessions will correctly detect the driver has been unregistered and
799  * reroute requests.
800  */
801 int
802 crypto_unregister(u_int32_t driverid, int alg)
803 {
804 	struct cryptocap *cap;
805 	int i, err;
806 
807 	CRYPTO_DRIVER_LOCK();
808 	cap = crypto_checkdriver(driverid);
809 	if (cap != NULL &&
810 	    (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
811 	    cap->cc_alg[alg] != 0) {
812 		cap->cc_alg[alg] = 0;
813 		cap->cc_max_op_len[alg] = 0;
814 
815 		/* Was this the last algorithm ? */
816 		for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
817 			if (cap->cc_alg[i] != 0)
818 				break;
819 
820 		if (i == CRYPTO_ALGORITHM_MAX + 1)
821 			driver_finis(cap);
822 		err = 0;
823 	} else
824 		err = EINVAL;
825 	CRYPTO_DRIVER_UNLOCK();
826 
827 	return err;
828 }
829 
830 /*
831  * Unregister all algorithms associated with a crypto driver.
832  * If there are pending sessions using it, leave enough information
833  * around so that subsequent calls using those sessions will
834  * correctly detect the driver has been unregistered and reroute
835  * requests.
836  */
837 int
838 crypto_unregister_all(u_int32_t driverid)
839 {
840 	struct cryptocap *cap;
841 	int err;
842 
843 	CRYPTO_DRIVER_LOCK();
844 	cap = crypto_checkdriver(driverid);
845 	if (cap != NULL) {
846 		driver_finis(cap);
847 		err = 0;
848 	} else
849 		err = EINVAL;
850 	CRYPTO_DRIVER_UNLOCK();
851 
852 	return err;
853 }
854 
855 /*
856  * Clear blockage on a driver.  The what parameter indicates whether
857  * the driver is now ready for cryptop's and/or cryptokop's.
858  */
859 int
860 crypto_unblock(u_int32_t driverid, int what)
861 {
862 	struct cryptocap *cap;
863 	int err;
864 
865 	CRYPTO_Q_LOCK();
866 	cap = crypto_checkdriver(driverid);
867 	if (cap != NULL) {
868 		if (what & CRYPTO_SYMQ)
869 			cap->cc_qblocked = 0;
870 		if (what & CRYPTO_ASYMQ)
871 			cap->cc_kqblocked = 0;
872 		if (crp_sleep)
873 			wakeup_one(&crp_q);
874 		err = 0;
875 	} else
876 		err = EINVAL;
877 	CRYPTO_Q_UNLOCK();
878 
879 	return err;
880 }
881 
882 /*
883  * Add a crypto request to a queue, to be processed by the kernel thread.
884  */
885 int
886 crypto_dispatch(struct cryptop *crp)
887 {
888 	struct cryptocap *cap;
889 	u_int32_t hid;
890 	int result;
891 
892 	cryptostats.cs_ops++;
893 
894 #ifdef CRYPTO_TIMING
895 	if (crypto_timing)
896 		binuptime(&crp->crp_tstamp);
897 #endif
898 
899 	if (CRYPTOP_ASYNC(crp)) {
900 		if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
901 			struct crypto_ret_worker *ret_worker;
902 
903 			crp->crp_retw_id = crp->crp_sid % crypto_workers_num;
904 			ret_worker = CRYPTO_RETW(crp->crp_retw_id);
905 
906 			CRYPTO_RETW_LOCK(ret_worker);
907 			crp->crp_seq = ret_worker->reorder_ops++;
908 			CRYPTO_RETW_UNLOCK(ret_worker);
909 		}
910 
911 		TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
912 		taskqueue_enqueue(crypto_tq, &crp->crp_task);
913 		return (0);
914 	}
915 
916 	if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
917 		hid = CRYPTO_SESID2HID(crp->crp_sid);
918 
919 		/*
920 		 * Caller marked the request to be processed
921 		 * immediately; dispatch it directly to the
922 		 * driver unless the driver is currently blocked.
923 		 */
924 		cap = crypto_checkdriver(hid);
925 		/* Driver cannot disappeared when there is an active session. */
926 		KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
927 		if (!cap->cc_qblocked) {
928 			result = crypto_invoke(cap, crp, 0);
929 			if (result != ERESTART)
930 				return (result);
931 			/*
932 			 * The driver ran out of resources, put the request on
933 			 * the queue.
934 			 */
935 		}
936 	}
937 	crypto_batch_enqueue(crp);
938 	return 0;
939 }
940 
941 void
942 crypto_batch_enqueue(struct cryptop *crp)
943 {
944 
945 	CRYPTO_Q_LOCK();
946 	TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
947 	if (crp_sleep)
948 		wakeup_one(&crp_q);
949 	CRYPTO_Q_UNLOCK();
950 }
951 
952 /*
953  * Add an asymetric crypto request to a queue,
954  * to be processed by the kernel thread.
955  */
956 int
957 crypto_kdispatch(struct cryptkop *krp)
958 {
959 	int error;
960 
961 	cryptostats.cs_kops++;
962 
963 	error = crypto_kinvoke(krp, krp->krp_crid);
964 	if (error == ERESTART) {
965 		CRYPTO_Q_LOCK();
966 		TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
967 		if (crp_sleep)
968 			wakeup_one(&crp_q);
969 		CRYPTO_Q_UNLOCK();
970 		error = 0;
971 	}
972 	return error;
973 }
974 
975 /*
976  * Verify a driver is suitable for the specified operation.
977  */
978 static __inline int
979 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp)
980 {
981 	return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0;
982 }
983 
984 /*
985  * Select a driver for an asym operation.  The driver must
986  * support the necessary algorithm.  The caller can constrain
987  * which device is selected with the flags parameter.  The
988  * algorithm we use here is pretty stupid; just use the first
989  * driver that supports the algorithms we need. If there are
990  * multiple suitable drivers we choose the driver with the
991  * fewest active operations.  We prefer hardware-backed
992  * drivers to software ones when either may be used.
993  */
994 static struct cryptocap *
995 crypto_select_kdriver(const struct cryptkop *krp, int flags)
996 {
997 	struct cryptocap *cap, *best, *blocked;
998 	int match, hid;
999 
1000 	CRYPTO_DRIVER_ASSERT();
1001 
1002 	/*
1003 	 * Look first for hardware crypto devices if permitted.
1004 	 */
1005 	if (flags & CRYPTOCAP_F_HARDWARE)
1006 		match = CRYPTOCAP_F_HARDWARE;
1007 	else
1008 		match = CRYPTOCAP_F_SOFTWARE;
1009 	best = NULL;
1010 	blocked = NULL;
1011 again:
1012 	for (hid = 0; hid < crypto_drivers_num; hid++) {
1013 		cap = &crypto_drivers[hid];
1014 		/*
1015 		 * If it's not initialized, is in the process of
1016 		 * going away, or is not appropriate (hardware
1017 		 * or software based on match), then skip.
1018 		 */
1019 		if (cap->cc_dev == NULL ||
1020 		    (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
1021 		    (cap->cc_flags & match) == 0)
1022 			continue;
1023 
1024 		/* verify all the algorithms are supported. */
1025 		if (kdriver_suitable(cap, krp)) {
1026 			if (best == NULL ||
1027 			    cap->cc_koperations < best->cc_koperations)
1028 				best = cap;
1029 		}
1030 	}
1031 	if (best != NULL)
1032 		return best;
1033 	if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
1034 		/* sort of an Algol 68-style for loop */
1035 		match = CRYPTOCAP_F_SOFTWARE;
1036 		goto again;
1037 	}
1038 	return best;
1039 }
1040 
1041 /*
1042  * Dispatch an asymmetric crypto request.
1043  */
1044 static int
1045 crypto_kinvoke(struct cryptkop *krp, int crid)
1046 {
1047 	struct cryptocap *cap = NULL;
1048 	int error;
1049 
1050 	KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
1051 	KASSERT(krp->krp_callback != NULL,
1052 	    ("%s: krp->crp_callback == NULL", __func__));
1053 
1054 	CRYPTO_DRIVER_LOCK();
1055 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1056 		cap = crypto_checkdriver(crid);
1057 		if (cap != NULL) {
1058 			/*
1059 			 * Driver present, it must support the necessary
1060 			 * algorithm and, if s/w drivers are excluded,
1061 			 * it must be registered as hardware-backed.
1062 			 */
1063 			if (!kdriver_suitable(cap, krp) ||
1064 			    (!crypto_devallowsoft &&
1065 			     (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
1066 				cap = NULL;
1067 		}
1068 	} else {
1069 		/*
1070 		 * No requested driver; select based on crid flags.
1071 		 */
1072 		if (!crypto_devallowsoft)	/* NB: disallow s/w drivers */
1073 			crid &= ~CRYPTOCAP_F_SOFTWARE;
1074 		cap = crypto_select_kdriver(krp, crid);
1075 	}
1076 	if (cap != NULL && !cap->cc_kqblocked) {
1077 		krp->krp_hid = cap - crypto_drivers;
1078 		cap->cc_koperations++;
1079 		CRYPTO_DRIVER_UNLOCK();
1080 		error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
1081 		CRYPTO_DRIVER_LOCK();
1082 		if (error == ERESTART) {
1083 			cap->cc_koperations--;
1084 			CRYPTO_DRIVER_UNLOCK();
1085 			return (error);
1086 		}
1087 	} else {
1088 		/*
1089 		 * NB: cap is !NULL if device is blocked; in
1090 		 *     that case return ERESTART so the operation
1091 		 *     is resubmitted if possible.
1092 		 */
1093 		error = (cap == NULL) ? ENODEV : ERESTART;
1094 	}
1095 	CRYPTO_DRIVER_UNLOCK();
1096 
1097 	if (error) {
1098 		krp->krp_status = error;
1099 		crypto_kdone(krp);
1100 	}
1101 	return 0;
1102 }
1103 
1104 #ifdef CRYPTO_TIMING
1105 static void
1106 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
1107 {
1108 	struct bintime now, delta;
1109 	struct timespec t;
1110 	uint64_t u;
1111 
1112 	binuptime(&now);
1113 	u = now.frac;
1114 	delta.frac = now.frac - bt->frac;
1115 	delta.sec = now.sec - bt->sec;
1116 	if (u < delta.frac)
1117 		delta.sec--;
1118 	bintime2timespec(&delta, &t);
1119 	timespecadd(&ts->acc, &t);
1120 	if (timespeccmp(&t, &ts->min, <))
1121 		ts->min = t;
1122 	if (timespeccmp(&t, &ts->max, >))
1123 		ts->max = t;
1124 	ts->count++;
1125 
1126 	*bt = now;
1127 }
1128 #endif
1129 
1130 static void
1131 crypto_task_invoke(void *ctx, int pending)
1132 {
1133 	struct cryptocap *cap;
1134 	struct cryptop *crp;
1135 	int hid, result;
1136 
1137 	crp = (struct cryptop *)ctx;
1138 
1139 	hid = CRYPTO_SESID2HID(crp->crp_sid);
1140 	cap = crypto_checkdriver(hid);
1141 
1142 	result = crypto_invoke(cap, crp, 0);
1143 	if (result == ERESTART)
1144 		crypto_batch_enqueue(crp);
1145 }
1146 
1147 /*
1148  * Dispatch a crypto request to the appropriate crypto devices.
1149  */
1150 static int
1151 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1152 {
1153 
1154 	KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1155 	KASSERT(crp->crp_callback != NULL,
1156 	    ("%s: crp->crp_callback == NULL", __func__));
1157 	KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__));
1158 
1159 #ifdef CRYPTO_TIMING
1160 	if (crypto_timing)
1161 		crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
1162 #endif
1163 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1164 		struct cryptodesc *crd;
1165 		u_int64_t nid;
1166 
1167 		/*
1168 		 * Driver has unregistered; migrate the session and return
1169 		 * an error to the caller so they'll resubmit the op.
1170 		 *
1171 		 * XXX: What if there are more already queued requests for this
1172 		 *      session?
1173 		 */
1174 		crypto_freesession(crp->crp_sid);
1175 
1176 		for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
1177 			crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
1178 
1179 		/* XXX propagate flags from initial session? */
1180 		if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI),
1181 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1182 			crp->crp_sid = nid;
1183 
1184 		crp->crp_etype = EAGAIN;
1185 		crypto_done(crp);
1186 		return 0;
1187 	} else {
1188 		/*
1189 		 * Invoke the driver to process the request.
1190 		 */
1191 		return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1192 	}
1193 }
1194 
1195 /*
1196  * Release a set of crypto descriptors.
1197  */
1198 void
1199 crypto_freereq(struct cryptop *crp)
1200 {
1201 	struct cryptodesc *crd;
1202 
1203 	if (crp == NULL)
1204 		return;
1205 
1206 #ifdef DIAGNOSTIC
1207 	{
1208 		struct cryptop *crp2;
1209 		struct crypto_ret_worker *ret_worker;
1210 
1211 		CRYPTO_Q_LOCK();
1212 		TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1213 			KASSERT(crp2 != crp,
1214 			    ("Freeing cryptop from the crypto queue (%p).",
1215 			    crp));
1216 		}
1217 		CRYPTO_Q_UNLOCK();
1218 
1219 		FOREACH_CRYPTO_RETW(ret_worker) {
1220 			CRYPTO_RETW_LOCK(ret_worker);
1221 			TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
1222 				KASSERT(crp2 != crp,
1223 				    ("Freeing cryptop from the return queue (%p).",
1224 				    crp));
1225 			}
1226 			CRYPTO_RETW_UNLOCK(ret_worker);
1227 		}
1228 	}
1229 #endif
1230 
1231 	while ((crd = crp->crp_desc) != NULL) {
1232 		crp->crp_desc = crd->crd_next;
1233 		uma_zfree(cryptodesc_zone, crd);
1234 	}
1235 	uma_zfree(cryptop_zone, crp);
1236 }
1237 
1238 /*
1239  * Acquire a set of crypto descriptors.
1240  */
1241 struct cryptop *
1242 crypto_getreq(int num)
1243 {
1244 	struct cryptodesc *crd;
1245 	struct cryptop *crp;
1246 
1247 	crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
1248 	if (crp != NULL) {
1249 		while (num--) {
1250 			crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
1251 			if (crd == NULL) {
1252 				crypto_freereq(crp);
1253 				return NULL;
1254 			}
1255 
1256 			crd->crd_next = crp->crp_desc;
1257 			crp->crp_desc = crd;
1258 		}
1259 	}
1260 	return crp;
1261 }
1262 
1263 /*
1264  * Invoke the callback on behalf of the driver.
1265  */
1266 void
1267 crypto_done(struct cryptop *crp)
1268 {
1269 	KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
1270 		("crypto_done: op already done, flags 0x%x", crp->crp_flags));
1271 	crp->crp_flags |= CRYPTO_F_DONE;
1272 	if (crp->crp_etype != 0)
1273 		cryptostats.cs_errs++;
1274 #ifdef CRYPTO_TIMING
1275 	if (crypto_timing)
1276 		crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
1277 #endif
1278 	/*
1279 	 * CBIMM means unconditionally do the callback immediately;
1280 	 * CBIFSYNC means do the callback immediately only if the
1281 	 * operation was done synchronously.  Both are used to avoid
1282 	 * doing extraneous context switches; the latter is mostly
1283 	 * used with the software crypto driver.
1284 	 */
1285 	if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
1286 	    ((crp->crp_flags & CRYPTO_F_CBIMM) ||
1287 	    ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
1288 	     (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC)))) {
1289 		/*
1290 		 * Do the callback directly.  This is ok when the
1291 		 * callback routine does very little (e.g. the
1292 		 * /dev/crypto callback method just does a wakeup).
1293 		 */
1294 #ifdef CRYPTO_TIMING
1295 		if (crypto_timing) {
1296 			/*
1297 			 * NB: We must copy the timestamp before
1298 			 * doing the callback as the cryptop is
1299 			 * likely to be reclaimed.
1300 			 */
1301 			struct bintime t = crp->crp_tstamp;
1302 			crypto_tstat(&cryptostats.cs_cb, &t);
1303 			crp->crp_callback(crp);
1304 			crypto_tstat(&cryptostats.cs_finis, &t);
1305 		} else
1306 #endif
1307 			crp->crp_callback(crp);
1308 	} else {
1309 		struct crypto_ret_worker *ret_worker;
1310 		bool wake;
1311 
1312 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1313 		wake = false;
1314 
1315 		/*
1316 		 * Normal case; queue the callback for the thread.
1317 		 */
1318 		CRYPTO_RETW_LOCK(ret_worker);
1319 		if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
1320 			struct cryptop *tmp;
1321 
1322 			TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
1323 					cryptop_q, crp_next) {
1324 				if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1325 					TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
1326 							tmp, crp, crp_next);
1327 					break;
1328 				}
1329 			}
1330 			if (tmp == NULL) {
1331 				TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
1332 						crp, crp_next);
1333 			}
1334 
1335 			if (crp->crp_seq == ret_worker->reorder_cur_seq)
1336 				wake = true;
1337 		}
1338 		else {
1339 			if (CRYPTO_RETW_EMPTY(ret_worker))
1340 				wake = true;
1341 
1342 			TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
1343 		}
1344 
1345 		if (wake)
1346 			wakeup_one(&ret_worker->crp_ret_q);	/* shared wait channel */
1347 		CRYPTO_RETW_UNLOCK(ret_worker);
1348 	}
1349 }
1350 
1351 /*
1352  * Invoke the callback on behalf of the driver.
1353  */
1354 void
1355 crypto_kdone(struct cryptkop *krp)
1356 {
1357 	struct crypto_ret_worker *ret_worker;
1358 	struct cryptocap *cap;
1359 
1360 	if (krp->krp_status != 0)
1361 		cryptostats.cs_kerrs++;
1362 	CRYPTO_DRIVER_LOCK();
1363 	/* XXX: What if driver is loaded in the meantime? */
1364 	if (krp->krp_hid < crypto_drivers_num) {
1365 		cap = &crypto_drivers[krp->krp_hid];
1366 		KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
1367 		cap->cc_koperations--;
1368 		if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
1369 			crypto_remove(cap);
1370 	}
1371 	CRYPTO_DRIVER_UNLOCK();
1372 
1373 	ret_worker = CRYPTO_RETW(0);
1374 
1375 	CRYPTO_RETW_LOCK(ret_worker);
1376 	if (CRYPTO_RETW_EMPTY(ret_worker))
1377 		wakeup_one(&ret_worker->crp_ret_q);		/* shared wait channel */
1378 	TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
1379 	CRYPTO_RETW_UNLOCK(ret_worker);
1380 }
1381 
1382 int
1383 crypto_getfeat(int *featp)
1384 {
1385 	int hid, kalg, feat = 0;
1386 
1387 	CRYPTO_DRIVER_LOCK();
1388 	for (hid = 0; hid < crypto_drivers_num; hid++) {
1389 		const struct cryptocap *cap = &crypto_drivers[hid];
1390 
1391 		if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
1392 		    !crypto_devallowsoft) {
1393 			continue;
1394 		}
1395 		for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
1396 			if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED)
1397 				feat |=  1 << kalg;
1398 	}
1399 	CRYPTO_DRIVER_UNLOCK();
1400 	*featp = feat;
1401 	return (0);
1402 }
1403 
1404 /*
1405  * Terminate a thread at module unload.  The process that
1406  * initiated this is waiting for us to signal that we're gone;
1407  * wake it up and exit.  We use the driver table lock to insure
1408  * we don't do the wakeup before they're waiting.  There is no
1409  * race here because the waiter sleeps on the proc lock for the
1410  * thread so it gets notified at the right time because of an
1411  * extra wakeup that's done in exit1().
1412  */
1413 static void
1414 crypto_finis(void *chan)
1415 {
1416 	CRYPTO_DRIVER_LOCK();
1417 	wakeup_one(chan);
1418 	CRYPTO_DRIVER_UNLOCK();
1419 	kproc_exit(0);
1420 }
1421 
1422 /*
1423  * Crypto thread, dispatches crypto requests.
1424  */
1425 static void
1426 crypto_proc(void)
1427 {
1428 	struct cryptop *crp, *submit;
1429 	struct cryptkop *krp;
1430 	struct cryptocap *cap;
1431 	u_int32_t hid;
1432 	int result, hint;
1433 
1434 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1435 	fpu_kern_thread(FPU_KERN_NORMAL);
1436 #endif
1437 
1438 	CRYPTO_Q_LOCK();
1439 	for (;;) {
1440 		/*
1441 		 * Find the first element in the queue that can be
1442 		 * processed and look-ahead to see if multiple ops
1443 		 * are ready for the same driver.
1444 		 */
1445 		submit = NULL;
1446 		hint = 0;
1447 		TAILQ_FOREACH(crp, &crp_q, crp_next) {
1448 			hid = CRYPTO_SESID2HID(crp->crp_sid);
1449 			cap = crypto_checkdriver(hid);
1450 			/*
1451 			 * Driver cannot disappeared when there is an active
1452 			 * session.
1453 			 */
1454 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1455 			    __func__, __LINE__));
1456 			if (cap == NULL || cap->cc_dev == NULL) {
1457 				/* Op needs to be migrated, process it. */
1458 				if (submit == NULL)
1459 					submit = crp;
1460 				break;
1461 			}
1462 			if (!cap->cc_qblocked) {
1463 				if (submit != NULL) {
1464 					/*
1465 					 * We stop on finding another op,
1466 					 * regardless whether its for the same
1467 					 * driver or not.  We could keep
1468 					 * searching the queue but it might be
1469 					 * better to just use a per-driver
1470 					 * queue instead.
1471 					 */
1472 					if (CRYPTO_SESID2HID(submit->crp_sid) == hid)
1473 						hint = CRYPTO_HINT_MORE;
1474 					break;
1475 				} else {
1476 					submit = crp;
1477 					if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1478 						break;
1479 					/* keep scanning for more are q'd */
1480 				}
1481 			}
1482 		}
1483 		if (submit != NULL) {
1484 			TAILQ_REMOVE(&crp_q, submit, crp_next);
1485 			hid = CRYPTO_SESID2HID(submit->crp_sid);
1486 			cap = crypto_checkdriver(hid);
1487 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1488 			    __func__, __LINE__));
1489 			result = crypto_invoke(cap, submit, hint);
1490 			if (result == ERESTART) {
1491 				/*
1492 				 * The driver ran out of resources, mark the
1493 				 * driver ``blocked'' for cryptop's and put
1494 				 * the request back in the queue.  It would
1495 				 * best to put the request back where we got
1496 				 * it but that's hard so for now we put it
1497 				 * at the front.  This should be ok; putting
1498 				 * it at the end does not work.
1499 				 */
1500 				/* XXX validate sid again? */
1501 				crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
1502 				TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1503 				cryptostats.cs_blocks++;
1504 			}
1505 		}
1506 
1507 		/* As above, but for key ops */
1508 		TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1509 			cap = crypto_checkdriver(krp->krp_hid);
1510 			if (cap == NULL || cap->cc_dev == NULL) {
1511 				/*
1512 				 * Operation needs to be migrated, invalidate
1513 				 * the assigned device so it will reselect a
1514 				 * new one below.  Propagate the original
1515 				 * crid selection flags if supplied.
1516 				 */
1517 				krp->krp_hid = krp->krp_crid &
1518 				    (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE);
1519 				if (krp->krp_hid == 0)
1520 					krp->krp_hid =
1521 				    CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE;
1522 				break;
1523 			}
1524 			if (!cap->cc_kqblocked)
1525 				break;
1526 		}
1527 		if (krp != NULL) {
1528 			TAILQ_REMOVE(&crp_kq, krp, krp_next);
1529 			result = crypto_kinvoke(krp, krp->krp_hid);
1530 			if (result == ERESTART) {
1531 				/*
1532 				 * The driver ran out of resources, mark the
1533 				 * driver ``blocked'' for cryptkop's and put
1534 				 * the request back in the queue.  It would
1535 				 * best to put the request back where we got
1536 				 * it but that's hard so for now we put it
1537 				 * at the front.  This should be ok; putting
1538 				 * it at the end does not work.
1539 				 */
1540 				/* XXX validate sid again? */
1541 				crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1542 				TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1543 				cryptostats.cs_kblocks++;
1544 			}
1545 		}
1546 
1547 		if (submit == NULL && krp == NULL) {
1548 			/*
1549 			 * Nothing more to be processed.  Sleep until we're
1550 			 * woken because there are more ops to process.
1551 			 * This happens either by submission or by a driver
1552 			 * becoming unblocked and notifying us through
1553 			 * crypto_unblock.  Note that when we wakeup we
1554 			 * start processing each queue again from the
1555 			 * front. It's not clear that it's important to
1556 			 * preserve this ordering since ops may finish
1557 			 * out of order if dispatched to different devices
1558 			 * and some become blocked while others do not.
1559 			 */
1560 			crp_sleep = 1;
1561 			msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
1562 			crp_sleep = 0;
1563 			if (cryptoproc == NULL)
1564 				break;
1565 			cryptostats.cs_intrs++;
1566 		}
1567 	}
1568 	CRYPTO_Q_UNLOCK();
1569 
1570 	crypto_finis(&crp_q);
1571 }
1572 
1573 /*
1574  * Crypto returns thread, does callbacks for processed crypto requests.
1575  * Callbacks are done here, rather than in the crypto drivers, because
1576  * callbacks typically are expensive and would slow interrupt handling.
1577  */
1578 static void
1579 crypto_ret_proc(struct crypto_ret_worker *ret_worker)
1580 {
1581 	struct cryptop *crpt;
1582 	struct cryptkop *krpt;
1583 
1584 	CRYPTO_RETW_LOCK(ret_worker);
1585 	for (;;) {
1586 		/* Harvest return q's for completed ops */
1587 		crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
1588 		if (crpt != NULL) {
1589 			if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
1590 				TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
1591 				ret_worker->reorder_cur_seq++;
1592 			} else {
1593 				crpt = NULL;
1594 			}
1595 		}
1596 
1597 		if (crpt == NULL) {
1598 			crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
1599 			if (crpt != NULL)
1600 				TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
1601 		}
1602 
1603 		krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
1604 		if (krpt != NULL)
1605 			TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
1606 
1607 		if (crpt != NULL || krpt != NULL) {
1608 			CRYPTO_RETW_UNLOCK(ret_worker);
1609 			/*
1610 			 * Run callbacks unlocked.
1611 			 */
1612 			if (crpt != NULL) {
1613 #ifdef CRYPTO_TIMING
1614 				if (crypto_timing) {
1615 					/*
1616 					 * NB: We must copy the timestamp before
1617 					 * doing the callback as the cryptop is
1618 					 * likely to be reclaimed.
1619 					 */
1620 					struct bintime t = crpt->crp_tstamp;
1621 					crypto_tstat(&cryptostats.cs_cb, &t);
1622 					crpt->crp_callback(crpt);
1623 					crypto_tstat(&cryptostats.cs_finis, &t);
1624 				} else
1625 #endif
1626 					crpt->crp_callback(crpt);
1627 			}
1628 			if (krpt != NULL)
1629 				krpt->krp_callback(krpt);
1630 			CRYPTO_RETW_LOCK(ret_worker);
1631 		} else {
1632 			/*
1633 			 * Nothing more to be processed.  Sleep until we're
1634 			 * woken because there are more returns to process.
1635 			 */
1636 			msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
1637 				"crypto_ret_wait", 0);
1638 			if (ret_worker->cryptoretproc == NULL)
1639 				break;
1640 			cryptostats.cs_rets++;
1641 		}
1642 	}
1643 	CRYPTO_RETW_UNLOCK(ret_worker);
1644 
1645 	crypto_finis(&ret_worker->crp_ret_q);
1646 }
1647 
1648 #ifdef DDB
1649 static void
1650 db_show_drivers(void)
1651 {
1652 	int hid;
1653 
1654 	db_printf("%12s %4s %4s %8s %2s %2s\n"
1655 		, "Device"
1656 		, "Ses"
1657 		, "Kops"
1658 		, "Flags"
1659 		, "QB"
1660 		, "KB"
1661 	);
1662 	for (hid = 0; hid < crypto_drivers_num; hid++) {
1663 		const struct cryptocap *cap = &crypto_drivers[hid];
1664 		if (cap->cc_dev == NULL)
1665 			continue;
1666 		db_printf("%-12s %4u %4u %08x %2u %2u\n"
1667 		    , device_get_nameunit(cap->cc_dev)
1668 		    , cap->cc_sessions
1669 		    , cap->cc_koperations
1670 		    , cap->cc_flags
1671 		    , cap->cc_qblocked
1672 		    , cap->cc_kqblocked
1673 		);
1674 	}
1675 }
1676 
1677 DB_SHOW_COMMAND(crypto, db_show_crypto)
1678 {
1679 	struct cryptop *crp;
1680 	struct crypto_ret_worker *ret_worker;
1681 
1682 	db_show_drivers();
1683 	db_printf("\n");
1684 
1685 	db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
1686 	    "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
1687 	    "Desc", "Callback");
1688 	TAILQ_FOREACH(crp, &crp_q, crp_next) {
1689 		db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
1690 		    , (int) CRYPTO_SESID2HID(crp->crp_sid)
1691 		    , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
1692 		    , crp->crp_ilen, crp->crp_olen
1693 		    , crp->crp_etype
1694 		    , crp->crp_flags
1695 		    , crp->crp_desc
1696 		    , crp->crp_callback
1697 		);
1698 	}
1699 	FOREACH_CRYPTO_RETW(ret_worker) {
1700 		db_printf("\n%8s %4s %4s %4s %8s\n",
1701 		    "ret_worker", "HID", "Etype", "Flags", "Callback");
1702 		if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
1703 			TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
1704 				db_printf("%8td %4u %4u %04x %8p\n"
1705 				    , CRYPTO_RETW_ID(ret_worker)
1706 				    , (int) CRYPTO_SESID2HID(crp->crp_sid)
1707 				    , crp->crp_etype
1708 				    , crp->crp_flags
1709 				    , crp->crp_callback
1710 				);
1711 			}
1712 		}
1713 	}
1714 }
1715 
1716 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
1717 {
1718 	struct cryptkop *krp;
1719 	struct crypto_ret_worker *ret_worker;
1720 
1721 	db_show_drivers();
1722 	db_printf("\n");
1723 
1724 	db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
1725 	    "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
1726 	TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1727 		db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
1728 		    , krp->krp_op
1729 		    , krp->krp_status
1730 		    , krp->krp_iparams, krp->krp_oparams
1731 		    , krp->krp_crid, krp->krp_hid
1732 		    , krp->krp_callback
1733 		);
1734 	}
1735 
1736 	ret_worker = CRYPTO_RETW(0);
1737 	if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
1738 		db_printf("%4s %5s %8s %4s %8s\n",
1739 		    "Op", "Status", "CRID", "HID", "Callback");
1740 		TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
1741 			db_printf("%4u %5u %08x %4u %8p\n"
1742 			    , krp->krp_op
1743 			    , krp->krp_status
1744 			    , krp->krp_crid, krp->krp_hid
1745 			    , krp->krp_callback
1746 			);
1747 		}
1748 	}
1749 }
1750 #endif
1751 
1752 int crypto_modevent(module_t mod, int type, void *unused);
1753 
1754 /*
1755  * Initialization code, both for static and dynamic loading.
1756  * Note this is not invoked with the usual MODULE_DECLARE
1757  * mechanism but instead is listed as a dependency by the
1758  * cryptosoft driver.  This guarantees proper ordering of
1759  * calls on module load/unload.
1760  */
1761 int
1762 crypto_modevent(module_t mod, int type, void *unused)
1763 {
1764 	int error = EINVAL;
1765 
1766 	switch (type) {
1767 	case MOD_LOAD:
1768 		error = crypto_init();
1769 		if (error == 0 && bootverbose)
1770 			printf("crypto: <crypto core>\n");
1771 		break;
1772 	case MOD_UNLOAD:
1773 		/*XXX disallow if active sessions */
1774 		error = 0;
1775 		crypto_destroy();
1776 		return 0;
1777 	}
1778 	return error;
1779 }
1780 MODULE_VERSION(crypto, 1);
1781 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
1782