1 /*-
2 * Copyright (c) 2002-2006 Sam Leffler. All rights reserved.
3 * Copyright (c) 2021 The FreeBSD Foundation
4 *
5 * Portions of this software were developed by Ararat River
6 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 /*
31 * Cryptographic Subsystem.
32 *
33 * This code is derived from the Openbsd Cryptographic Framework (OCF)
34 * that has the copyright shown below. Very little of the original
35 * code remains.
36 */
37
38 /*-
39 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
40 *
41 * This code was written by Angelos D. Keromytis in Athens, Greece, in
42 * February 2000. Network Security Technologies Inc. (NSTI) kindly
43 * supported the development of this code.
44 *
45 * Copyright (c) 2000, 2001 Angelos D. Keromytis
46 *
47 * Permission to use, copy, and modify this software with or without fee
48 * is hereby granted, provided that this entire notice is included in
49 * all source code copies of any software which is or includes a copy or
50 * modification of this software.
51 *
52 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
53 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
54 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
55 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
56 * PURPOSE.
57 */
58
59 #include "opt_ddb.h"
60
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/counter.h>
64 #include <sys/fail.h>
65 #include <sys/kernel.h>
66 #include <sys/kthread.h>
67 #include <sys/linker.h>
68 #include <sys/lock.h>
69 #include <sys/module.h>
70 #include <sys/mutex.h>
71 #include <sys/malloc.h>
72 #include <sys/mbuf.h>
73 #include <sys/proc.h>
74 #include <sys/refcount.h>
75 #include <sys/sdt.h>
76 #include <sys/smp.h>
77 #include <sys/sysctl.h>
78 #include <sys/taskqueue.h>
79 #include <sys/uio.h>
80
81 #include <ddb/ddb.h>
82
83 #include <machine/vmparam.h>
84 #include <vm/uma.h>
85
86 #include <crypto/intake.h>
87 #include <opencrypto/cryptodev.h>
88 #include <opencrypto/xform_auth.h>
89 #include <opencrypto/xform_enc.h>
90
91 #include <sys/kobj.h>
92 #include <sys/bus.h>
93 #include "cryptodev_if.h"
94
95 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
96 #include <machine/pcb.h>
97 #endif
98
99 SDT_PROVIDER_DEFINE(opencrypto);
100
101 /*
102 * Crypto drivers register themselves by allocating a slot in the
103 * crypto_drivers table with crypto_get_driverid().
104 */
105 static struct mtx crypto_drivers_mtx; /* lock on driver table */
106 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx)
107 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx)
108 #define CRYPTO_DRIVER_ASSERT() mtx_assert(&crypto_drivers_mtx, MA_OWNED)
109
110 /*
111 * Crypto device/driver capabilities structure.
112 *
113 * Synchronization:
114 * (d) - protected by CRYPTO_DRIVER_LOCK()
115 * (q) - protected by CRYPTO_Q_LOCK()
116 * Not tagged fields are read-only.
117 */
118 struct cryptocap {
119 device_t cc_dev;
120 uint32_t cc_hid;
121 uint32_t cc_sessions; /* (d) # of sessions */
122
123 int cc_flags; /* (d) flags */
124 #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */
125 int cc_qblocked; /* (q) symmetric q blocked */
126 size_t cc_session_size;
127 volatile int cc_refs;
128 };
129
130 static struct cryptocap **crypto_drivers = NULL;
131 static int crypto_drivers_size = 0;
132
133 struct crypto_session {
134 struct cryptocap *cap;
135 struct crypto_session_params csp;
136 uint64_t id;
137 /* Driver softc follows. */
138 };
139
140 static int crp_sleep = 0;
141 static TAILQ_HEAD(cryptop_q ,cryptop) crp_q; /* request queues */
142 static struct mtx crypto_q_mtx;
143 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx)
144 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx)
145
146 SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
147 "In-kernel cryptography");
148
149 static SYSCTL_NODE(_debug_fail_point, OID_AUTO, crypto, CTLFLAG_RW, 0,
150 "OCF fail points");
151
152 /*
153 * Taskqueue used to dispatch the crypto requests submitted with
154 * crypto_dispatch_async .
155 */
156 static struct taskqueue *crypto_tq;
157
158 /*
159 * Crypto seq numbers are operated on with modular arithmetic
160 */
161 #define CRYPTO_SEQ_GT(a,b) ((int)((a)-(b)) > 0)
162
163 struct crypto_ret_worker {
164 struct mtx crypto_ret_mtx;
165
166 TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */
167 TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queue for symetric jobs */
168
169 uint32_t reorder_ops; /* total ordered sym jobs received */
170 uint32_t reorder_cur_seq; /* current sym job dispatched */
171
172 struct thread *td;
173 };
174 static struct crypto_ret_worker *crypto_ret_workers = NULL;
175
176 #define CRYPTO_RETW(i) (&crypto_ret_workers[i])
177 #define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers)
178 #define FOREACH_CRYPTO_RETW(w) \
179 for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
180
181 #define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx)
182 #define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx)
183
184 static int crypto_workers_num = 0;
185 SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
186 &crypto_workers_num, 0,
187 "Number of crypto workers used to dispatch crypto jobs");
188 #ifdef COMPAT_FREEBSD12
189 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
190 &crypto_workers_num, 0,
191 "Number of crypto workers used to dispatch crypto jobs");
192 #endif
193
194 static uma_zone_t cryptop_zone;
195
196 int crypto_devallowsoft = 0;
197 SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RWTUN,
198 &crypto_devallowsoft, 0,
199 "Enable use of software crypto by /dev/crypto");
200 #ifdef COMPAT_FREEBSD12
201 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RWTUN,
202 &crypto_devallowsoft, 0,
203 "Enable/disable use of software crypto by /dev/crypto");
204 #endif
205
206 #ifdef DIAGNOSTIC
207 bool crypto_destroyreq_check;
208 SYSCTL_BOOL(_kern_crypto, OID_AUTO, destroyreq_check, CTLFLAG_RWTUN,
209 &crypto_destroyreq_check, 0,
210 "Enable checks when destroying a request");
211 #endif
212
213 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
214
215 static void crypto_dispatch_thread(void *arg);
216 static struct thread *cryptotd;
217 static void crypto_ret_thread(void *arg);
218 static void crypto_destroy(void);
219 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
220 static void crypto_task_invoke(void *ctx, int pending);
221 static void crypto_batch_enqueue(struct cryptop *crp);
222
223 static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)];
224 SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW,
225 cryptostats, nitems(cryptostats),
226 "Crypto system statistics");
227
228 #define CRYPTOSTAT_INC(stat) do { \
229 counter_u64_add( \
230 cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\
231 1); \
232 } while (0)
233
234 static void
cryptostats_init(void * arg __unused)235 cryptostats_init(void *arg __unused)
236 {
237 COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK);
238 }
239 SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL);
240
241 static void
cryptostats_fini(void * arg __unused)242 cryptostats_fini(void *arg __unused)
243 {
244 COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats));
245 }
246 SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini,
247 NULL);
248
249 /* Try to avoid directly exposing the key buffer as a symbol */
250 static struct keybuf *keybuf;
251
252 static struct keybuf empty_keybuf = {
253 .kb_nents = 0
254 };
255
256 /* Obtain the key buffer from boot metadata */
257 static void
keybuf_init(void)258 keybuf_init(void)
259 {
260 keybuf = (struct keybuf *)preload_search_info(preload_kmdp,
261 MODINFO_METADATA | MODINFOMD_KEYBUF);
262
263 if (keybuf == NULL)
264 keybuf = &empty_keybuf;
265 }
266
267 /* It'd be nice if we could store these in some kind of secure memory... */
268 struct keybuf *
get_keybuf(void)269 get_keybuf(void)
270 {
271
272 return (keybuf);
273 }
274
275 static struct cryptocap *
cap_ref(struct cryptocap * cap)276 cap_ref(struct cryptocap *cap)
277 {
278
279 refcount_acquire(&cap->cc_refs);
280 return (cap);
281 }
282
283 static void
cap_rele(struct cryptocap * cap)284 cap_rele(struct cryptocap *cap)
285 {
286
287 if (refcount_release(&cap->cc_refs) == 0)
288 return;
289
290 KASSERT(cap->cc_sessions == 0,
291 ("freeing crypto driver with active sessions"));
292
293 free(cap, M_CRYPTO_DATA);
294 }
295
296 static int
crypto_init(void)297 crypto_init(void)
298 {
299 struct crypto_ret_worker *ret_worker;
300 struct proc *p;
301 int error;
302
303 mtx_init(&crypto_drivers_mtx, "crypto driver table", NULL, MTX_DEF);
304
305 TAILQ_INIT(&crp_q);
306 mtx_init(&crypto_q_mtx, "crypto op queues", NULL, MTX_DEF);
307
308 cryptop_zone = uma_zcreate("cryptop",
309 sizeof(struct cryptop), NULL, NULL, NULL, NULL,
310 UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
311
312 crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
313 crypto_drivers = malloc(crypto_drivers_size *
314 sizeof(struct cryptocap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
315
316 if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
317 crypto_workers_num = mp_ncpus;
318
319 crypto_tq = taskqueue_create("crypto", M_WAITOK | M_ZERO,
320 taskqueue_thread_enqueue, &crypto_tq);
321
322 taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
323 "crypto");
324
325 p = NULL;
326 error = kproc_kthread_add(crypto_dispatch_thread, NULL, &p, &cryptotd,
327 0, 0, "crypto", "crypto");
328 if (error) {
329 printf("crypto_init: cannot start crypto thread; error %d",
330 error);
331 goto bad;
332 }
333
334 crypto_ret_workers = mallocarray(crypto_workers_num,
335 sizeof(struct crypto_ret_worker), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
336
337 FOREACH_CRYPTO_RETW(ret_worker) {
338 TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
339 TAILQ_INIT(&ret_worker->crp_ret_q);
340
341 ret_worker->reorder_ops = 0;
342 ret_worker->reorder_cur_seq = 0;
343
344 mtx_init(&ret_worker->crypto_ret_mtx, "crypto return queues",
345 NULL, MTX_DEF);
346
347 error = kthread_add(crypto_ret_thread, ret_worker, p,
348 &ret_worker->td, 0, 0, "crypto returns %td",
349 CRYPTO_RETW_ID(ret_worker));
350 if (error) {
351 printf("crypto_init: cannot start cryptoret thread; error %d",
352 error);
353 goto bad;
354 }
355 }
356
357 keybuf_init();
358
359 return 0;
360 bad:
361 crypto_destroy();
362 return error;
363 }
364
365 /*
366 * Signal a crypto thread to terminate. We use the driver
367 * table lock to synchronize the sleep/wakeups so that we
368 * are sure the threads have terminated before we release
369 * the data structures they use. See crypto_finis below
370 * for the other half of this song-and-dance.
371 */
372 static void
crypto_terminate(struct thread ** tdp,void * q)373 crypto_terminate(struct thread **tdp, void *q)
374 {
375 struct thread *td;
376
377 mtx_assert(&crypto_drivers_mtx, MA_OWNED);
378 td = *tdp;
379 *tdp = NULL;
380 if (td != NULL) {
381 wakeup_one(q);
382 mtx_sleep(td, &crypto_drivers_mtx, PWAIT, "crypto_destroy", 0);
383 }
384 }
385
386 static void
hmac_init_pad(const struct auth_hash * axf,const char * key,int klen,void * auth_ctx,uint8_t padval)387 hmac_init_pad(const struct auth_hash *axf, const char *key, int klen,
388 void *auth_ctx, uint8_t padval)
389 {
390 uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
391 u_int i;
392
393 KASSERT(axf->blocksize <= sizeof(hmac_key),
394 ("Invalid HMAC block size %d", axf->blocksize));
395
396 /*
397 * If the key is larger than the block size, use the digest of
398 * the key as the key instead.
399 */
400 memset(hmac_key, 0, sizeof(hmac_key));
401 if (klen > axf->blocksize) {
402 axf->Init(auth_ctx);
403 axf->Update(auth_ctx, key, klen);
404 axf->Final(hmac_key, auth_ctx);
405 klen = axf->hashsize;
406 } else
407 memcpy(hmac_key, key, klen);
408
409 for (i = 0; i < axf->blocksize; i++)
410 hmac_key[i] ^= padval;
411
412 axf->Init(auth_ctx);
413 axf->Update(auth_ctx, hmac_key, axf->blocksize);
414 explicit_bzero(hmac_key, sizeof(hmac_key));
415 }
416
417 void
hmac_init_ipad(const struct auth_hash * axf,const char * key,int klen,void * auth_ctx)418 hmac_init_ipad(const struct auth_hash *axf, const char *key, int klen,
419 void *auth_ctx)
420 {
421
422 hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
423 }
424
425 void
hmac_init_opad(const struct auth_hash * axf,const char * key,int klen,void * auth_ctx)426 hmac_init_opad(const struct auth_hash *axf, const char *key, int klen,
427 void *auth_ctx)
428 {
429
430 hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
431 }
432
433 static void
crypto_destroy(void)434 crypto_destroy(void)
435 {
436 struct crypto_ret_worker *ret_worker;
437 int i;
438
439 /*
440 * Terminate any crypto threads.
441 */
442 if (crypto_tq != NULL)
443 taskqueue_drain_all(crypto_tq);
444 CRYPTO_DRIVER_LOCK();
445 crypto_terminate(&cryptotd, &crp_q);
446 FOREACH_CRYPTO_RETW(ret_worker)
447 crypto_terminate(&ret_worker->td, &ret_worker->crp_ret_q);
448 CRYPTO_DRIVER_UNLOCK();
449
450 /* XXX flush queues??? */
451
452 /*
453 * Reclaim dynamically allocated resources.
454 */
455 for (i = 0; i < crypto_drivers_size; i++) {
456 if (crypto_drivers[i] != NULL)
457 cap_rele(crypto_drivers[i]);
458 }
459 free(crypto_drivers, M_CRYPTO_DATA);
460
461 if (cryptop_zone != NULL)
462 uma_zdestroy(cryptop_zone);
463 mtx_destroy(&crypto_q_mtx);
464 FOREACH_CRYPTO_RETW(ret_worker)
465 mtx_destroy(&ret_worker->crypto_ret_mtx);
466 free(crypto_ret_workers, M_CRYPTO_DATA);
467 if (crypto_tq != NULL)
468 taskqueue_free(crypto_tq);
469 mtx_destroy(&crypto_drivers_mtx);
470 }
471
472 uint32_t
crypto_ses2hid(crypto_session_t crypto_session)473 crypto_ses2hid(crypto_session_t crypto_session)
474 {
475 return (crypto_session->cap->cc_hid);
476 }
477
478 uint32_t
crypto_ses2caps(crypto_session_t crypto_session)479 crypto_ses2caps(crypto_session_t crypto_session)
480 {
481 return (crypto_session->cap->cc_flags & 0xff000000);
482 }
483
484 void *
crypto_get_driver_session(crypto_session_t crypto_session)485 crypto_get_driver_session(crypto_session_t crypto_session)
486 {
487 return (crypto_session + 1);
488 }
489
490 const struct crypto_session_params *
crypto_get_params(crypto_session_t crypto_session)491 crypto_get_params(crypto_session_t crypto_session)
492 {
493 return (&crypto_session->csp);
494 }
495
496 const struct auth_hash *
crypto_auth_hash(const struct crypto_session_params * csp)497 crypto_auth_hash(const struct crypto_session_params *csp)
498 {
499
500 switch (csp->csp_auth_alg) {
501 case CRYPTO_SHA1_HMAC:
502 return (&auth_hash_hmac_sha1);
503 case CRYPTO_SHA2_224_HMAC:
504 return (&auth_hash_hmac_sha2_224);
505 case CRYPTO_SHA2_256_HMAC:
506 return (&auth_hash_hmac_sha2_256);
507 case CRYPTO_SHA2_384_HMAC:
508 return (&auth_hash_hmac_sha2_384);
509 case CRYPTO_SHA2_512_HMAC:
510 return (&auth_hash_hmac_sha2_512);
511 case CRYPTO_NULL_HMAC:
512 return (&auth_hash_null);
513 case CRYPTO_RIPEMD160_HMAC:
514 return (&auth_hash_hmac_ripemd_160);
515 case CRYPTO_RIPEMD160:
516 return (&auth_hash_ripemd_160);
517 case CRYPTO_SHA1:
518 return (&auth_hash_sha1);
519 case CRYPTO_SHA2_224:
520 return (&auth_hash_sha2_224);
521 case CRYPTO_SHA2_256:
522 return (&auth_hash_sha2_256);
523 case CRYPTO_SHA2_384:
524 return (&auth_hash_sha2_384);
525 case CRYPTO_SHA2_512:
526 return (&auth_hash_sha2_512);
527 case CRYPTO_AES_NIST_GMAC:
528 switch (csp->csp_auth_klen) {
529 case 128 / 8:
530 return (&auth_hash_nist_gmac_aes_128);
531 case 192 / 8:
532 return (&auth_hash_nist_gmac_aes_192);
533 case 256 / 8:
534 return (&auth_hash_nist_gmac_aes_256);
535 default:
536 return (NULL);
537 }
538 case CRYPTO_BLAKE2B:
539 return (&auth_hash_blake2b);
540 case CRYPTO_BLAKE2S:
541 return (&auth_hash_blake2s);
542 case CRYPTO_POLY1305:
543 return (&auth_hash_poly1305);
544 case CRYPTO_AES_CCM_CBC_MAC:
545 switch (csp->csp_auth_klen) {
546 case 128 / 8:
547 return (&auth_hash_ccm_cbc_mac_128);
548 case 192 / 8:
549 return (&auth_hash_ccm_cbc_mac_192);
550 case 256 / 8:
551 return (&auth_hash_ccm_cbc_mac_256);
552 default:
553 return (NULL);
554 }
555 default:
556 return (NULL);
557 }
558 }
559
560 const struct enc_xform *
crypto_cipher(const struct crypto_session_params * csp)561 crypto_cipher(const struct crypto_session_params *csp)
562 {
563
564 switch (csp->csp_cipher_alg) {
565 case CRYPTO_AES_CBC:
566 return (&enc_xform_aes_cbc);
567 case CRYPTO_AES_XTS:
568 return (&enc_xform_aes_xts);
569 case CRYPTO_AES_ICM:
570 return (&enc_xform_aes_icm);
571 case CRYPTO_AES_NIST_GCM_16:
572 return (&enc_xform_aes_nist_gcm);
573 case CRYPTO_CAMELLIA_CBC:
574 return (&enc_xform_camellia);
575 case CRYPTO_NULL_CBC:
576 return (&enc_xform_null);
577 case CRYPTO_CHACHA20:
578 return (&enc_xform_chacha20);
579 case CRYPTO_AES_CCM_16:
580 return (&enc_xform_ccm);
581 case CRYPTO_CHACHA20_POLY1305:
582 return (&enc_xform_chacha20_poly1305);
583 case CRYPTO_XCHACHA20_POLY1305:
584 return (&enc_xform_xchacha20_poly1305);
585 default:
586 return (NULL);
587 }
588 }
589
590 static struct cryptocap *
crypto_checkdriver(uint32_t hid)591 crypto_checkdriver(uint32_t hid)
592 {
593
594 return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
595 }
596
597 /*
598 * Select a driver for a new session that supports the specified
599 * algorithms and, optionally, is constrained according to the flags.
600 */
601 static struct cryptocap *
crypto_select_driver(const struct crypto_session_params * csp,int flags)602 crypto_select_driver(const struct crypto_session_params *csp, int flags)
603 {
604 struct cryptocap *cap, *best;
605 int best_match, error, hid;
606
607 CRYPTO_DRIVER_ASSERT();
608
609 best = NULL;
610 for (hid = 0; hid < crypto_drivers_size; hid++) {
611 /*
612 * If there is no driver for this slot, or the driver
613 * is not appropriate (hardware or software based on
614 * match), then skip.
615 */
616 cap = crypto_drivers[hid];
617 if (cap == NULL ||
618 (cap->cc_flags & flags) == 0)
619 continue;
620
621 error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
622 if (error >= 0)
623 continue;
624
625 /*
626 * Use the driver with the highest probe value.
627 * Hardware drivers use a higher probe value than
628 * software. In case of a tie, prefer the driver with
629 * the fewest active sessions.
630 */
631 if (best == NULL || error > best_match ||
632 (error == best_match &&
633 cap->cc_sessions < best->cc_sessions)) {
634 best = cap;
635 best_match = error;
636 }
637 }
638 return best;
639 }
640
641 static enum alg_type {
642 ALG_NONE = 0,
643 ALG_CIPHER,
644 ALG_DIGEST,
645 ALG_KEYED_DIGEST,
646 ALG_COMPRESSION,
647 ALG_AEAD
648 } alg_types[] = {
649 [CRYPTO_SHA1_HMAC] = ALG_KEYED_DIGEST,
650 [CRYPTO_RIPEMD160_HMAC] = ALG_KEYED_DIGEST,
651 [CRYPTO_AES_CBC] = ALG_CIPHER,
652 [CRYPTO_SHA1] = ALG_DIGEST,
653 [CRYPTO_NULL_HMAC] = ALG_DIGEST,
654 [CRYPTO_NULL_CBC] = ALG_CIPHER,
655 [CRYPTO_DEFLATE_COMP] = ALG_COMPRESSION,
656 [CRYPTO_SHA2_256_HMAC] = ALG_KEYED_DIGEST,
657 [CRYPTO_SHA2_384_HMAC] = ALG_KEYED_DIGEST,
658 [CRYPTO_SHA2_512_HMAC] = ALG_KEYED_DIGEST,
659 [CRYPTO_CAMELLIA_CBC] = ALG_CIPHER,
660 [CRYPTO_AES_XTS] = ALG_CIPHER,
661 [CRYPTO_AES_ICM] = ALG_CIPHER,
662 [CRYPTO_AES_NIST_GMAC] = ALG_KEYED_DIGEST,
663 [CRYPTO_AES_NIST_GCM_16] = ALG_AEAD,
664 [CRYPTO_BLAKE2B] = ALG_KEYED_DIGEST,
665 [CRYPTO_BLAKE2S] = ALG_KEYED_DIGEST,
666 [CRYPTO_CHACHA20] = ALG_CIPHER,
667 [CRYPTO_SHA2_224_HMAC] = ALG_KEYED_DIGEST,
668 [CRYPTO_RIPEMD160] = ALG_DIGEST,
669 [CRYPTO_SHA2_224] = ALG_DIGEST,
670 [CRYPTO_SHA2_256] = ALG_DIGEST,
671 [CRYPTO_SHA2_384] = ALG_DIGEST,
672 [CRYPTO_SHA2_512] = ALG_DIGEST,
673 [CRYPTO_POLY1305] = ALG_KEYED_DIGEST,
674 [CRYPTO_AES_CCM_CBC_MAC] = ALG_KEYED_DIGEST,
675 [CRYPTO_AES_CCM_16] = ALG_AEAD,
676 [CRYPTO_CHACHA20_POLY1305] = ALG_AEAD,
677 [CRYPTO_XCHACHA20_POLY1305] = ALG_AEAD,
678 };
679
680 static enum alg_type
alg_type(int alg)681 alg_type(int alg)
682 {
683
684 if (alg < nitems(alg_types))
685 return (alg_types[alg]);
686 return (ALG_NONE);
687 }
688
689 static bool
alg_is_compression(int alg)690 alg_is_compression(int alg)
691 {
692
693 return (alg_type(alg) == ALG_COMPRESSION);
694 }
695
696 static bool
alg_is_cipher(int alg)697 alg_is_cipher(int alg)
698 {
699
700 return (alg_type(alg) == ALG_CIPHER);
701 }
702
703 static bool
alg_is_digest(int alg)704 alg_is_digest(int alg)
705 {
706
707 return (alg_type(alg) == ALG_DIGEST ||
708 alg_type(alg) == ALG_KEYED_DIGEST);
709 }
710
711 static bool
alg_is_keyed_digest(int alg)712 alg_is_keyed_digest(int alg)
713 {
714
715 return (alg_type(alg) == ALG_KEYED_DIGEST);
716 }
717
718 static bool
alg_is_aead(int alg)719 alg_is_aead(int alg)
720 {
721
722 return (alg_type(alg) == ALG_AEAD);
723 }
724
725 static bool
ccm_tag_length_valid(int len)726 ccm_tag_length_valid(int len)
727 {
728 /* RFC 3610 */
729 switch (len) {
730 case 4:
731 case 6:
732 case 8:
733 case 10:
734 case 12:
735 case 14:
736 case 16:
737 return (true);
738 default:
739 return (false);
740 }
741 }
742
743 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
744
745 /* Various sanity checks on crypto session parameters. */
746 static bool
check_csp(const struct crypto_session_params * csp)747 check_csp(const struct crypto_session_params *csp)
748 {
749 const struct auth_hash *axf;
750
751 /* Mode-independent checks. */
752 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
753 return (false);
754 if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
755 csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
756 return (false);
757 if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
758 return (false);
759 if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
760 return (false);
761
762 switch (csp->csp_mode) {
763 case CSP_MODE_COMPRESS:
764 if (!alg_is_compression(csp->csp_cipher_alg))
765 return (false);
766 if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT)
767 return (false);
768 if (csp->csp_flags & CSP_F_SEPARATE_AAD)
769 return (false);
770 if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
771 csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
772 csp->csp_auth_mlen != 0)
773 return (false);
774 break;
775 case CSP_MODE_CIPHER:
776 if (!alg_is_cipher(csp->csp_cipher_alg))
777 return (false);
778 if (csp->csp_flags & CSP_F_SEPARATE_AAD)
779 return (false);
780 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
781 if (csp->csp_cipher_klen == 0)
782 return (false);
783 if (csp->csp_ivlen == 0)
784 return (false);
785 }
786 if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
787 return (false);
788 if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
789 csp->csp_auth_mlen != 0)
790 return (false);
791 break;
792 case CSP_MODE_DIGEST:
793 if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
794 return (false);
795
796 if (csp->csp_flags & CSP_F_SEPARATE_AAD)
797 return (false);
798
799 /* IV is optional for digests (e.g. GMAC). */
800 switch (csp->csp_auth_alg) {
801 case CRYPTO_AES_CCM_CBC_MAC:
802 if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
803 return (false);
804 break;
805 case CRYPTO_AES_NIST_GMAC:
806 if (csp->csp_ivlen != AES_GCM_IV_LEN)
807 return (false);
808 break;
809 default:
810 if (csp->csp_ivlen != 0)
811 return (false);
812 break;
813 }
814
815 if (!alg_is_digest(csp->csp_auth_alg))
816 return (false);
817
818 /* Key is optional for BLAKE2 digests. */
819 if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
820 csp->csp_auth_alg == CRYPTO_BLAKE2S)
821 ;
822 else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
823 if (csp->csp_auth_klen == 0)
824 return (false);
825 } else {
826 if (csp->csp_auth_klen != 0)
827 return (false);
828 }
829 if (csp->csp_auth_mlen != 0) {
830 axf = crypto_auth_hash(csp);
831 if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
832 return (false);
833
834 if (csp->csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC &&
835 !ccm_tag_length_valid(csp->csp_auth_mlen))
836 return (false);
837 }
838 break;
839 case CSP_MODE_AEAD:
840 if (!alg_is_aead(csp->csp_cipher_alg))
841 return (false);
842 if (csp->csp_cipher_klen == 0)
843 return (false);
844 if (csp->csp_ivlen == 0 ||
845 csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
846 return (false);
847 if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
848 return (false);
849
850 switch (csp->csp_cipher_alg) {
851 case CRYPTO_AES_CCM_16:
852 if (csp->csp_auth_mlen != 0 &&
853 !ccm_tag_length_valid(csp->csp_auth_mlen))
854 return (false);
855
856 if (csp->csp_ivlen < 7 || csp->csp_ivlen > 13)
857 return (false);
858 break;
859 case CRYPTO_AES_NIST_GCM_16:
860 if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
861 return (false);
862
863 if (csp->csp_ivlen != AES_GCM_IV_LEN)
864 return (false);
865 break;
866 case CRYPTO_CHACHA20_POLY1305:
867 if (csp->csp_ivlen != 8 && csp->csp_ivlen != 12)
868 return (false);
869 if (csp->csp_auth_mlen > POLY1305_HASH_LEN)
870 return (false);
871 break;
872 case CRYPTO_XCHACHA20_POLY1305:
873 if (csp->csp_ivlen != XCHACHA20_POLY1305_IV_LEN)
874 return (false);
875 if (csp->csp_auth_mlen > POLY1305_HASH_LEN)
876 return (false);
877 break;
878 }
879 break;
880 case CSP_MODE_ETA:
881 if (!alg_is_cipher(csp->csp_cipher_alg))
882 return (false);
883 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
884 if (csp->csp_cipher_klen == 0)
885 return (false);
886 if (csp->csp_ivlen == 0)
887 return (false);
888 }
889 if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
890 return (false);
891 if (!alg_is_digest(csp->csp_auth_alg))
892 return (false);
893
894 /* Key is optional for BLAKE2 digests. */
895 if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
896 csp->csp_auth_alg == CRYPTO_BLAKE2S)
897 ;
898 else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
899 if (csp->csp_auth_klen == 0)
900 return (false);
901 } else {
902 if (csp->csp_auth_klen != 0)
903 return (false);
904 }
905 if (csp->csp_auth_mlen != 0) {
906 axf = crypto_auth_hash(csp);
907 if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
908 return (false);
909 }
910 break;
911 default:
912 return (false);
913 }
914
915 return (true);
916 }
917
918 /*
919 * Delete a session after it has been detached from its driver.
920 */
921 static void
crypto_deletesession(crypto_session_t cses)922 crypto_deletesession(crypto_session_t cses)
923 {
924 struct cryptocap *cap;
925
926 cap = cses->cap;
927
928 zfree(cses, M_CRYPTO_DATA);
929
930 CRYPTO_DRIVER_LOCK();
931 cap->cc_sessions--;
932 if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
933 wakeup(cap);
934 CRYPTO_DRIVER_UNLOCK();
935 cap_rele(cap);
936 }
937
938 /*
939 * Create a new session. The crid argument specifies a crypto
940 * driver to use or constraints on a driver to select (hardware
941 * only, software only, either). Whatever driver is selected
942 * must be capable of the requested crypto algorithms.
943 */
944 int
crypto_newsession(crypto_session_t * cses,const struct crypto_session_params * csp,int crid)945 crypto_newsession(crypto_session_t *cses,
946 const struct crypto_session_params *csp, int crid)
947 {
948 static uint64_t sessid = 0;
949 crypto_session_t res;
950 struct cryptocap *cap;
951 int err;
952
953 if (!check_csp(csp))
954 return (EINVAL);
955
956 res = NULL;
957
958 CRYPTO_DRIVER_LOCK();
959 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
960 /*
961 * Use specified driver; verify it is capable.
962 */
963 cap = crypto_checkdriver(crid);
964 if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
965 cap = NULL;
966 } else {
967 /*
968 * No requested driver; select based on crid flags.
969 */
970 cap = crypto_select_driver(csp, crid);
971 }
972 if (cap == NULL) {
973 CRYPTO_DRIVER_UNLOCK();
974 CRYPTDEB("no driver");
975 return (EOPNOTSUPP);
976 }
977 cap_ref(cap);
978 cap->cc_sessions++;
979 CRYPTO_DRIVER_UNLOCK();
980
981 /* Allocate a single block for the generic session and driver softc. */
982 res = malloc(sizeof(*res) + cap->cc_session_size, M_CRYPTO_DATA,
983 M_WAITOK | M_ZERO);
984 res->cap = cap;
985 res->csp = *csp;
986 res->id = atomic_fetchadd_64(&sessid, 1);
987
988 /* Call the driver initialization routine. */
989 err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
990 if (err != 0) {
991 CRYPTDEB("dev newsession failed: %d", err);
992 crypto_deletesession(res);
993 return (err);
994 }
995
996 *cses = res;
997 return (0);
998 }
999
1000 /*
1001 * Delete an existing session (or a reserved session on an unregistered
1002 * driver).
1003 */
1004 void
crypto_freesession(crypto_session_t cses)1005 crypto_freesession(crypto_session_t cses)
1006 {
1007 struct cryptocap *cap;
1008
1009 if (cses == NULL)
1010 return;
1011
1012 cap = cses->cap;
1013
1014 /* Call the driver cleanup routine, if available. */
1015 CRYPTODEV_FREESESSION(cap->cc_dev, cses);
1016
1017 crypto_deletesession(cses);
1018 }
1019
1020 /*
1021 * Return a new driver id. Registers a driver with the system so that
1022 * it can be probed by subsequent sessions.
1023 */
1024 int32_t
crypto_get_driverid(device_t dev,size_t sessionsize,int flags)1025 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
1026 {
1027 struct cryptocap *cap, **newdrv;
1028 int i;
1029
1030 if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
1031 device_printf(dev,
1032 "no flags specified when registering driver\n");
1033 return -1;
1034 }
1035
1036 cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1037 cap->cc_dev = dev;
1038 cap->cc_session_size = sessionsize;
1039 cap->cc_flags = flags;
1040 refcount_init(&cap->cc_refs, 1);
1041
1042 CRYPTO_DRIVER_LOCK();
1043 for (;;) {
1044 for (i = 0; i < crypto_drivers_size; i++) {
1045 if (crypto_drivers[i] == NULL)
1046 break;
1047 }
1048
1049 if (i < crypto_drivers_size)
1050 break;
1051
1052 /* Out of entries, allocate some more. */
1053
1054 if (2 * crypto_drivers_size <= crypto_drivers_size) {
1055 CRYPTO_DRIVER_UNLOCK();
1056 printf("crypto: driver count wraparound!\n");
1057 cap_rele(cap);
1058 return (-1);
1059 }
1060 CRYPTO_DRIVER_UNLOCK();
1061
1062 newdrv = malloc(2 * crypto_drivers_size *
1063 sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1064
1065 CRYPTO_DRIVER_LOCK();
1066 memcpy(newdrv, crypto_drivers,
1067 crypto_drivers_size * sizeof(*crypto_drivers));
1068
1069 crypto_drivers_size *= 2;
1070
1071 free(crypto_drivers, M_CRYPTO_DATA);
1072 crypto_drivers = newdrv;
1073 }
1074
1075 cap->cc_hid = i;
1076 crypto_drivers[i] = cap;
1077 CRYPTO_DRIVER_UNLOCK();
1078
1079 if (bootverbose)
1080 printf("crypto: assign %s driver id %u, flags 0x%x\n",
1081 device_get_nameunit(dev), i, flags);
1082
1083 return i;
1084 }
1085
1086 /*
1087 * Lookup a driver by name. We match against the full device
1088 * name and unit, and against just the name. The latter gives
1089 * us a simple widlcarding by device name. On success return the
1090 * driver/hardware identifier; otherwise return -1.
1091 */
1092 int
crypto_find_driver(const char * match)1093 crypto_find_driver(const char *match)
1094 {
1095 struct cryptocap *cap;
1096 int i, len = strlen(match);
1097
1098 CRYPTO_DRIVER_LOCK();
1099 for (i = 0; i < crypto_drivers_size; i++) {
1100 if (crypto_drivers[i] == NULL)
1101 continue;
1102 cap = crypto_drivers[i];
1103 if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
1104 strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
1105 CRYPTO_DRIVER_UNLOCK();
1106 return (i);
1107 }
1108 }
1109 CRYPTO_DRIVER_UNLOCK();
1110 return (-1);
1111 }
1112
1113 /*
1114 * Return the device_t for the specified driver or NULL
1115 * if the driver identifier is invalid.
1116 */
1117 device_t
crypto_find_device_byhid(int hid)1118 crypto_find_device_byhid(int hid)
1119 {
1120 struct cryptocap *cap;
1121 device_t dev;
1122
1123 dev = NULL;
1124 CRYPTO_DRIVER_LOCK();
1125 cap = crypto_checkdriver(hid);
1126 if (cap != NULL)
1127 dev = cap->cc_dev;
1128 CRYPTO_DRIVER_UNLOCK();
1129 return (dev);
1130 }
1131
1132 /*
1133 * Return the device/driver capabilities.
1134 */
1135 int
crypto_getcaps(int hid)1136 crypto_getcaps(int hid)
1137 {
1138 struct cryptocap *cap;
1139 int flags;
1140
1141 flags = 0;
1142 CRYPTO_DRIVER_LOCK();
1143 cap = crypto_checkdriver(hid);
1144 if (cap != NULL)
1145 flags = cap->cc_flags;
1146 CRYPTO_DRIVER_UNLOCK();
1147 return (flags);
1148 }
1149
1150 /*
1151 * Unregister all algorithms associated with a crypto driver.
1152 * If there are pending sessions using it, leave enough information
1153 * around so that subsequent calls using those sessions will
1154 * correctly detect the driver has been unregistered and reroute
1155 * requests.
1156 */
1157 int
crypto_unregister_all(uint32_t driverid)1158 crypto_unregister_all(uint32_t driverid)
1159 {
1160 struct cryptocap *cap;
1161
1162 CRYPTO_DRIVER_LOCK();
1163 cap = crypto_checkdriver(driverid);
1164 if (cap == NULL) {
1165 CRYPTO_DRIVER_UNLOCK();
1166 return (EINVAL);
1167 }
1168
1169 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
1170 crypto_drivers[driverid] = NULL;
1171
1172 /*
1173 * XXX: This doesn't do anything to kick sessions that
1174 * have no pending operations.
1175 */
1176 while (cap->cc_sessions != 0)
1177 mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
1178 CRYPTO_DRIVER_UNLOCK();
1179 cap_rele(cap);
1180
1181 return (0);
1182 }
1183
1184 /*
1185 * Clear blockage on a driver. The what parameter indicates whether
1186 * the driver is now ready for cryptop's and/or cryptokop's.
1187 */
1188 int
crypto_unblock(uint32_t driverid,int what)1189 crypto_unblock(uint32_t driverid, int what)
1190 {
1191 struct cryptocap *cap;
1192 int err;
1193
1194 CRYPTO_Q_LOCK();
1195 cap = crypto_checkdriver(driverid);
1196 if (cap != NULL) {
1197 if (what & CRYPTO_SYMQ)
1198 cap->cc_qblocked = 0;
1199 if (crp_sleep)
1200 wakeup_one(&crp_q);
1201 err = 0;
1202 } else
1203 err = EINVAL;
1204 CRYPTO_Q_UNLOCK();
1205
1206 return err;
1207 }
1208
1209 size_t
crypto_buffer_len(struct crypto_buffer * cb)1210 crypto_buffer_len(struct crypto_buffer *cb)
1211 {
1212 switch (cb->cb_type) {
1213 case CRYPTO_BUF_CONTIG:
1214 return (cb->cb_buf_len);
1215 case CRYPTO_BUF_MBUF:
1216 if (cb->cb_mbuf->m_flags & M_PKTHDR)
1217 return (cb->cb_mbuf->m_pkthdr.len);
1218 return (m_length(cb->cb_mbuf, NULL));
1219 case CRYPTO_BUF_SINGLE_MBUF:
1220 return (cb->cb_mbuf->m_len);
1221 case CRYPTO_BUF_VMPAGE:
1222 return (cb->cb_vm_page_len);
1223 case CRYPTO_BUF_UIO:
1224 return (cb->cb_uio->uio_resid);
1225 default:
1226 return (0);
1227 }
1228 }
1229
1230 #ifdef INVARIANTS
1231 /* Various sanity checks on crypto requests. */
1232 static void
cb_sanity(struct crypto_buffer * cb,const char * name)1233 cb_sanity(struct crypto_buffer *cb, const char *name)
1234 {
1235 KASSERT(cb->cb_type > CRYPTO_BUF_NONE && cb->cb_type <= CRYPTO_BUF_LAST,
1236 ("incoming crp with invalid %s buffer type", name));
1237 switch (cb->cb_type) {
1238 case CRYPTO_BUF_CONTIG:
1239 KASSERT(cb->cb_buf_len >= 0,
1240 ("incoming crp with -ve %s buffer length", name));
1241 break;
1242 case CRYPTO_BUF_VMPAGE:
1243 KASSERT(CRYPTO_HAS_VMPAGE,
1244 ("incoming crp uses dmap on supported arch"));
1245 KASSERT(cb->cb_vm_page_len >= 0,
1246 ("incoming crp with -ve %s buffer length", name));
1247 KASSERT(cb->cb_vm_page_offset >= 0,
1248 ("incoming crp with -ve %s buffer offset", name));
1249 KASSERT(cb->cb_vm_page_offset < PAGE_SIZE,
1250 ("incoming crp with %s buffer offset greater than page size"
1251 , name));
1252 break;
1253 default:
1254 break;
1255 }
1256 }
1257
1258 static void
crp_sanity(struct cryptop * crp)1259 crp_sanity(struct cryptop *crp)
1260 {
1261 struct crypto_session_params *csp;
1262 struct crypto_buffer *out;
1263 size_t ilen, len, olen;
1264
1265 KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
1266 KASSERT(crp->crp_obuf.cb_type >= CRYPTO_BUF_NONE &&
1267 crp->crp_obuf.cb_type <= CRYPTO_BUF_LAST,
1268 ("incoming crp with invalid output buffer type"));
1269 KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
1270
1271 csp = &crp->crp_session->csp;
1272 cb_sanity(&crp->crp_buf, "input");
1273 ilen = crypto_buffer_len(&crp->crp_buf);
1274 olen = ilen;
1275 out = NULL;
1276 if (csp->csp_flags & CSP_F_SEPARATE_OUTPUT) {
1277 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE) {
1278 cb_sanity(&crp->crp_obuf, "output");
1279 out = &crp->crp_obuf;
1280 olen = crypto_buffer_len(out);
1281 }
1282 } else
1283 KASSERT(crp->crp_obuf.cb_type == CRYPTO_BUF_NONE,
1284 ("incoming crp with separate output buffer "
1285 "but no session support"));
1286
1287 switch (csp->csp_mode) {
1288 case CSP_MODE_COMPRESS:
1289 KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
1290 crp->crp_op == CRYPTO_OP_DECOMPRESS,
1291 ("invalid compression op %x", crp->crp_op));
1292 break;
1293 case CSP_MODE_CIPHER:
1294 KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
1295 crp->crp_op == CRYPTO_OP_DECRYPT,
1296 ("invalid cipher op %x", crp->crp_op));
1297 break;
1298 case CSP_MODE_DIGEST:
1299 KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
1300 crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
1301 ("invalid digest op %x", crp->crp_op));
1302 break;
1303 case CSP_MODE_AEAD:
1304 KASSERT(crp->crp_op ==
1305 (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1306 crp->crp_op ==
1307 (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1308 ("invalid AEAD op %x", crp->crp_op));
1309 KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
1310 ("AEAD without a separate IV"));
1311 break;
1312 case CSP_MODE_ETA:
1313 KASSERT(crp->crp_op ==
1314 (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
1315 crp->crp_op ==
1316 (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
1317 ("invalid ETA op %x", crp->crp_op));
1318 break;
1319 }
1320 if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1321 if (crp->crp_aad == NULL) {
1322 KASSERT(crp->crp_aad_start == 0 ||
1323 crp->crp_aad_start < ilen,
1324 ("invalid AAD start"));
1325 KASSERT(crp->crp_aad_length != 0 ||
1326 crp->crp_aad_start == 0,
1327 ("AAD with zero length and non-zero start"));
1328 KASSERT(crp->crp_aad_length == 0 ||
1329 crp->crp_aad_start + crp->crp_aad_length <= ilen,
1330 ("AAD outside input length"));
1331 } else {
1332 KASSERT(csp->csp_flags & CSP_F_SEPARATE_AAD,
1333 ("session doesn't support separate AAD buffer"));
1334 KASSERT(crp->crp_aad_start == 0,
1335 ("separate AAD buffer with non-zero AAD start"));
1336 KASSERT(crp->crp_aad_length != 0,
1337 ("separate AAD buffer with zero length"));
1338 }
1339 } else {
1340 KASSERT(crp->crp_aad == NULL && crp->crp_aad_start == 0 &&
1341 crp->crp_aad_length == 0,
1342 ("AAD region in request not supporting AAD"));
1343 }
1344 if (csp->csp_ivlen == 0) {
1345 KASSERT((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0,
1346 ("IV_SEPARATE set when IV isn't used"));
1347 KASSERT(crp->crp_iv_start == 0,
1348 ("crp_iv_start set when IV isn't used"));
1349 } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
1350 KASSERT(crp->crp_iv_start == 0,
1351 ("IV_SEPARATE used with non-zero IV start"));
1352 } else {
1353 KASSERT(crp->crp_iv_start < ilen,
1354 ("invalid IV start"));
1355 KASSERT(crp->crp_iv_start + csp->csp_ivlen <= ilen,
1356 ("IV outside buffer length"));
1357 }
1358 /* XXX: payload_start of 0 should always be < ilen? */
1359 KASSERT(crp->crp_payload_start == 0 ||
1360 crp->crp_payload_start < ilen,
1361 ("invalid payload start"));
1362 KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
1363 ilen, ("payload outside input buffer"));
1364 if (out == NULL) {
1365 KASSERT(crp->crp_payload_output_start == 0,
1366 ("payload output start non-zero without output buffer"));
1367 } else if (csp->csp_mode == CSP_MODE_DIGEST) {
1368 KASSERT(!(crp->crp_op & CRYPTO_OP_VERIFY_DIGEST),
1369 ("digest verify with separate output buffer"));
1370 KASSERT(crp->crp_payload_output_start == 0,
1371 ("digest operation with non-zero payload output start"));
1372 } else {
1373 KASSERT(crp->crp_payload_output_start == 0 ||
1374 crp->crp_payload_output_start < olen,
1375 ("invalid payload output start"));
1376 KASSERT(crp->crp_payload_output_start +
1377 crp->crp_payload_length <= olen,
1378 ("payload outside output buffer"));
1379 }
1380 if (csp->csp_mode == CSP_MODE_DIGEST ||
1381 csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
1382 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST)
1383 len = ilen;
1384 else
1385 len = olen;
1386 KASSERT(crp->crp_digest_start == 0 ||
1387 crp->crp_digest_start < len,
1388 ("invalid digest start"));
1389 /* XXX: For the mlen == 0 case this check isn't perfect. */
1390 KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <= len,
1391 ("digest outside buffer"));
1392 } else {
1393 KASSERT(crp->crp_digest_start == 0,
1394 ("non-zero digest start for request without a digest"));
1395 }
1396 if (csp->csp_cipher_klen != 0)
1397 KASSERT(csp->csp_cipher_key != NULL ||
1398 crp->crp_cipher_key != NULL,
1399 ("cipher request without a key"));
1400 if (csp->csp_auth_klen != 0)
1401 KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
1402 ("auth request without a key"));
1403 KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
1404 }
1405 #endif
1406
1407 static int
crypto_dispatch_one(struct cryptop * crp,int hint)1408 crypto_dispatch_one(struct cryptop *crp, int hint)
1409 {
1410 struct cryptocap *cap;
1411 int result;
1412
1413 #ifdef INVARIANTS
1414 crp_sanity(crp);
1415 #endif
1416 CRYPTOSTAT_INC(cs_ops);
1417
1418 crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1419
1420 /*
1421 * Caller marked the request to be processed immediately; dispatch it
1422 * directly to the driver unless the driver is currently blocked, in
1423 * which case it is queued for deferred dispatch.
1424 */
1425 cap = crp->crp_session->cap;
1426 if (!atomic_load_int(&cap->cc_qblocked)) {
1427 result = crypto_invoke(cap, crp, hint);
1428 if (result != ERESTART)
1429 return (result);
1430
1431 /*
1432 * The driver ran out of resources, put the request on the
1433 * queue.
1434 */
1435 }
1436 crypto_batch_enqueue(crp);
1437 return (0);
1438 }
1439
1440 int
crypto_dispatch(struct cryptop * crp)1441 crypto_dispatch(struct cryptop *crp)
1442 {
1443 return (crypto_dispatch_one(crp, 0));
1444 }
1445
1446 int
crypto_dispatch_async(struct cryptop * crp,int flags)1447 crypto_dispatch_async(struct cryptop *crp, int flags)
1448 {
1449 struct crypto_ret_worker *ret_worker;
1450
1451 if (!CRYPTO_SESS_SYNC(crp->crp_session)) {
1452 /*
1453 * The driver issues completions asynchonously, don't bother
1454 * deferring dispatch to a worker thread.
1455 */
1456 return (crypto_dispatch(crp));
1457 }
1458
1459 #ifdef INVARIANTS
1460 crp_sanity(crp);
1461 #endif
1462 CRYPTOSTAT_INC(cs_ops);
1463
1464 crp->crp_retw_id = crp->crp_session->id % crypto_workers_num;
1465 if ((flags & CRYPTO_ASYNC_ORDERED) != 0) {
1466 crp->crp_flags |= CRYPTO_F_ASYNC_ORDERED;
1467 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1468 CRYPTO_RETW_LOCK(ret_worker);
1469 crp->crp_seq = ret_worker->reorder_ops++;
1470 CRYPTO_RETW_UNLOCK(ret_worker);
1471 }
1472 TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
1473 taskqueue_enqueue(crypto_tq, &crp->crp_task);
1474 return (0);
1475 }
1476
1477 void
crypto_dispatch_batch(struct cryptopq * crpq,int flags)1478 crypto_dispatch_batch(struct cryptopq *crpq, int flags)
1479 {
1480 struct cryptop *crp;
1481 int hint;
1482
1483 while ((crp = TAILQ_FIRST(crpq)) != NULL) {
1484 hint = TAILQ_NEXT(crp, crp_next) != NULL ? CRYPTO_HINT_MORE : 0;
1485 TAILQ_REMOVE(crpq, crp, crp_next);
1486 if (crypto_dispatch_one(crp, hint) != 0)
1487 crypto_batch_enqueue(crp);
1488 }
1489 }
1490
1491 static void
crypto_batch_enqueue(struct cryptop * crp)1492 crypto_batch_enqueue(struct cryptop *crp)
1493 {
1494
1495 CRYPTO_Q_LOCK();
1496 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
1497 if (crp_sleep)
1498 wakeup_one(&crp_q);
1499 CRYPTO_Q_UNLOCK();
1500 }
1501
1502 static void
crypto_task_invoke(void * ctx,int pending)1503 crypto_task_invoke(void *ctx, int pending)
1504 {
1505 struct cryptocap *cap;
1506 struct cryptop *crp;
1507 int result;
1508
1509 crp = (struct cryptop *)ctx;
1510 cap = crp->crp_session->cap;
1511 result = crypto_invoke(cap, crp, 0);
1512 if (result == ERESTART)
1513 crypto_batch_enqueue(crp);
1514 }
1515
1516 /*
1517 * Dispatch a crypto request to the appropriate crypto devices.
1518 */
1519 static int
crypto_invoke(struct cryptocap * cap,struct cryptop * crp,int hint)1520 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
1521 {
1522 int error;
1523
1524 KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
1525 KASSERT(crp->crp_callback != NULL,
1526 ("%s: crp->crp_callback == NULL", __func__));
1527 KASSERT(crp->crp_session != NULL,
1528 ("%s: crp->crp_session == NULL", __func__));
1529
1530 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1531 struct crypto_session_params csp;
1532 crypto_session_t nses;
1533
1534 /*
1535 * Driver has unregistered; migrate the session and return
1536 * an error to the caller so they'll resubmit the op.
1537 *
1538 * XXX: What if there are more already queued requests for this
1539 * session?
1540 *
1541 * XXX: Real solution is to make sessions refcounted
1542 * and force callers to hold a reference when
1543 * assigning to crp_session. Could maybe change
1544 * crypto_getreq to accept a session pointer to make
1545 * that work. Alternatively, we could abandon the
1546 * notion of rewriting crp_session in requests forcing
1547 * the caller to deal with allocating a new session.
1548 * Perhaps provide a method to allow a crp's session to
1549 * be swapped that callers could use.
1550 */
1551 csp = crp->crp_session->csp;
1552 crypto_freesession(crp->crp_session);
1553
1554 /*
1555 * XXX: Key pointers may no longer be valid. If we
1556 * really want to support this we need to define the
1557 * KPI such that 'csp' is required to be valid for the
1558 * duration of a session by the caller perhaps.
1559 *
1560 * XXX: If the keys have been changed this will reuse
1561 * the old keys. This probably suggests making
1562 * rekeying more explicit and updating the key
1563 * pointers in 'csp' when the keys change.
1564 */
1565 if (crypto_newsession(&nses, &csp,
1566 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
1567 crp->crp_session = nses;
1568
1569 crp->crp_etype = EAGAIN;
1570 crypto_done(crp);
1571 error = 0;
1572 } else {
1573 /*
1574 * Invoke the driver to process the request. Errors are
1575 * signaled by setting crp_etype before invoking the completion
1576 * callback.
1577 */
1578 error = CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
1579 KASSERT(error == 0 || error == ERESTART,
1580 ("%s: invalid error %d from CRYPTODEV_PROCESS",
1581 __func__, error));
1582 }
1583 return (error);
1584 }
1585
1586 void
crypto_destroyreq(struct cryptop * crp)1587 crypto_destroyreq(struct cryptop *crp)
1588 {
1589 #ifdef DIAGNOSTIC
1590 {
1591 struct cryptop *crp2;
1592 struct crypto_ret_worker *ret_worker;
1593
1594 if (!crypto_destroyreq_check)
1595 return;
1596
1597 CRYPTO_Q_LOCK();
1598 TAILQ_FOREACH(crp2, &crp_q, crp_next) {
1599 KASSERT(crp2 != crp,
1600 ("Freeing cryptop from the crypto queue (%p).",
1601 crp));
1602 }
1603 CRYPTO_Q_UNLOCK();
1604
1605 FOREACH_CRYPTO_RETW(ret_worker) {
1606 CRYPTO_RETW_LOCK(ret_worker);
1607 TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
1608 KASSERT(crp2 != crp,
1609 ("Freeing cryptop from the return queue (%p).",
1610 crp));
1611 }
1612 CRYPTO_RETW_UNLOCK(ret_worker);
1613 }
1614 }
1615 #endif
1616 }
1617
1618 void
crypto_freereq(struct cryptop * crp)1619 crypto_freereq(struct cryptop *crp)
1620 {
1621 if (crp == NULL)
1622 return;
1623
1624 crypto_destroyreq(crp);
1625 uma_zfree(cryptop_zone, crp);
1626 }
1627
1628 void
crypto_initreq(struct cryptop * crp,crypto_session_t cses)1629 crypto_initreq(struct cryptop *crp, crypto_session_t cses)
1630 {
1631 memset(crp, 0, sizeof(*crp));
1632 crp->crp_session = cses;
1633 }
1634
1635 struct cryptop *
crypto_getreq(crypto_session_t cses,int how)1636 crypto_getreq(crypto_session_t cses, int how)
1637 {
1638 struct cryptop *crp;
1639
1640 MPASS(how == M_WAITOK || how == M_NOWAIT);
1641 crp = uma_zalloc(cryptop_zone, how);
1642 if (crp != NULL)
1643 crypto_initreq(crp, cses);
1644 return (crp);
1645 }
1646
1647 /*
1648 * Clone a crypto request, but associate it with the specified session
1649 * rather than inheriting the session from the original request. The
1650 * fields describing the request buffers are copied, but not the
1651 * opaque field or callback function.
1652 */
1653 struct cryptop *
crypto_clonereq(struct cryptop * crp,crypto_session_t cses,int how)1654 crypto_clonereq(struct cryptop *crp, crypto_session_t cses, int how)
1655 {
1656 struct cryptop *new;
1657
1658 new = crypto_getreq(cses, how);
1659 if (new == NULL)
1660 return (NULL);
1661
1662 memcpy(&new->crp_startcopy, &crp->crp_startcopy,
1663 __rangeof(struct cryptop, crp_startcopy, crp_endcopy));
1664 return (new);
1665 }
1666
1667 /*
1668 * Invoke the callback on behalf of the driver.
1669 */
1670 void
crypto_done(struct cryptop * crp)1671 crypto_done(struct cryptop *crp)
1672 {
1673 if (crp->crp_etype == 0) {
1674 switch (crp->crp_session->csp.csp_mode) {
1675 case CSP_MODE_DIGEST:
1676 case CSP_MODE_AEAD:
1677 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0)
1678 KFAIL_POINT_CODE(_debug_fail_point_crypto,
1679 inject_badmsg, crp->crp_etype = EBADMSG);
1680 break;
1681 }
1682 }
1683
1684 if (crp->crp_etype != 0)
1685 CRYPTOSTAT_INC(cs_errs);
1686
1687 /*
1688 * CBIMM means unconditionally do the callback immediately;
1689 * CBIFSYNC means do the callback immediately only if the
1690 * operation was done synchronously. Both are used to avoid
1691 * doing extraneous context switches; the latter is mostly
1692 * used with the software crypto driver.
1693 */
1694 if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) == 0 &&
1695 ((crp->crp_flags & CRYPTO_F_CBIMM) != 0 ||
1696 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) != 0 &&
1697 CRYPTO_SESS_SYNC(crp->crp_session)))) {
1698 /*
1699 * Do the callback directly. This is ok when the
1700 * callback routine does very little (e.g. the
1701 * /dev/crypto callback method just does a wakeup).
1702 */
1703 crp->crp_callback(crp);
1704 } else {
1705 struct crypto_ret_worker *ret_worker;
1706 bool wake;
1707
1708 ret_worker = CRYPTO_RETW(crp->crp_retw_id);
1709
1710 /*
1711 * Normal case; queue the callback for the thread.
1712 */
1713 CRYPTO_RETW_LOCK(ret_worker);
1714 if ((crp->crp_flags & CRYPTO_F_ASYNC_ORDERED) != 0) {
1715 struct cryptop *tmp;
1716
1717 TAILQ_FOREACH_REVERSE(tmp,
1718 &ret_worker->crp_ordered_ret_q, cryptop_q,
1719 crp_next) {
1720 if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
1721 TAILQ_INSERT_AFTER(
1722 &ret_worker->crp_ordered_ret_q, tmp,
1723 crp, crp_next);
1724 break;
1725 }
1726 }
1727 if (tmp == NULL) {
1728 TAILQ_INSERT_HEAD(
1729 &ret_worker->crp_ordered_ret_q, crp,
1730 crp_next);
1731 }
1732
1733 wake = crp->crp_seq == ret_worker->reorder_cur_seq;
1734 } else {
1735 wake = TAILQ_EMPTY(&ret_worker->crp_ret_q);
1736 TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp,
1737 crp_next);
1738 }
1739
1740 if (wake)
1741 wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
1742 CRYPTO_RETW_UNLOCK(ret_worker);
1743 }
1744 }
1745
1746 /*
1747 * Terminate a thread at module unload. The process that
1748 * initiated this is waiting for us to signal that we're gone;
1749 * wake it up and exit. We use the driver table lock to insure
1750 * we don't do the wakeup before they're waiting. There is no
1751 * race here because the waiter sleeps on the proc lock for the
1752 * thread so it gets notified at the right time because of an
1753 * extra wakeup that's done in exit1().
1754 */
1755 static void
crypto_finis(void * chan)1756 crypto_finis(void *chan)
1757 {
1758 CRYPTO_DRIVER_LOCK();
1759 wakeup_one(chan);
1760 CRYPTO_DRIVER_UNLOCK();
1761 kthread_exit();
1762 }
1763
1764 /*
1765 * Crypto thread, dispatches crypto requests.
1766 */
1767 static void
crypto_dispatch_thread(void * arg __unused)1768 crypto_dispatch_thread(void *arg __unused)
1769 {
1770 struct cryptop *crp, *submit;
1771 struct cryptocap *cap;
1772 int result, hint;
1773
1774 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
1775 fpu_kern_thread(FPU_KERN_NORMAL);
1776 #endif
1777
1778 CRYPTO_Q_LOCK();
1779 for (;;) {
1780 /*
1781 * Find the first element in the queue that can be
1782 * processed and look-ahead to see if multiple ops
1783 * are ready for the same driver.
1784 */
1785 submit = NULL;
1786 hint = 0;
1787 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1788 cap = crp->crp_session->cap;
1789 /*
1790 * Driver cannot disappeared when there is an active
1791 * session.
1792 */
1793 KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1794 __func__, __LINE__));
1795 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
1796 /* Op needs to be migrated, process it. */
1797 if (submit == NULL)
1798 submit = crp;
1799 break;
1800 }
1801 if (!cap->cc_qblocked) {
1802 if (submit != NULL) {
1803 /*
1804 * We stop on finding another op,
1805 * regardless whether its for the same
1806 * driver or not. We could keep
1807 * searching the queue but it might be
1808 * better to just use a per-driver
1809 * queue instead.
1810 */
1811 if (submit->crp_session->cap == cap)
1812 hint = CRYPTO_HINT_MORE;
1813 } else {
1814 submit = crp;
1815 }
1816 break;
1817 }
1818 }
1819 if (submit != NULL) {
1820 TAILQ_REMOVE(&crp_q, submit, crp_next);
1821 cap = submit->crp_session->cap;
1822 KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
1823 __func__, __LINE__));
1824 CRYPTO_Q_UNLOCK();
1825 result = crypto_invoke(cap, submit, hint);
1826 CRYPTO_Q_LOCK();
1827 if (result == ERESTART) {
1828 /*
1829 * The driver ran out of resources, mark the
1830 * driver ``blocked'' for cryptop's and put
1831 * the request back in the queue. It would
1832 * best to put the request back where we got
1833 * it but that's hard so for now we put it
1834 * at the front. This should be ok; putting
1835 * it at the end does not work.
1836 */
1837 cap->cc_qblocked = 1;
1838 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1839 CRYPTOSTAT_INC(cs_blocks);
1840 }
1841 } else {
1842 /*
1843 * Nothing more to be processed. Sleep until we're
1844 * woken because there are more ops to process.
1845 * This happens either by submission or by a driver
1846 * becoming unblocked and notifying us through
1847 * crypto_unblock. Note that when we wakeup we
1848 * start processing each queue again from the
1849 * front. It's not clear that it's important to
1850 * preserve this ordering since ops may finish
1851 * out of order if dispatched to different devices
1852 * and some become blocked while others do not.
1853 */
1854 crp_sleep = 1;
1855 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
1856 crp_sleep = 0;
1857 if (cryptotd == NULL)
1858 break;
1859 CRYPTOSTAT_INC(cs_intrs);
1860 }
1861 }
1862 CRYPTO_Q_UNLOCK();
1863
1864 crypto_finis(&crp_q);
1865 }
1866
1867 /*
1868 * Crypto returns thread, does callbacks for processed crypto requests.
1869 * Callbacks are done here, rather than in the crypto drivers, because
1870 * callbacks typically are expensive and would slow interrupt handling.
1871 */
1872 static void
crypto_ret_thread(void * arg)1873 crypto_ret_thread(void *arg)
1874 {
1875 struct crypto_ret_worker *ret_worker = arg;
1876 struct cryptop *crpt;
1877
1878 CRYPTO_RETW_LOCK(ret_worker);
1879 for (;;) {
1880 /* Harvest return q's for completed ops */
1881 crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
1882 if (crpt != NULL) {
1883 if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
1884 TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
1885 ret_worker->reorder_cur_seq++;
1886 } else {
1887 crpt = NULL;
1888 }
1889 }
1890
1891 if (crpt == NULL) {
1892 crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
1893 if (crpt != NULL)
1894 TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
1895 }
1896
1897 if (crpt != NULL) {
1898 CRYPTO_RETW_UNLOCK(ret_worker);
1899 /*
1900 * Run callbacks unlocked.
1901 */
1902 if (crpt != NULL)
1903 crpt->crp_callback(crpt);
1904 CRYPTO_RETW_LOCK(ret_worker);
1905 } else {
1906 /*
1907 * Nothing more to be processed. Sleep until we're
1908 * woken because there are more returns to process.
1909 */
1910 msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
1911 "crypto_ret_wait", 0);
1912 if (ret_worker->td == NULL)
1913 break;
1914 CRYPTOSTAT_INC(cs_rets);
1915 }
1916 }
1917 CRYPTO_RETW_UNLOCK(ret_worker);
1918
1919 crypto_finis(&ret_worker->crp_ret_q);
1920 }
1921
1922 #ifdef DDB
1923 static void
db_show_drivers(void)1924 db_show_drivers(void)
1925 {
1926 int hid;
1927
1928 db_printf("%12s %4s %8s %2s\n"
1929 , "Device"
1930 , "Ses"
1931 , "Flags"
1932 , "QB"
1933 );
1934 for (hid = 0; hid < crypto_drivers_size; hid++) {
1935 const struct cryptocap *cap = crypto_drivers[hid];
1936 if (cap == NULL)
1937 continue;
1938 db_printf("%-12s %4u %08x %2u\n"
1939 , device_get_nameunit(cap->cc_dev)
1940 , cap->cc_sessions
1941 , cap->cc_flags
1942 , cap->cc_qblocked
1943 );
1944 }
1945 }
1946
DB_SHOW_COMMAND_FLAGS(crypto,db_show_crypto,DB_CMD_MEMSAFE)1947 DB_SHOW_COMMAND_FLAGS(crypto, db_show_crypto, DB_CMD_MEMSAFE)
1948 {
1949 struct cryptop *crp;
1950 struct crypto_ret_worker *ret_worker;
1951
1952 db_show_drivers();
1953 db_printf("\n");
1954
1955 db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
1956 "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
1957 "Device", "Callback");
1958 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1959 db_printf("%4u %08x %4u %4u %04x %8p %8p\n"
1960 , crp->crp_session->cap->cc_hid
1961 , (int) crypto_ses2caps(crp->crp_session)
1962 , crp->crp_olen
1963 , crp->crp_etype
1964 , crp->crp_flags
1965 , device_get_nameunit(crp->crp_session->cap->cc_dev)
1966 , crp->crp_callback
1967 );
1968 }
1969 FOREACH_CRYPTO_RETW(ret_worker) {
1970 db_printf("\n%8s %4s %4s %4s %8s\n",
1971 "ret_worker", "HID", "Etype", "Flags", "Callback");
1972 if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
1973 TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
1974 db_printf("%8td %4u %4u %04x %8p\n"
1975 , CRYPTO_RETW_ID(ret_worker)
1976 , crp->crp_session->cap->cc_hid
1977 , crp->crp_etype
1978 , crp->crp_flags
1979 , crp->crp_callback
1980 );
1981 }
1982 }
1983 }
1984 }
1985 #endif
1986
1987 int crypto_modevent(module_t mod, int type, void *unused);
1988
1989 /*
1990 * Initialization code, both for static and dynamic loading.
1991 * Note this is not invoked with the usual MODULE_DECLARE
1992 * mechanism but instead is listed as a dependency by the
1993 * cryptosoft driver. This guarantees proper ordering of
1994 * calls on module load/unload.
1995 */
1996 int
crypto_modevent(module_t mod,int type,void * unused)1997 crypto_modevent(module_t mod, int type, void *unused)
1998 {
1999 int error = EINVAL;
2000
2001 switch (type) {
2002 case MOD_LOAD:
2003 error = crypto_init();
2004 if (error == 0 && bootverbose)
2005 printf("crypto: <crypto core>\n");
2006 break;
2007 case MOD_UNLOAD:
2008 /*XXX disallow if active sessions */
2009 error = 0;
2010 crypto_destroy();
2011 return 0;
2012 }
2013 return error;
2014 }
2015 MODULE_VERSION(crypto, 1);
2016 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
2017