1 /*- 2 * Copyright (c) 2002-2006 Sam Leffler. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 14 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 15 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 16 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 17 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 18 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 20 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 22 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 */ 24 25 #include <sys/cdefs.h> 26 __FBSDID("$FreeBSD$"); 27 28 /* 29 * Cryptographic Subsystem. 30 * 31 * This code is derived from the Openbsd Cryptographic Framework (OCF) 32 * that has the copyright shown below. Very little of the original 33 * code remains. 34 */ 35 36 /*- 37 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 38 * 39 * This code was written by Angelos D. Keromytis in Athens, Greece, in 40 * February 2000. Network Security Technologies Inc. (NSTI) kindly 41 * supported the development of this code. 42 * 43 * Copyright (c) 2000, 2001 Angelos D. Keromytis 44 * 45 * Permission to use, copy, and modify this software with or without fee 46 * is hereby granted, provided that this entire notice is included in 47 * all source code copies of any software which is or includes a copy or 48 * modification of this software. 49 * 50 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 51 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 52 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 53 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 54 * PURPOSE. 55 */ 56 57 #define CRYPTO_TIMING /* enable timing support */ 58 59 #include "opt_ddb.h" 60 #include "opt_kdtrace.h" 61 62 #include <sys/param.h> 63 #include <sys/systm.h> 64 #include <sys/eventhandler.h> 65 #include <sys/kernel.h> 66 #include <sys/kthread.h> 67 #include <sys/lock.h> 68 #include <sys/module.h> 69 #include <sys/mutex.h> 70 #include <sys/malloc.h> 71 #include <sys/proc.h> 72 #include <sys/sdt.h> 73 #include <sys/sysctl.h> 74 75 #include <ddb/ddb.h> 76 77 #include <vm/uma.h> 78 #include <opencrypto/cryptodev.h> 79 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 80 81 #include <sys/kobj.h> 82 #include <sys/bus.h> 83 #include "cryptodev_if.h" 84 85 #if defined(__i386__) || defined(__amd64__) 86 #include <machine/pcb.h> 87 #endif 88 89 SDT_PROVIDER_DEFINE(opencrypto); 90 91 /* 92 * Crypto drivers register themselves by allocating a slot in the 93 * crypto_drivers table with crypto_get_driverid() and then registering 94 * each algorithm they support with crypto_register() and crypto_kregister(). 95 */ 96 static struct mtx crypto_drivers_mtx; /* lock on driver table */ 97 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 98 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 99 #define CRYPTO_DRIVER_ASSERT() mtx_assert(&crypto_drivers_mtx, MA_OWNED) 100 101 /* 102 * Crypto device/driver capabilities structure. 103 * 104 * Synchronization: 105 * (d) - protected by CRYPTO_DRIVER_LOCK() 106 * (q) - protected by CRYPTO_Q_LOCK() 107 * Not tagged fields are read-only. 108 */ 109 struct cryptocap { 110 device_t cc_dev; /* (d) device/driver */ 111 u_int32_t cc_sessions; /* (d) # of sessions */ 112 u_int32_t cc_koperations; /* (d) # os asym operations */ 113 /* 114 * Largest possible operator length (in bits) for each type of 115 * encryption algorithm. XXX not used 116 */ 117 u_int16_t cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1]; 118 u_int8_t cc_alg[CRYPTO_ALGORITHM_MAX + 1]; 119 u_int8_t cc_kalg[CRK_ALGORITHM_MAX + 1]; 120 121 int cc_flags; /* (d) flags */ 122 #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */ 123 int cc_qblocked; /* (q) symmetric q blocked */ 124 int cc_kqblocked; /* (q) asymmetric q blocked */ 125 }; 126 static struct cryptocap *crypto_drivers = NULL; 127 static int crypto_drivers_num = 0; 128 129 /* 130 * There are two queues for crypto requests; one for symmetric (e.g. 131 * cipher) operations and one for asymmetric (e.g. MOD)operations. 132 * A single mutex is used to lock access to both queues. We could 133 * have one per-queue but having one simplifies handling of block/unblock 134 * operations. 135 */ 136 static int crp_sleep = 0; 137 static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 138 static TAILQ_HEAD(,cryptkop) crp_kq; 139 static struct mtx crypto_q_mtx; 140 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 141 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 142 143 /* 144 * There are two queues for processing completed crypto requests; one 145 * for the symmetric and one for the asymmetric ops. We only need one 146 * but have two to avoid type futzing (cryptop vs. cryptkop). A single 147 * mutex is used to lock access to both queues. Note that this lock 148 * must be separate from the lock on request queues to insure driver 149 * callbacks don't generate lock order reversals. 150 */ 151 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 152 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 153 static struct mtx crypto_ret_q_mtx; 154 #define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 155 #define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 156 #define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq)) 157 158 static uma_zone_t cryptop_zone; 159 static uma_zone_t cryptodesc_zone; 160 161 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 162 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 163 &crypto_userasymcrypto, 0, 164 "Enable/disable user-mode access to asymmetric crypto support"); 165 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 166 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 167 &crypto_devallowsoft, 0, 168 "Enable/disable use of software asym crypto support"); 169 170 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 171 172 static void crypto_proc(void); 173 static struct proc *cryptoproc; 174 static void crypto_ret_proc(void); 175 static struct proc *cryptoretproc; 176 static void crypto_destroy(void); 177 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 178 static int crypto_kinvoke(struct cryptkop *krp, int flags); 179 180 static struct cryptostats cryptostats; 181 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 182 cryptostats, "Crypto system statistics"); 183 184 #ifdef CRYPTO_TIMING 185 static int crypto_timing = 0; 186 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 187 &crypto_timing, 0, "Enable/disable crypto timing support"); 188 #endif 189 190 static int 191 crypto_init(void) 192 { 193 int error; 194 195 mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", 196 MTX_DEF|MTX_QUIET); 197 198 TAILQ_INIT(&crp_q); 199 TAILQ_INIT(&crp_kq); 200 mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); 201 202 TAILQ_INIT(&crp_ret_q); 203 TAILQ_INIT(&crp_ret_kq); 204 mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF); 205 206 cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 207 0, 0, 0, 0, 208 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 209 cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 210 0, 0, 0, 0, 211 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 212 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 213 printf("crypto_init: cannot setup crypto zones\n"); 214 error = ENOMEM; 215 goto bad; 216 } 217 218 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 219 crypto_drivers = malloc(crypto_drivers_num * 220 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 221 if (crypto_drivers == NULL) { 222 printf("crypto_init: cannot setup crypto drivers\n"); 223 error = ENOMEM; 224 goto bad; 225 } 226 227 error = kproc_create((void (*)(void *)) crypto_proc, NULL, 228 &cryptoproc, 0, 0, "crypto"); 229 if (error) { 230 printf("crypto_init: cannot start crypto thread; error %d", 231 error); 232 goto bad; 233 } 234 235 error = kproc_create((void (*)(void *)) crypto_ret_proc, NULL, 236 &cryptoretproc, 0, 0, "crypto returns"); 237 if (error) { 238 printf("crypto_init: cannot start cryptoret thread; error %d", 239 error); 240 goto bad; 241 } 242 return 0; 243 bad: 244 crypto_destroy(); 245 return error; 246 } 247 248 /* 249 * Signal a crypto thread to terminate. We use the driver 250 * table lock to synchronize the sleep/wakeups so that we 251 * are sure the threads have terminated before we release 252 * the data structures they use. See crypto_finis below 253 * for the other half of this song-and-dance. 254 */ 255 static void 256 crypto_terminate(struct proc **pp, void *q) 257 { 258 struct proc *p; 259 260 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 261 p = *pp; 262 *pp = NULL; 263 if (p) { 264 wakeup_one(q); 265 PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 266 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 267 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 268 PROC_UNLOCK(p); 269 CRYPTO_DRIVER_LOCK(); 270 } 271 } 272 273 static void 274 crypto_destroy(void) 275 { 276 /* 277 * Terminate any crypto threads. 278 */ 279 CRYPTO_DRIVER_LOCK(); 280 crypto_terminate(&cryptoproc, &crp_q); 281 crypto_terminate(&cryptoretproc, &crp_ret_q); 282 CRYPTO_DRIVER_UNLOCK(); 283 284 /* XXX flush queues??? */ 285 286 /* 287 * Reclaim dynamically allocated resources. 288 */ 289 if (crypto_drivers != NULL) 290 free(crypto_drivers, M_CRYPTO_DATA); 291 292 if (cryptodesc_zone != NULL) 293 uma_zdestroy(cryptodesc_zone); 294 if (cryptop_zone != NULL) 295 uma_zdestroy(cryptop_zone); 296 mtx_destroy(&crypto_q_mtx); 297 mtx_destroy(&crypto_ret_q_mtx); 298 mtx_destroy(&crypto_drivers_mtx); 299 } 300 301 static struct cryptocap * 302 crypto_checkdriver(u_int32_t hid) 303 { 304 if (crypto_drivers == NULL) 305 return NULL; 306 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 307 } 308 309 /* 310 * Compare a driver's list of supported algorithms against another 311 * list; return non-zero if all algorithms are supported. 312 */ 313 static int 314 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri) 315 { 316 const struct cryptoini *cr; 317 318 /* See if all the algorithms are supported. */ 319 for (cr = cri; cr; cr = cr->cri_next) 320 if (cap->cc_alg[cr->cri_alg] == 0) 321 return 0; 322 return 1; 323 } 324 325 /* 326 * Select a driver for a new session that supports the specified 327 * algorithms and, optionally, is constrained according to the flags. 328 * The algorithm we use here is pretty stupid; just use the 329 * first driver that supports all the algorithms we need. If there 330 * are multiple drivers we choose the driver with the fewest active 331 * sessions. We prefer hardware-backed drivers to software ones. 332 * 333 * XXX We need more smarts here (in real life too, but that's 334 * XXX another story altogether). 335 */ 336 static struct cryptocap * 337 crypto_select_driver(const struct cryptoini *cri, int flags) 338 { 339 struct cryptocap *cap, *best; 340 int match, hid; 341 342 CRYPTO_DRIVER_ASSERT(); 343 344 /* 345 * Look first for hardware crypto devices if permitted. 346 */ 347 if (flags & CRYPTOCAP_F_HARDWARE) 348 match = CRYPTOCAP_F_HARDWARE; 349 else 350 match = CRYPTOCAP_F_SOFTWARE; 351 best = NULL; 352 again: 353 for (hid = 0; hid < crypto_drivers_num; hid++) { 354 cap = &crypto_drivers[hid]; 355 /* 356 * If it's not initialized, is in the process of 357 * going away, or is not appropriate (hardware 358 * or software based on match), then skip. 359 */ 360 if (cap->cc_dev == NULL || 361 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 362 (cap->cc_flags & match) == 0) 363 continue; 364 365 /* verify all the algorithms are supported. */ 366 if (driver_suitable(cap, cri)) { 367 if (best == NULL || 368 cap->cc_sessions < best->cc_sessions) 369 best = cap; 370 } 371 } 372 if (best != NULL) 373 return best; 374 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 375 /* sort of an Algol 68-style for loop */ 376 match = CRYPTOCAP_F_SOFTWARE; 377 goto again; 378 } 379 return best; 380 } 381 382 /* 383 * Create a new session. The crid argument specifies a crypto 384 * driver to use or constraints on a driver to select (hardware 385 * only, software only, either). Whatever driver is selected 386 * must be capable of the requested crypto algorithms. 387 */ 388 int 389 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid) 390 { 391 struct cryptocap *cap; 392 u_int32_t hid, lid; 393 int err; 394 395 CRYPTO_DRIVER_LOCK(); 396 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 397 /* 398 * Use specified driver; verify it is capable. 399 */ 400 cap = crypto_checkdriver(crid); 401 if (cap != NULL && !driver_suitable(cap, cri)) 402 cap = NULL; 403 } else { 404 /* 405 * No requested driver; select based on crid flags. 406 */ 407 cap = crypto_select_driver(cri, crid); 408 /* 409 * if NULL then can't do everything in one session. 410 * XXX Fix this. We need to inject a "virtual" session 411 * XXX layer right about here. 412 */ 413 } 414 if (cap != NULL) { 415 /* Call the driver initialization routine. */ 416 hid = cap - crypto_drivers; 417 lid = hid; /* Pass the driver ID. */ 418 err = CRYPTODEV_NEWSESSION(cap->cc_dev, &lid, cri); 419 if (err == 0) { 420 (*sid) = (cap->cc_flags & 0xff000000) 421 | (hid & 0x00ffffff); 422 (*sid) <<= 32; 423 (*sid) |= (lid & 0xffffffff); 424 cap->cc_sessions++; 425 } 426 } else 427 err = EINVAL; 428 CRYPTO_DRIVER_UNLOCK(); 429 return err; 430 } 431 432 static void 433 crypto_remove(struct cryptocap *cap) 434 { 435 436 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 437 if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 438 bzero(cap, sizeof(*cap)); 439 } 440 441 /* 442 * Delete an existing session (or a reserved session on an unregistered 443 * driver). 444 */ 445 int 446 crypto_freesession(u_int64_t sid) 447 { 448 struct cryptocap *cap; 449 u_int32_t hid; 450 int err; 451 452 CRYPTO_DRIVER_LOCK(); 453 454 if (crypto_drivers == NULL) { 455 err = EINVAL; 456 goto done; 457 } 458 459 /* Determine two IDs. */ 460 hid = CRYPTO_SESID2HID(sid); 461 462 if (hid >= crypto_drivers_num) { 463 err = ENOENT; 464 goto done; 465 } 466 cap = &crypto_drivers[hid]; 467 468 if (cap->cc_sessions) 469 cap->cc_sessions--; 470 471 /* Call the driver cleanup routine, if available. */ 472 err = CRYPTODEV_FREESESSION(cap->cc_dev, sid); 473 474 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 475 crypto_remove(cap); 476 477 done: 478 CRYPTO_DRIVER_UNLOCK(); 479 return err; 480 } 481 482 /* 483 * Return an unused driver id. Used by drivers prior to registering 484 * support for the algorithms they handle. 485 */ 486 int32_t 487 crypto_get_driverid(device_t dev, int flags) 488 { 489 struct cryptocap *newdrv; 490 int i; 491 492 if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 493 printf("%s: no flags specified when registering driver\n", 494 device_get_nameunit(dev)); 495 return -1; 496 } 497 498 CRYPTO_DRIVER_LOCK(); 499 500 for (i = 0; i < crypto_drivers_num; i++) { 501 if (crypto_drivers[i].cc_dev == NULL && 502 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 503 break; 504 } 505 } 506 507 /* Out of entries, allocate some more. */ 508 if (i == crypto_drivers_num) { 509 /* Be careful about wrap-around. */ 510 if (2 * crypto_drivers_num <= crypto_drivers_num) { 511 CRYPTO_DRIVER_UNLOCK(); 512 printf("crypto: driver count wraparound!\n"); 513 return -1; 514 } 515 516 newdrv = malloc(2 * crypto_drivers_num * 517 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 518 if (newdrv == NULL) { 519 CRYPTO_DRIVER_UNLOCK(); 520 printf("crypto: no space to expand driver table!\n"); 521 return -1; 522 } 523 524 bcopy(crypto_drivers, newdrv, 525 crypto_drivers_num * sizeof(struct cryptocap)); 526 527 crypto_drivers_num *= 2; 528 529 free(crypto_drivers, M_CRYPTO_DATA); 530 crypto_drivers = newdrv; 531 } 532 533 /* NB: state is zero'd on free */ 534 crypto_drivers[i].cc_sessions = 1; /* Mark */ 535 crypto_drivers[i].cc_dev = dev; 536 crypto_drivers[i].cc_flags = flags; 537 if (bootverbose) 538 printf("crypto: assign %s driver id %u, flags %u\n", 539 device_get_nameunit(dev), i, flags); 540 541 CRYPTO_DRIVER_UNLOCK(); 542 543 return i; 544 } 545 546 /* 547 * Lookup a driver by name. We match against the full device 548 * name and unit, and against just the name. The latter gives 549 * us a simple widlcarding by device name. On success return the 550 * driver/hardware identifier; otherwise return -1. 551 */ 552 int 553 crypto_find_driver(const char *match) 554 { 555 int i, len = strlen(match); 556 557 CRYPTO_DRIVER_LOCK(); 558 for (i = 0; i < crypto_drivers_num; i++) { 559 device_t dev = crypto_drivers[i].cc_dev; 560 if (dev == NULL || 561 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP)) 562 continue; 563 if (strncmp(match, device_get_nameunit(dev), len) == 0 || 564 strncmp(match, device_get_name(dev), len) == 0) 565 break; 566 } 567 CRYPTO_DRIVER_UNLOCK(); 568 return i < crypto_drivers_num ? i : -1; 569 } 570 571 /* 572 * Return the device_t for the specified driver or NULL 573 * if the driver identifier is invalid. 574 */ 575 device_t 576 crypto_find_device_byhid(int hid) 577 { 578 struct cryptocap *cap = crypto_checkdriver(hid); 579 return cap != NULL ? cap->cc_dev : NULL; 580 } 581 582 /* 583 * Return the device/driver capabilities. 584 */ 585 int 586 crypto_getcaps(int hid) 587 { 588 struct cryptocap *cap = crypto_checkdriver(hid); 589 return cap != NULL ? cap->cc_flags : 0; 590 } 591 592 /* 593 * Register support for a key-related algorithm. This routine 594 * is called once for each algorithm supported a driver. 595 */ 596 int 597 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags) 598 { 599 struct cryptocap *cap; 600 int err; 601 602 CRYPTO_DRIVER_LOCK(); 603 604 cap = crypto_checkdriver(driverid); 605 if (cap != NULL && 606 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 607 /* 608 * XXX Do some performance testing to determine placing. 609 * XXX We probably need an auxiliary data structure that 610 * XXX describes relative performances. 611 */ 612 613 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 614 if (bootverbose) 615 printf("crypto: %s registers key alg %u flags %u\n" 616 , device_get_nameunit(cap->cc_dev) 617 , kalg 618 , flags 619 ); 620 err = 0; 621 } else 622 err = EINVAL; 623 624 CRYPTO_DRIVER_UNLOCK(); 625 return err; 626 } 627 628 /* 629 * Register support for a non-key-related algorithm. This routine 630 * is called once for each such algorithm supported by a driver. 631 */ 632 int 633 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 634 u_int32_t flags) 635 { 636 struct cryptocap *cap; 637 int err; 638 639 CRYPTO_DRIVER_LOCK(); 640 641 cap = crypto_checkdriver(driverid); 642 /* NB: algorithms are in the range [1..max] */ 643 if (cap != NULL && 644 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 645 /* 646 * XXX Do some performance testing to determine placing. 647 * XXX We probably need an auxiliary data structure that 648 * XXX describes relative performances. 649 */ 650 651 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 652 cap->cc_max_op_len[alg] = maxoplen; 653 if (bootverbose) 654 printf("crypto: %s registers alg %u flags %u maxoplen %u\n" 655 , device_get_nameunit(cap->cc_dev) 656 , alg 657 , flags 658 , maxoplen 659 ); 660 cap->cc_sessions = 0; /* Unmark */ 661 err = 0; 662 } else 663 err = EINVAL; 664 665 CRYPTO_DRIVER_UNLOCK(); 666 return err; 667 } 668 669 static void 670 driver_finis(struct cryptocap *cap) 671 { 672 u_int32_t ses, kops; 673 674 CRYPTO_DRIVER_ASSERT(); 675 676 ses = cap->cc_sessions; 677 kops = cap->cc_koperations; 678 bzero(cap, sizeof(*cap)); 679 if (ses != 0 || kops != 0) { 680 /* 681 * If there are pending sessions, 682 * just mark as invalid. 683 */ 684 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 685 cap->cc_sessions = ses; 686 cap->cc_koperations = kops; 687 } 688 } 689 690 /* 691 * Unregister a crypto driver. If there are pending sessions using it, 692 * leave enough information around so that subsequent calls using those 693 * sessions will correctly detect the driver has been unregistered and 694 * reroute requests. 695 */ 696 int 697 crypto_unregister(u_int32_t driverid, int alg) 698 { 699 struct cryptocap *cap; 700 int i, err; 701 702 CRYPTO_DRIVER_LOCK(); 703 cap = crypto_checkdriver(driverid); 704 if (cap != NULL && 705 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 706 cap->cc_alg[alg] != 0) { 707 cap->cc_alg[alg] = 0; 708 cap->cc_max_op_len[alg] = 0; 709 710 /* Was this the last algorithm ? */ 711 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 712 if (cap->cc_alg[i] != 0) 713 break; 714 715 if (i == CRYPTO_ALGORITHM_MAX + 1) 716 driver_finis(cap); 717 err = 0; 718 } else 719 err = EINVAL; 720 CRYPTO_DRIVER_UNLOCK(); 721 722 return err; 723 } 724 725 /* 726 * Unregister all algorithms associated with a crypto driver. 727 * If there are pending sessions using it, leave enough information 728 * around so that subsequent calls using those sessions will 729 * correctly detect the driver has been unregistered and reroute 730 * requests. 731 */ 732 int 733 crypto_unregister_all(u_int32_t driverid) 734 { 735 struct cryptocap *cap; 736 int err; 737 738 CRYPTO_DRIVER_LOCK(); 739 cap = crypto_checkdriver(driverid); 740 if (cap != NULL) { 741 driver_finis(cap); 742 err = 0; 743 } else 744 err = EINVAL; 745 CRYPTO_DRIVER_UNLOCK(); 746 747 return err; 748 } 749 750 /* 751 * Clear blockage on a driver. The what parameter indicates whether 752 * the driver is now ready for cryptop's and/or cryptokop's. 753 */ 754 int 755 crypto_unblock(u_int32_t driverid, int what) 756 { 757 struct cryptocap *cap; 758 int err; 759 760 CRYPTO_Q_LOCK(); 761 cap = crypto_checkdriver(driverid); 762 if (cap != NULL) { 763 if (what & CRYPTO_SYMQ) 764 cap->cc_qblocked = 0; 765 if (what & CRYPTO_ASYMQ) 766 cap->cc_kqblocked = 0; 767 if (crp_sleep) 768 wakeup_one(&crp_q); 769 err = 0; 770 } else 771 err = EINVAL; 772 CRYPTO_Q_UNLOCK(); 773 774 return err; 775 } 776 777 /* 778 * Add a crypto request to a queue, to be processed by the kernel thread. 779 */ 780 int 781 crypto_dispatch(struct cryptop *crp) 782 { 783 struct cryptocap *cap; 784 u_int32_t hid; 785 int result; 786 787 cryptostats.cs_ops++; 788 789 #ifdef CRYPTO_TIMING 790 if (crypto_timing) 791 binuptime(&crp->crp_tstamp); 792 #endif 793 794 hid = CRYPTO_SESID2HID(crp->crp_sid); 795 796 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 797 /* 798 * Caller marked the request to be processed 799 * immediately; dispatch it directly to the 800 * driver unless the driver is currently blocked. 801 */ 802 cap = crypto_checkdriver(hid); 803 /* Driver cannot disappeared when there is an active session. */ 804 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 805 if (!cap->cc_qblocked) { 806 result = crypto_invoke(cap, crp, 0); 807 if (result != ERESTART) 808 return (result); 809 /* 810 * The driver ran out of resources, put the request on 811 * the queue. 812 */ 813 } 814 } 815 CRYPTO_Q_LOCK(); 816 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 817 if (crp_sleep) 818 wakeup_one(&crp_q); 819 CRYPTO_Q_UNLOCK(); 820 return 0; 821 } 822 823 /* 824 * Add an asymetric crypto request to a queue, 825 * to be processed by the kernel thread. 826 */ 827 int 828 crypto_kdispatch(struct cryptkop *krp) 829 { 830 int error; 831 832 cryptostats.cs_kops++; 833 834 error = crypto_kinvoke(krp, krp->krp_crid); 835 if (error == ERESTART) { 836 CRYPTO_Q_LOCK(); 837 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 838 if (crp_sleep) 839 wakeup_one(&crp_q); 840 CRYPTO_Q_UNLOCK(); 841 error = 0; 842 } 843 return error; 844 } 845 846 /* 847 * Verify a driver is suitable for the specified operation. 848 */ 849 static __inline int 850 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp) 851 { 852 return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0; 853 } 854 855 /* 856 * Select a driver for an asym operation. The driver must 857 * support the necessary algorithm. The caller can constrain 858 * which device is selected with the flags parameter. The 859 * algorithm we use here is pretty stupid; just use the first 860 * driver that supports the algorithms we need. If there are 861 * multiple suitable drivers we choose the driver with the 862 * fewest active operations. We prefer hardware-backed 863 * drivers to software ones when either may be used. 864 */ 865 static struct cryptocap * 866 crypto_select_kdriver(const struct cryptkop *krp, int flags) 867 { 868 struct cryptocap *cap, *best, *blocked; 869 int match, hid; 870 871 CRYPTO_DRIVER_ASSERT(); 872 873 /* 874 * Look first for hardware crypto devices if permitted. 875 */ 876 if (flags & CRYPTOCAP_F_HARDWARE) 877 match = CRYPTOCAP_F_HARDWARE; 878 else 879 match = CRYPTOCAP_F_SOFTWARE; 880 best = NULL; 881 blocked = NULL; 882 again: 883 for (hid = 0; hid < crypto_drivers_num; hid++) { 884 cap = &crypto_drivers[hid]; 885 /* 886 * If it's not initialized, is in the process of 887 * going away, or is not appropriate (hardware 888 * or software based on match), then skip. 889 */ 890 if (cap->cc_dev == NULL || 891 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 892 (cap->cc_flags & match) == 0) 893 continue; 894 895 /* verify all the algorithms are supported. */ 896 if (kdriver_suitable(cap, krp)) { 897 if (best == NULL || 898 cap->cc_koperations < best->cc_koperations) 899 best = cap; 900 } 901 } 902 if (best != NULL) 903 return best; 904 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 905 /* sort of an Algol 68-style for loop */ 906 match = CRYPTOCAP_F_SOFTWARE; 907 goto again; 908 } 909 return best; 910 } 911 912 /* 913 * Dispatch an assymetric crypto request. 914 */ 915 static int 916 crypto_kinvoke(struct cryptkop *krp, int crid) 917 { 918 struct cryptocap *cap = NULL; 919 int error; 920 921 KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 922 KASSERT(krp->krp_callback != NULL, 923 ("%s: krp->crp_callback == NULL", __func__)); 924 925 CRYPTO_DRIVER_LOCK(); 926 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 927 cap = crypto_checkdriver(crid); 928 if (cap != NULL) { 929 /* 930 * Driver present, it must support the necessary 931 * algorithm and, if s/w drivers are excluded, 932 * it must be registered as hardware-backed. 933 */ 934 if (!kdriver_suitable(cap, krp) || 935 (!crypto_devallowsoft && 936 (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) 937 cap = NULL; 938 } 939 } else { 940 /* 941 * No requested driver; select based on crid flags. 942 */ 943 if (!crypto_devallowsoft) /* NB: disallow s/w drivers */ 944 crid &= ~CRYPTOCAP_F_SOFTWARE; 945 cap = crypto_select_kdriver(krp, crid); 946 } 947 if (cap != NULL && !cap->cc_kqblocked) { 948 krp->krp_hid = cap - crypto_drivers; 949 cap->cc_koperations++; 950 CRYPTO_DRIVER_UNLOCK(); 951 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); 952 CRYPTO_DRIVER_LOCK(); 953 if (error == ERESTART) { 954 cap->cc_koperations--; 955 CRYPTO_DRIVER_UNLOCK(); 956 return (error); 957 } 958 } else { 959 /* 960 * NB: cap is !NULL if device is blocked; in 961 * that case return ERESTART so the operation 962 * is resubmitted if possible. 963 */ 964 error = (cap == NULL) ? ENODEV : ERESTART; 965 } 966 CRYPTO_DRIVER_UNLOCK(); 967 968 if (error) { 969 krp->krp_status = error; 970 crypto_kdone(krp); 971 } 972 return 0; 973 } 974 975 #ifdef CRYPTO_TIMING 976 static void 977 crypto_tstat(struct cryptotstat *ts, struct bintime *bt) 978 { 979 struct bintime now, delta; 980 struct timespec t; 981 uint64_t u; 982 983 binuptime(&now); 984 u = now.frac; 985 delta.frac = now.frac - bt->frac; 986 delta.sec = now.sec - bt->sec; 987 if (u < delta.frac) 988 delta.sec--; 989 bintime2timespec(&delta, &t); 990 timespecadd(&ts->acc, &t); 991 if (timespeccmp(&t, &ts->min, <)) 992 ts->min = t; 993 if (timespeccmp(&t, &ts->max, >)) 994 ts->max = t; 995 ts->count++; 996 997 *bt = now; 998 } 999 #endif 1000 1001 /* 1002 * Dispatch a crypto request to the appropriate crypto devices. 1003 */ 1004 static int 1005 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 1006 { 1007 1008 KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 1009 KASSERT(crp->crp_callback != NULL, 1010 ("%s: crp->crp_callback == NULL", __func__)); 1011 KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 1012 1013 #ifdef CRYPTO_TIMING 1014 if (crypto_timing) 1015 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 1016 #endif 1017 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 1018 struct cryptodesc *crd; 1019 u_int64_t nid; 1020 1021 /* 1022 * Driver has unregistered; migrate the session and return 1023 * an error to the caller so they'll resubmit the op. 1024 * 1025 * XXX: What if there are more already queued requests for this 1026 * session? 1027 */ 1028 crypto_freesession(crp->crp_sid); 1029 1030 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 1031 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 1032 1033 /* XXX propagate flags from initial session? */ 1034 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 1035 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) 1036 crp->crp_sid = nid; 1037 1038 crp->crp_etype = EAGAIN; 1039 crypto_done(crp); 1040 return 0; 1041 } else { 1042 /* 1043 * Invoke the driver to process the request. 1044 */ 1045 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint); 1046 } 1047 } 1048 1049 /* 1050 * Release a set of crypto descriptors. 1051 */ 1052 void 1053 crypto_freereq(struct cryptop *crp) 1054 { 1055 struct cryptodesc *crd; 1056 1057 if (crp == NULL) 1058 return; 1059 1060 #ifdef DIAGNOSTIC 1061 { 1062 struct cryptop *crp2; 1063 1064 CRYPTO_Q_LOCK(); 1065 TAILQ_FOREACH(crp2, &crp_q, crp_next) { 1066 KASSERT(crp2 != crp, 1067 ("Freeing cryptop from the crypto queue (%p).", 1068 crp)); 1069 } 1070 CRYPTO_Q_UNLOCK(); 1071 CRYPTO_RETQ_LOCK(); 1072 TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) { 1073 KASSERT(crp2 != crp, 1074 ("Freeing cryptop from the return queue (%p).", 1075 crp)); 1076 } 1077 CRYPTO_RETQ_UNLOCK(); 1078 } 1079 #endif 1080 1081 while ((crd = crp->crp_desc) != NULL) { 1082 crp->crp_desc = crd->crd_next; 1083 uma_zfree(cryptodesc_zone, crd); 1084 } 1085 uma_zfree(cryptop_zone, crp); 1086 } 1087 1088 /* 1089 * Acquire a set of crypto descriptors. 1090 */ 1091 struct cryptop * 1092 crypto_getreq(int num) 1093 { 1094 struct cryptodesc *crd; 1095 struct cryptop *crp; 1096 1097 crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 1098 if (crp != NULL) { 1099 while (num--) { 1100 crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 1101 if (crd == NULL) { 1102 crypto_freereq(crp); 1103 return NULL; 1104 } 1105 1106 crd->crd_next = crp->crp_desc; 1107 crp->crp_desc = crd; 1108 } 1109 } 1110 return crp; 1111 } 1112 1113 /* 1114 * Invoke the callback on behalf of the driver. 1115 */ 1116 void 1117 crypto_done(struct cryptop *crp) 1118 { 1119 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 1120 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 1121 crp->crp_flags |= CRYPTO_F_DONE; 1122 if (crp->crp_etype != 0) 1123 cryptostats.cs_errs++; 1124 #ifdef CRYPTO_TIMING 1125 if (crypto_timing) 1126 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 1127 #endif 1128 /* 1129 * CBIMM means unconditionally do the callback immediately; 1130 * CBIFSYNC means do the callback immediately only if the 1131 * operation was done synchronously. Both are used to avoid 1132 * doing extraneous context switches; the latter is mostly 1133 * used with the software crypto driver. 1134 */ 1135 if ((crp->crp_flags & CRYPTO_F_CBIMM) || 1136 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 1137 (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 1138 /* 1139 * Do the callback directly. This is ok when the 1140 * callback routine does very little (e.g. the 1141 * /dev/crypto callback method just does a wakeup). 1142 */ 1143 #ifdef CRYPTO_TIMING 1144 if (crypto_timing) { 1145 /* 1146 * NB: We must copy the timestamp before 1147 * doing the callback as the cryptop is 1148 * likely to be reclaimed. 1149 */ 1150 struct bintime t = crp->crp_tstamp; 1151 crypto_tstat(&cryptostats.cs_cb, &t); 1152 crp->crp_callback(crp); 1153 crypto_tstat(&cryptostats.cs_finis, &t); 1154 } else 1155 #endif 1156 crp->crp_callback(crp); 1157 } else { 1158 /* 1159 * Normal case; queue the callback for the thread. 1160 */ 1161 CRYPTO_RETQ_LOCK(); 1162 if (CRYPTO_RETQ_EMPTY()) 1163 wakeup_one(&crp_ret_q); /* shared wait channel */ 1164 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 1165 CRYPTO_RETQ_UNLOCK(); 1166 } 1167 } 1168 1169 /* 1170 * Invoke the callback on behalf of the driver. 1171 */ 1172 void 1173 crypto_kdone(struct cryptkop *krp) 1174 { 1175 struct cryptocap *cap; 1176 1177 if (krp->krp_status != 0) 1178 cryptostats.cs_kerrs++; 1179 CRYPTO_DRIVER_LOCK(); 1180 /* XXX: What if driver is loaded in the meantime? */ 1181 if (krp->krp_hid < crypto_drivers_num) { 1182 cap = &crypto_drivers[krp->krp_hid]; 1183 cap->cc_koperations--; 1184 KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); 1185 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 1186 crypto_remove(cap); 1187 } 1188 CRYPTO_DRIVER_UNLOCK(); 1189 CRYPTO_RETQ_LOCK(); 1190 if (CRYPTO_RETQ_EMPTY()) 1191 wakeup_one(&crp_ret_q); /* shared wait channel */ 1192 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 1193 CRYPTO_RETQ_UNLOCK(); 1194 } 1195 1196 int 1197 crypto_getfeat(int *featp) 1198 { 1199 int hid, kalg, feat = 0; 1200 1201 CRYPTO_DRIVER_LOCK(); 1202 for (hid = 0; hid < crypto_drivers_num; hid++) { 1203 const struct cryptocap *cap = &crypto_drivers[hid]; 1204 1205 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1206 !crypto_devallowsoft) { 1207 continue; 1208 } 1209 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1210 if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED) 1211 feat |= 1 << kalg; 1212 } 1213 CRYPTO_DRIVER_UNLOCK(); 1214 *featp = feat; 1215 return (0); 1216 } 1217 1218 /* 1219 * Terminate a thread at module unload. The process that 1220 * initiated this is waiting for us to signal that we're gone; 1221 * wake it up and exit. We use the driver table lock to insure 1222 * we don't do the wakeup before they're waiting. There is no 1223 * race here because the waiter sleeps on the proc lock for the 1224 * thread so it gets notified at the right time because of an 1225 * extra wakeup that's done in exit1(). 1226 */ 1227 static void 1228 crypto_finis(void *chan) 1229 { 1230 CRYPTO_DRIVER_LOCK(); 1231 wakeup_one(chan); 1232 CRYPTO_DRIVER_UNLOCK(); 1233 kproc_exit(0); 1234 } 1235 1236 /* 1237 * Crypto thread, dispatches crypto requests. 1238 */ 1239 static void 1240 crypto_proc(void) 1241 { 1242 struct cryptop *crp, *submit; 1243 struct cryptkop *krp; 1244 struct cryptocap *cap; 1245 u_int32_t hid; 1246 int result, hint; 1247 1248 #if defined(__i386__) || defined(__amd64__) 1249 fpu_kern_thread(FPU_KERN_NORMAL); 1250 #endif 1251 1252 CRYPTO_Q_LOCK(); 1253 for (;;) { 1254 /* 1255 * Find the first element in the queue that can be 1256 * processed and look-ahead to see if multiple ops 1257 * are ready for the same driver. 1258 */ 1259 submit = NULL; 1260 hint = 0; 1261 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1262 hid = CRYPTO_SESID2HID(crp->crp_sid); 1263 cap = crypto_checkdriver(hid); 1264 /* 1265 * Driver cannot disappeared when there is an active 1266 * session. 1267 */ 1268 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1269 __func__, __LINE__)); 1270 if (cap == NULL || cap->cc_dev == NULL) { 1271 /* Op needs to be migrated, process it. */ 1272 if (submit == NULL) 1273 submit = crp; 1274 break; 1275 } 1276 if (!cap->cc_qblocked) { 1277 if (submit != NULL) { 1278 /* 1279 * We stop on finding another op, 1280 * regardless whether its for the same 1281 * driver or not. We could keep 1282 * searching the queue but it might be 1283 * better to just use a per-driver 1284 * queue instead. 1285 */ 1286 if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1287 hint = CRYPTO_HINT_MORE; 1288 break; 1289 } else { 1290 submit = crp; 1291 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1292 break; 1293 /* keep scanning for more are q'd */ 1294 } 1295 } 1296 } 1297 if (submit != NULL) { 1298 TAILQ_REMOVE(&crp_q, submit, crp_next); 1299 hid = CRYPTO_SESID2HID(submit->crp_sid); 1300 cap = crypto_checkdriver(hid); 1301 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1302 __func__, __LINE__)); 1303 result = crypto_invoke(cap, submit, hint); 1304 if (result == ERESTART) { 1305 /* 1306 * The driver ran out of resources, mark the 1307 * driver ``blocked'' for cryptop's and put 1308 * the request back in the queue. It would 1309 * best to put the request back where we got 1310 * it but that's hard so for now we put it 1311 * at the front. This should be ok; putting 1312 * it at the end does not work. 1313 */ 1314 /* XXX validate sid again? */ 1315 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1316 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1317 cryptostats.cs_blocks++; 1318 } 1319 } 1320 1321 /* As above, but for key ops */ 1322 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1323 cap = crypto_checkdriver(krp->krp_hid); 1324 if (cap == NULL || cap->cc_dev == NULL) { 1325 /* 1326 * Operation needs to be migrated, invalidate 1327 * the assigned device so it will reselect a 1328 * new one below. Propagate the original 1329 * crid selection flags if supplied. 1330 */ 1331 krp->krp_hid = krp->krp_crid & 1332 (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); 1333 if (krp->krp_hid == 0) 1334 krp->krp_hid = 1335 CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; 1336 break; 1337 } 1338 if (!cap->cc_kqblocked) 1339 break; 1340 } 1341 if (krp != NULL) { 1342 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1343 result = crypto_kinvoke(krp, krp->krp_hid); 1344 if (result == ERESTART) { 1345 /* 1346 * The driver ran out of resources, mark the 1347 * driver ``blocked'' for cryptkop's and put 1348 * the request back in the queue. It would 1349 * best to put the request back where we got 1350 * it but that's hard so for now we put it 1351 * at the front. This should be ok; putting 1352 * it at the end does not work. 1353 */ 1354 /* XXX validate sid again? */ 1355 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1356 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1357 cryptostats.cs_kblocks++; 1358 } 1359 } 1360 1361 if (submit == NULL && krp == NULL) { 1362 /* 1363 * Nothing more to be processed. Sleep until we're 1364 * woken because there are more ops to process. 1365 * This happens either by submission or by a driver 1366 * becoming unblocked and notifying us through 1367 * crypto_unblock. Note that when we wakeup we 1368 * start processing each queue again from the 1369 * front. It's not clear that it's important to 1370 * preserve this ordering since ops may finish 1371 * out of order if dispatched to different devices 1372 * and some become blocked while others do not. 1373 */ 1374 crp_sleep = 1; 1375 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1376 crp_sleep = 0; 1377 if (cryptoproc == NULL) 1378 break; 1379 cryptostats.cs_intrs++; 1380 } 1381 } 1382 CRYPTO_Q_UNLOCK(); 1383 1384 crypto_finis(&crp_q); 1385 } 1386 1387 /* 1388 * Crypto returns thread, does callbacks for processed crypto requests. 1389 * Callbacks are done here, rather than in the crypto drivers, because 1390 * callbacks typically are expensive and would slow interrupt handling. 1391 */ 1392 static void 1393 crypto_ret_proc(void) 1394 { 1395 struct cryptop *crpt; 1396 struct cryptkop *krpt; 1397 1398 CRYPTO_RETQ_LOCK(); 1399 for (;;) { 1400 /* Harvest return q's for completed ops */ 1401 crpt = TAILQ_FIRST(&crp_ret_q); 1402 if (crpt != NULL) 1403 TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1404 1405 krpt = TAILQ_FIRST(&crp_ret_kq); 1406 if (krpt != NULL) 1407 TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1408 1409 if (crpt != NULL || krpt != NULL) { 1410 CRYPTO_RETQ_UNLOCK(); 1411 /* 1412 * Run callbacks unlocked. 1413 */ 1414 if (crpt != NULL) { 1415 #ifdef CRYPTO_TIMING 1416 if (crypto_timing) { 1417 /* 1418 * NB: We must copy the timestamp before 1419 * doing the callback as the cryptop is 1420 * likely to be reclaimed. 1421 */ 1422 struct bintime t = crpt->crp_tstamp; 1423 crypto_tstat(&cryptostats.cs_cb, &t); 1424 crpt->crp_callback(crpt); 1425 crypto_tstat(&cryptostats.cs_finis, &t); 1426 } else 1427 #endif 1428 crpt->crp_callback(crpt); 1429 } 1430 if (krpt != NULL) 1431 krpt->krp_callback(krpt); 1432 CRYPTO_RETQ_LOCK(); 1433 } else { 1434 /* 1435 * Nothing more to be processed. Sleep until we're 1436 * woken because there are more returns to process. 1437 */ 1438 msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1439 "crypto_ret_wait", 0); 1440 if (cryptoretproc == NULL) 1441 break; 1442 cryptostats.cs_rets++; 1443 } 1444 } 1445 CRYPTO_RETQ_UNLOCK(); 1446 1447 crypto_finis(&crp_ret_q); 1448 } 1449 1450 #ifdef DDB 1451 static void 1452 db_show_drivers(void) 1453 { 1454 int hid; 1455 1456 db_printf("%12s %4s %4s %8s %2s %2s\n" 1457 , "Device" 1458 , "Ses" 1459 , "Kops" 1460 , "Flags" 1461 , "QB" 1462 , "KB" 1463 ); 1464 for (hid = 0; hid < crypto_drivers_num; hid++) { 1465 const struct cryptocap *cap = &crypto_drivers[hid]; 1466 if (cap->cc_dev == NULL) 1467 continue; 1468 db_printf("%-12s %4u %4u %08x %2u %2u\n" 1469 , device_get_nameunit(cap->cc_dev) 1470 , cap->cc_sessions 1471 , cap->cc_koperations 1472 , cap->cc_flags 1473 , cap->cc_qblocked 1474 , cap->cc_kqblocked 1475 ); 1476 } 1477 } 1478 1479 DB_SHOW_COMMAND(crypto, db_show_crypto) 1480 { 1481 struct cryptop *crp; 1482 1483 db_show_drivers(); 1484 db_printf("\n"); 1485 1486 db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", 1487 "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", 1488 "Desc", "Callback"); 1489 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1490 db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n" 1491 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1492 , (int) CRYPTO_SESID2CAPS(crp->crp_sid) 1493 , crp->crp_ilen, crp->crp_olen 1494 , crp->crp_etype 1495 , crp->crp_flags 1496 , crp->crp_desc 1497 , crp->crp_callback 1498 ); 1499 } 1500 if (!TAILQ_EMPTY(&crp_ret_q)) { 1501 db_printf("\n%4s %4s %4s %8s\n", 1502 "HID", "Etype", "Flags", "Callback"); 1503 TAILQ_FOREACH(crp, &crp_ret_q, crp_next) { 1504 db_printf("%4u %4u %04x %8p\n" 1505 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1506 , crp->crp_etype 1507 , crp->crp_flags 1508 , crp->crp_callback 1509 ); 1510 } 1511 } 1512 } 1513 1514 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) 1515 { 1516 struct cryptkop *krp; 1517 1518 db_show_drivers(); 1519 db_printf("\n"); 1520 1521 db_printf("%4s %5s %4s %4s %8s %4s %8s\n", 1522 "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); 1523 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1524 db_printf("%4u %5u %4u %4u %08x %4u %8p\n" 1525 , krp->krp_op 1526 , krp->krp_status 1527 , krp->krp_iparams, krp->krp_oparams 1528 , krp->krp_crid, krp->krp_hid 1529 , krp->krp_callback 1530 ); 1531 } 1532 if (!TAILQ_EMPTY(&crp_ret_q)) { 1533 db_printf("%4s %5s %8s %4s %8s\n", 1534 "Op", "Status", "CRID", "HID", "Callback"); 1535 TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) { 1536 db_printf("%4u %5u %08x %4u %8p\n" 1537 , krp->krp_op 1538 , krp->krp_status 1539 , krp->krp_crid, krp->krp_hid 1540 , krp->krp_callback 1541 ); 1542 } 1543 } 1544 } 1545 #endif 1546 1547 int crypto_modevent(module_t mod, int type, void *unused); 1548 1549 /* 1550 * Initialization code, both for static and dynamic loading. 1551 * Note this is not invoked with the usual MODULE_DECLARE 1552 * mechanism but instead is listed as a dependency by the 1553 * cryptosoft driver. This guarantees proper ordering of 1554 * calls on module load/unload. 1555 */ 1556 int 1557 crypto_modevent(module_t mod, int type, void *unused) 1558 { 1559 int error = EINVAL; 1560 1561 switch (type) { 1562 case MOD_LOAD: 1563 error = crypto_init(); 1564 if (error == 0 && bootverbose) 1565 printf("crypto: <crypto core>\n"); 1566 break; 1567 case MOD_UNLOAD: 1568 /*XXX disallow if active sessions */ 1569 error = 0; 1570 crypto_destroy(); 1571 return 0; 1572 } 1573 return error; 1574 } 1575 MODULE_VERSION(crypto, 1); 1576 MODULE_DEPEND(crypto, zlib, 1, 1, 1); 1577