1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 30 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 34 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Effort sponsored in part by the Defense Advanced Research Projects 37 * Agency (DARPA) and Air Force Research Laboratory, Air Force 38 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/proc.h> 51 #include <sys/sysctl.h> 52 #include <sys/errno.h> 53 #include <sys/random.h> 54 #include <sys/conf.h> 55 #include <sys/kernel.h> 56 #include <sys/module.h> 57 #include <sys/fcntl.h> 58 #include <sys/bus.h> 59 #include <sys/sdt.h> 60 #include <sys/syscallsubr.h> 61 62 #include <opencrypto/cryptodev.h> 63 #include <opencrypto/xform.h> 64 65 SDT_PROVIDER_DECLARE(opencrypto); 66 67 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 68 69 #ifdef COMPAT_FREEBSD12 70 /* 71 * Previously, most ioctls were performed against a cloned descriptor 72 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed 73 * against /dev/crypto directly. 74 */ 75 #define CRIOGET _IOWR('c', 100, uint32_t) 76 #endif 77 78 /* the following are done against the cloned descriptor */ 79 80 #ifdef COMPAT_FREEBSD32 81 #include <sys/mount.h> 82 #include <compat/freebsd32/freebsd32.h> 83 84 struct session_op32 { 85 uint32_t cipher; 86 uint32_t mac; 87 uint32_t keylen; 88 uint32_t key; 89 int mackeylen; 90 uint32_t mackey; 91 uint32_t ses; 92 }; 93 94 struct session2_op32 { 95 uint32_t cipher; 96 uint32_t mac; 97 uint32_t keylen; 98 uint32_t key; 99 int mackeylen; 100 uint32_t mackey; 101 uint32_t ses; 102 int crid; 103 int pad[4]; 104 }; 105 106 struct crypt_op32 { 107 uint32_t ses; 108 uint16_t op; 109 uint16_t flags; 110 u_int len; 111 uint32_t src, dst; 112 uint32_t mac; 113 uint32_t iv; 114 }; 115 116 struct crypt_aead32 { 117 uint32_t ses; 118 uint16_t op; 119 uint16_t flags; 120 u_int len; 121 u_int aadlen; 122 u_int ivlen; 123 uint32_t src; 124 uint32_t dst; 125 uint32_t aad; 126 uint32_t tag; 127 uint32_t iv; 128 }; 129 130 struct crparam32 { 131 uint32_t crp_p; 132 u_int crp_nbits; 133 }; 134 135 struct crypt_kop32 { 136 u_int crk_op; 137 u_int crk_status; 138 u_short crk_iparams; 139 u_short crk_oparams; 140 u_int crk_crid; 141 struct crparam32 crk_param[CRK_MAXPARAM]; 142 }; 143 144 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 145 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 146 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32) 147 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 148 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32) 149 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 150 151 static void 152 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 153 { 154 155 memset(to, 0, sizeof(*to)); 156 CP(*from, *to, cipher); 157 CP(*from, *to, mac); 158 CP(*from, *to, keylen); 159 PTRIN_CP(*from, *to, key); 160 CP(*from, *to, mackeylen); 161 PTRIN_CP(*from, *to, mackey); 162 CP(*from, *to, ses); 163 to->crid = CRYPTOCAP_F_HARDWARE; 164 } 165 166 static void 167 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 168 { 169 170 session_op_from_32((const struct session_op32 *)from, to); 171 CP(*from, *to, crid); 172 } 173 174 static void 175 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 176 { 177 178 CP(*from, *to, cipher); 179 CP(*from, *to, mac); 180 CP(*from, *to, keylen); 181 PTROUT_CP(*from, *to, key); 182 CP(*from, *to, mackeylen); 183 PTROUT_CP(*from, *to, mackey); 184 CP(*from, *to, ses); 185 } 186 187 static void 188 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 189 { 190 191 session_op_to_32(from, (struct session_op32 *)to); 192 CP(*from, *to, crid); 193 } 194 195 static void 196 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 197 { 198 199 CP(*from, *to, ses); 200 CP(*from, *to, op); 201 CP(*from, *to, flags); 202 CP(*from, *to, len); 203 PTRIN_CP(*from, *to, src); 204 PTRIN_CP(*from, *to, dst); 205 PTRIN_CP(*from, *to, mac); 206 PTRIN_CP(*from, *to, iv); 207 } 208 209 static void 210 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 211 { 212 213 CP(*from, *to, ses); 214 CP(*from, *to, op); 215 CP(*from, *to, flags); 216 CP(*from, *to, len); 217 PTROUT_CP(*from, *to, src); 218 PTROUT_CP(*from, *to, dst); 219 PTROUT_CP(*from, *to, mac); 220 PTROUT_CP(*from, *to, iv); 221 } 222 223 static void 224 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 225 { 226 227 CP(*from, *to, ses); 228 CP(*from, *to, op); 229 CP(*from, *to, flags); 230 CP(*from, *to, len); 231 CP(*from, *to, aadlen); 232 CP(*from, *to, ivlen); 233 PTRIN_CP(*from, *to, src); 234 PTRIN_CP(*from, *to, dst); 235 PTRIN_CP(*from, *to, aad); 236 PTRIN_CP(*from, *to, tag); 237 PTRIN_CP(*from, *to, iv); 238 } 239 240 static void 241 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 242 { 243 244 CP(*from, *to, ses); 245 CP(*from, *to, op); 246 CP(*from, *to, flags); 247 CP(*from, *to, len); 248 CP(*from, *to, aadlen); 249 CP(*from, *to, ivlen); 250 PTROUT_CP(*from, *to, src); 251 PTROUT_CP(*from, *to, dst); 252 PTROUT_CP(*from, *to, aad); 253 PTROUT_CP(*from, *to, tag); 254 PTROUT_CP(*from, *to, iv); 255 } 256 257 static void 258 crparam_from_32(const struct crparam32 *from, struct crparam *to) 259 { 260 261 PTRIN_CP(*from, *to, crp_p); 262 CP(*from, *to, crp_nbits); 263 } 264 265 static void 266 crparam_to_32(const struct crparam *from, struct crparam32 *to) 267 { 268 269 PTROUT_CP(*from, *to, crp_p); 270 CP(*from, *to, crp_nbits); 271 } 272 273 static void 274 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to) 275 { 276 int i; 277 278 CP(*from, *to, crk_op); 279 CP(*from, *to, crk_status); 280 CP(*from, *to, crk_iparams); 281 CP(*from, *to, crk_oparams); 282 CP(*from, *to, crk_crid); 283 for (i = 0; i < CRK_MAXPARAM; i++) 284 crparam_from_32(&from->crk_param[i], &to->crk_param[i]); 285 } 286 287 static void 288 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to) 289 { 290 int i; 291 292 CP(*from, *to, crk_op); 293 CP(*from, *to, crk_status); 294 CP(*from, *to, crk_iparams); 295 CP(*from, *to, crk_oparams); 296 CP(*from, *to, crk_crid); 297 for (i = 0; i < CRK_MAXPARAM; i++) 298 crparam_to_32(&from->crk_param[i], &to->crk_param[i]); 299 } 300 #endif 301 302 static void 303 session2_op_from_op(const struct session_op *from, struct session2_op *to) 304 { 305 306 memset(to, 0, sizeof(*to)); 307 memcpy(to, from, sizeof(*from)); 308 to->crid = CRYPTOCAP_F_HARDWARE; 309 } 310 311 static void 312 session2_op_to_op(const struct session2_op *from, struct session_op *to) 313 { 314 315 memcpy(to, from, sizeof(*to)); 316 } 317 318 struct csession { 319 TAILQ_ENTRY(csession) next; 320 crypto_session_t cses; 321 volatile u_int refs; 322 uint32_t ses; 323 struct mtx lock; /* for op submission */ 324 325 struct enc_xform *txform; 326 int hashsize; 327 int ivsize; 328 int mode; 329 330 void *key; 331 void *mackey; 332 }; 333 334 struct cryptop_data { 335 struct csession *cse; 336 337 char *buf; 338 char *obuf; 339 char *aad; 340 bool done; 341 }; 342 343 struct fcrypt { 344 TAILQ_HEAD(csessionlist, csession) csessions; 345 int sesn; 346 struct mtx lock; 347 }; 348 349 static bool use_outputbuffers; 350 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 351 &use_outputbuffers, 0, 352 "Use separate output buffers for /dev/crypto requests."); 353 354 static bool use_separate_aad; 355 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 356 &use_separate_aad, 0, 357 "Use separate AAD buffer for /dev/crypto requests."); 358 359 static struct timeval warninterval = { .tv_sec = 60, .tv_usec = 0 }; 360 SYSCTL_TIMEVAL_SEC(_kern, OID_AUTO, cryptodev_warn_interval, CTLFLAG_RW, 361 &warninterval, 362 "Delay in seconds between warnings of deprecated /dev/crypto algorithms"); 363 364 /* 365 * Check a crypto identifier to see if it requested 366 * a software device/driver. This can be done either 367 * by device name/class or through search constraints. 368 */ 369 static int 370 checkforsoftware(int *cridp) 371 { 372 int crid; 373 374 crid = *cridp; 375 376 if (!crypto_devallowsoft) { 377 if (crid & CRYPTOCAP_F_SOFTWARE) { 378 if (crid & CRYPTOCAP_F_HARDWARE) { 379 *cridp = CRYPTOCAP_F_HARDWARE; 380 return 0; 381 } 382 return EINVAL; 383 } 384 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 385 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 386 return EINVAL; 387 } 388 return 0; 389 } 390 391 static int 392 cse_create(struct fcrypt *fcr, struct session2_op *sop) 393 { 394 struct crypto_session_params csp; 395 struct csession *cse; 396 struct enc_xform *txform; 397 struct auth_hash *thash; 398 void *key = NULL; 399 void *mackey = NULL; 400 crypto_session_t cses; 401 int crid, error; 402 403 switch (sop->cipher) { 404 case 0: 405 txform = NULL; 406 break; 407 case CRYPTO_AES_CBC: 408 txform = &enc_xform_rijndael128; 409 break; 410 case CRYPTO_AES_XTS: 411 txform = &enc_xform_aes_xts; 412 break; 413 case CRYPTO_NULL_CBC: 414 txform = &enc_xform_null; 415 break; 416 case CRYPTO_CAMELLIA_CBC: 417 txform = &enc_xform_camellia; 418 break; 419 case CRYPTO_AES_ICM: 420 txform = &enc_xform_aes_icm; 421 break; 422 case CRYPTO_AES_NIST_GCM_16: 423 txform = &enc_xform_aes_nist_gcm; 424 break; 425 case CRYPTO_CHACHA20: 426 txform = &enc_xform_chacha20; 427 break; 428 case CRYPTO_AES_CCM_16: 429 txform = &enc_xform_ccm; 430 break; 431 case CRYPTO_CHACHA20_POLY1305: 432 txform = &enc_xform_chacha20_poly1305; 433 break; 434 default: 435 CRYPTDEB("invalid cipher"); 436 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 437 return (EINVAL); 438 } 439 440 switch (sop->mac) { 441 case 0: 442 thash = NULL; 443 break; 444 case CRYPTO_POLY1305: 445 thash = &auth_hash_poly1305; 446 break; 447 case CRYPTO_SHA1_HMAC: 448 thash = &auth_hash_hmac_sha1; 449 break; 450 case CRYPTO_SHA2_224_HMAC: 451 thash = &auth_hash_hmac_sha2_224; 452 break; 453 case CRYPTO_SHA2_256_HMAC: 454 thash = &auth_hash_hmac_sha2_256; 455 break; 456 case CRYPTO_SHA2_384_HMAC: 457 thash = &auth_hash_hmac_sha2_384; 458 break; 459 case CRYPTO_SHA2_512_HMAC: 460 thash = &auth_hash_hmac_sha2_512; 461 break; 462 case CRYPTO_RIPEMD160_HMAC: 463 thash = &auth_hash_hmac_ripemd_160; 464 break; 465 #ifdef COMPAT_FREEBSD12 466 case CRYPTO_AES_128_NIST_GMAC: 467 case CRYPTO_AES_192_NIST_GMAC: 468 case CRYPTO_AES_256_NIST_GMAC: 469 /* Should always be paired with GCM. */ 470 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 471 CRYPTDEB("GMAC without GCM"); 472 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 473 return (EINVAL); 474 } 475 break; 476 #endif 477 case CRYPTO_AES_NIST_GMAC: 478 switch (sop->mackeylen * 8) { 479 case 128: 480 thash = &auth_hash_nist_gmac_aes_128; 481 break; 482 case 192: 483 thash = &auth_hash_nist_gmac_aes_192; 484 break; 485 case 256: 486 thash = &auth_hash_nist_gmac_aes_256; 487 break; 488 default: 489 CRYPTDEB("invalid GMAC key length"); 490 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 491 return (EINVAL); 492 } 493 break; 494 case CRYPTO_AES_CCM_CBC_MAC: 495 switch (sop->mackeylen) { 496 case 16: 497 thash = &auth_hash_ccm_cbc_mac_128; 498 break; 499 case 24: 500 thash = &auth_hash_ccm_cbc_mac_192; 501 break; 502 case 32: 503 thash = &auth_hash_ccm_cbc_mac_256; 504 break; 505 default: 506 CRYPTDEB("Invalid CBC MAC key size %d", sop->keylen); 507 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 508 return (EINVAL); 509 } 510 break; 511 case CRYPTO_SHA1: 512 thash = &auth_hash_sha1; 513 break; 514 case CRYPTO_SHA2_224: 515 thash = &auth_hash_sha2_224; 516 break; 517 case CRYPTO_SHA2_256: 518 thash = &auth_hash_sha2_256; 519 break; 520 case CRYPTO_SHA2_384: 521 thash = &auth_hash_sha2_384; 522 break; 523 case CRYPTO_SHA2_512: 524 thash = &auth_hash_sha2_512; 525 break; 526 527 case CRYPTO_NULL_HMAC: 528 thash = &auth_hash_null; 529 break; 530 531 case CRYPTO_BLAKE2B: 532 thash = &auth_hash_blake2b; 533 break; 534 case CRYPTO_BLAKE2S: 535 thash = &auth_hash_blake2s; 536 break; 537 538 default: 539 CRYPTDEB("invalid mac"); 540 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 541 return (EINVAL); 542 } 543 544 if (txform == NULL && thash == NULL) { 545 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 546 return (EINVAL); 547 } 548 549 memset(&csp, 0, sizeof(csp)); 550 if (use_outputbuffers) 551 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 552 553 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { 554 switch (sop->mac) { 555 #ifdef COMPAT_FREEBSD12 556 case CRYPTO_AES_128_NIST_GMAC: 557 case CRYPTO_AES_192_NIST_GMAC: 558 case CRYPTO_AES_256_NIST_GMAC: 559 if (sop->keylen != sop->mackeylen) { 560 SDT_PROBE1(opencrypto, dev, ioctl, error, 561 __LINE__); 562 return (EINVAL); 563 } 564 break; 565 #endif 566 case 0: 567 break; 568 default: 569 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 570 return (EINVAL); 571 } 572 csp.csp_mode = CSP_MODE_AEAD; 573 } else if (sop->cipher == CRYPTO_AES_CCM_16) { 574 switch (sop->mac) { 575 #ifdef COMPAT_FREEBSD12 576 case CRYPTO_AES_CCM_CBC_MAC: 577 if (sop->keylen != sop->mackeylen) { 578 SDT_PROBE1(opencrypto, dev, ioctl, error, 579 __LINE__); 580 return (EINVAL); 581 } 582 thash = NULL; 583 break; 584 #endif 585 case 0: 586 break; 587 default: 588 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 589 return (EINVAL); 590 } 591 csp.csp_mode = CSP_MODE_AEAD; 592 } else if (sop->cipher == CRYPTO_CHACHA20_POLY1305) { 593 if (sop->mac != 0) { 594 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 595 return (EINVAL); 596 } 597 csp.csp_mode = CSP_MODE_AEAD; 598 } else if (txform != NULL && thash != NULL) 599 csp.csp_mode = CSP_MODE_ETA; 600 else if (txform != NULL) 601 csp.csp_mode = CSP_MODE_CIPHER; 602 else 603 csp.csp_mode = CSP_MODE_DIGEST; 604 605 switch (csp.csp_mode) { 606 case CSP_MODE_AEAD: 607 case CSP_MODE_ETA: 608 if (use_separate_aad) 609 csp.csp_flags |= CSP_F_SEPARATE_AAD; 610 break; 611 } 612 613 if (txform != NULL) { 614 csp.csp_cipher_alg = txform->type; 615 csp.csp_cipher_klen = sop->keylen; 616 if (sop->keylen > txform->maxkey || 617 sop->keylen < txform->minkey) { 618 CRYPTDEB("invalid cipher parameters"); 619 error = EINVAL; 620 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 621 goto bail; 622 } 623 624 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 625 error = copyin(sop->key, key, csp.csp_cipher_klen); 626 if (error) { 627 CRYPTDEB("invalid key"); 628 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 629 goto bail; 630 } 631 csp.csp_cipher_key = key; 632 csp.csp_ivlen = txform->ivsize; 633 } 634 635 if (thash != NULL) { 636 csp.csp_auth_alg = thash->type; 637 csp.csp_auth_klen = sop->mackeylen; 638 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { 639 CRYPTDEB("invalid mac key length"); 640 error = EINVAL; 641 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 642 goto bail; 643 } 644 645 if (csp.csp_auth_klen != 0) { 646 mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK); 647 error = copyin(sop->mackey, mackey, csp.csp_auth_klen); 648 if (error) { 649 CRYPTDEB("invalid mac key"); 650 SDT_PROBE1(opencrypto, dev, ioctl, error, 651 __LINE__); 652 goto bail; 653 } 654 csp.csp_auth_key = mackey; 655 } 656 657 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 658 csp.csp_ivlen = AES_GCM_IV_LEN; 659 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 660 csp.csp_ivlen = AES_CCM_IV_LEN; 661 } 662 663 crid = sop->crid; 664 error = checkforsoftware(&crid); 665 if (error) { 666 CRYPTDEB("checkforsoftware"); 667 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 668 goto bail; 669 } 670 error = crypto_newsession(&cses, &csp, crid); 671 if (error) { 672 CRYPTDEB("crypto_newsession"); 673 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 674 goto bail; 675 } 676 677 cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO); 678 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 679 refcount_init(&cse->refs, 1); 680 cse->key = key; 681 cse->mackey = mackey; 682 cse->mode = csp.csp_mode; 683 cse->cses = cses; 684 cse->txform = txform; 685 if (thash != NULL) 686 cse->hashsize = thash->hashsize; 687 else if (csp.csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) 688 cse->hashsize = AES_GMAC_HASH_LEN; 689 else if (csp.csp_cipher_alg == CRYPTO_AES_CCM_16) 690 cse->hashsize = AES_CBC_MAC_HASH_LEN; 691 else if (csp.csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) 692 cse->hashsize = POLY1305_HASH_LEN; 693 cse->ivsize = csp.csp_ivlen; 694 695 mtx_lock(&fcr->lock); 696 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 697 cse->ses = fcr->sesn++; 698 mtx_unlock(&fcr->lock); 699 700 sop->ses = cse->ses; 701 702 /* return hardware/driver id */ 703 sop->crid = crypto_ses2hid(cse->cses); 704 bail: 705 if (error) { 706 free(key, M_XDATA); 707 free(mackey, M_XDATA); 708 } 709 return (error); 710 } 711 712 static struct csession * 713 cse_find(struct fcrypt *fcr, u_int ses) 714 { 715 struct csession *cse; 716 717 mtx_lock(&fcr->lock); 718 TAILQ_FOREACH(cse, &fcr->csessions, next) { 719 if (cse->ses == ses) { 720 refcount_acquire(&cse->refs); 721 mtx_unlock(&fcr->lock); 722 return (cse); 723 } 724 } 725 mtx_unlock(&fcr->lock); 726 return (NULL); 727 } 728 729 static void 730 cse_free(struct csession *cse) 731 { 732 733 if (!refcount_release(&cse->refs)) 734 return; 735 crypto_freesession(cse->cses); 736 mtx_destroy(&cse->lock); 737 if (cse->key) 738 free(cse->key, M_XDATA); 739 if (cse->mackey) 740 free(cse->mackey, M_XDATA); 741 free(cse, M_XDATA); 742 } 743 744 static bool 745 cse_delete(struct fcrypt *fcr, u_int ses) 746 { 747 struct csession *cse; 748 749 mtx_lock(&fcr->lock); 750 TAILQ_FOREACH(cse, &fcr->csessions, next) { 751 if (cse->ses == ses) { 752 TAILQ_REMOVE(&fcr->csessions, cse, next); 753 mtx_unlock(&fcr->lock); 754 cse_free(cse); 755 return (true); 756 } 757 } 758 mtx_unlock(&fcr->lock); 759 return (false); 760 } 761 762 static struct cryptop_data * 763 cod_alloc(struct csession *cse, size_t aad_len, size_t len) 764 { 765 struct cryptop_data *cod; 766 767 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 768 769 cod->cse = cse; 770 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 771 if (aad_len != 0) 772 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 773 cod->buf = malloc(len, M_XDATA, M_WAITOK); 774 } else 775 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 776 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 777 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 778 return (cod); 779 } 780 781 static void 782 cod_free(struct cryptop_data *cod) 783 { 784 785 free(cod->aad, M_XDATA); 786 free(cod->obuf, M_XDATA); 787 free(cod->buf, M_XDATA); 788 free(cod, M_XDATA); 789 } 790 791 static int 792 cryptodev_cb(struct cryptop *crp) 793 { 794 struct cryptop_data *cod = crp->crp_opaque; 795 796 /* 797 * Lock to ensure the wakeup() is not missed by the loops 798 * waiting on cod->done in cryptodev_op() and 799 * cryptodev_aead(). 800 */ 801 mtx_lock(&cod->cse->lock); 802 cod->done = true; 803 mtx_unlock(&cod->cse->lock); 804 wakeup(cod); 805 return (0); 806 } 807 808 static int 809 cryptodev_op(struct csession *cse, const struct crypt_op *cop) 810 { 811 struct cryptop_data *cod = NULL; 812 struct cryptop *crp = NULL; 813 char *dst; 814 int error; 815 816 if (cop->len > 256*1024-4) { 817 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 818 return (E2BIG); 819 } 820 821 if (cse->txform) { 822 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { 823 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 824 return (EINVAL); 825 } 826 } 827 828 if (cop->mac && cse->hashsize == 0) { 829 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 830 return (EINVAL); 831 } 832 833 /* 834 * The COP_F_CIPHER_FIRST flag predates explicit session 835 * modes, but the only way it was used was for EtA so allow it 836 * as long as it is consistent with EtA. 837 */ 838 if (cop->flags & COP_F_CIPHER_FIRST) { 839 if (cop->op != COP_ENCRYPT) { 840 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 841 return (EINVAL); 842 } 843 } 844 845 cod = cod_alloc(cse, 0, cop->len + cse->hashsize); 846 dst = cop->dst; 847 848 crp = crypto_getreq(cse->cses, M_WAITOK); 849 850 error = copyin(cop->src, cod->buf, cop->len); 851 if (error) { 852 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 853 goto bail; 854 } 855 crp->crp_payload_start = 0; 856 crp->crp_payload_length = cop->len; 857 if (cse->hashsize) 858 crp->crp_digest_start = cop->len; 859 860 switch (cse->mode) { 861 case CSP_MODE_COMPRESS: 862 switch (cop->op) { 863 case COP_ENCRYPT: 864 crp->crp_op = CRYPTO_OP_COMPRESS; 865 break; 866 case COP_DECRYPT: 867 crp->crp_op = CRYPTO_OP_DECOMPRESS; 868 break; 869 default: 870 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 871 error = EINVAL; 872 goto bail; 873 } 874 break; 875 case CSP_MODE_CIPHER: 876 switch (cop->op) { 877 case COP_ENCRYPT: 878 crp->crp_op = CRYPTO_OP_ENCRYPT; 879 break; 880 case COP_DECRYPT: 881 crp->crp_op = CRYPTO_OP_DECRYPT; 882 break; 883 default: 884 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 885 error = EINVAL; 886 goto bail; 887 } 888 break; 889 case CSP_MODE_DIGEST: 890 switch (cop->op) { 891 case 0: 892 case COP_ENCRYPT: 893 case COP_DECRYPT: 894 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 895 if (cod->obuf != NULL) 896 crp->crp_digest_start = 0; 897 break; 898 default: 899 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 900 error = EINVAL; 901 goto bail; 902 } 903 break; 904 case CSP_MODE_ETA: 905 switch (cop->op) { 906 case COP_ENCRYPT: 907 crp->crp_op = CRYPTO_OP_ENCRYPT | 908 CRYPTO_OP_COMPUTE_DIGEST; 909 break; 910 case COP_DECRYPT: 911 crp->crp_op = CRYPTO_OP_DECRYPT | 912 CRYPTO_OP_VERIFY_DIGEST; 913 break; 914 default: 915 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 916 error = EINVAL; 917 goto bail; 918 } 919 break; 920 default: 921 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 922 error = EINVAL; 923 goto bail; 924 } 925 926 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 927 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 928 if (cod->obuf) 929 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 930 crp->crp_callback = cryptodev_cb; 931 crp->crp_opaque = cod; 932 933 if (cop->iv) { 934 if (cse->ivsize == 0) { 935 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 936 error = EINVAL; 937 goto bail; 938 } 939 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 940 if (error) { 941 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 942 goto bail; 943 } 944 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 945 } else if (cse->ivsize != 0) { 946 crp->crp_iv_start = 0; 947 crp->crp_payload_start += cse->ivsize; 948 crp->crp_payload_length -= cse->ivsize; 949 dst += cse->ivsize; 950 } 951 952 if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 953 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 954 cse->hashsize); 955 if (error) { 956 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 957 goto bail; 958 } 959 } 960 again: 961 /* 962 * Let the dispatch run unlocked, then, interlock against the 963 * callback before checking if the operation completed and going 964 * to sleep. This insures drivers don't inherit our lock which 965 * results in a lock order reversal between crypto_dispatch forced 966 * entry and the crypto_done callback into us. 967 */ 968 error = crypto_dispatch(crp); 969 if (error != 0) { 970 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 971 goto bail; 972 } 973 974 mtx_lock(&cse->lock); 975 while (!cod->done) 976 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 977 mtx_unlock(&cse->lock); 978 979 if (crp->crp_etype == EAGAIN) { 980 crp->crp_etype = 0; 981 crp->crp_flags &= ~CRYPTO_F_DONE; 982 cod->done = false; 983 goto again; 984 } 985 986 if (crp->crp_etype != 0) { 987 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 988 error = crp->crp_etype; 989 goto bail; 990 } 991 992 if (cop->dst != NULL) { 993 error = copyout(cod->obuf != NULL ? cod->obuf : 994 cod->buf + crp->crp_payload_start, dst, 995 crp->crp_payload_length); 996 if (error) { 997 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 998 goto bail; 999 } 1000 } 1001 1002 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1003 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1004 crp->crp_digest_start, cop->mac, cse->hashsize); 1005 if (error) { 1006 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1007 goto bail; 1008 } 1009 } 1010 1011 bail: 1012 crypto_freereq(crp); 1013 cod_free(cod); 1014 1015 return (error); 1016 } 1017 1018 static int 1019 cryptodev_aead(struct csession *cse, struct crypt_aead *caead) 1020 { 1021 struct cryptop_data *cod = NULL; 1022 struct cryptop *crp = NULL; 1023 char *dst; 1024 int error; 1025 1026 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 1027 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1028 return (E2BIG); 1029 } 1030 1031 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || 1032 (caead->len % cse->txform->blocksize) != 0) { 1033 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1034 return (EINVAL); 1035 } 1036 1037 /* 1038 * The COP_F_CIPHER_FIRST flag predates explicit session 1039 * modes, but the only way it was used was for EtA so allow it 1040 * as long as it is consistent with EtA. 1041 */ 1042 if (caead->flags & COP_F_CIPHER_FIRST) { 1043 if (caead->op != COP_ENCRYPT) { 1044 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1045 return (EINVAL); 1046 } 1047 } 1048 1049 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize); 1050 dst = caead->dst; 1051 1052 crp = crypto_getreq(cse->cses, M_WAITOK); 1053 1054 if (cod->aad != NULL) 1055 error = copyin(caead->aad, cod->aad, caead->aadlen); 1056 else 1057 error = copyin(caead->aad, cod->buf, caead->aadlen); 1058 if (error) { 1059 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1060 goto bail; 1061 } 1062 crp->crp_aad = cod->aad; 1063 crp->crp_aad_start = 0; 1064 crp->crp_aad_length = caead->aadlen; 1065 1066 if (cod->aad != NULL) 1067 crp->crp_payload_start = 0; 1068 else 1069 crp->crp_payload_start = caead->aadlen; 1070 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 1071 caead->len); 1072 if (error) { 1073 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1074 goto bail; 1075 } 1076 crp->crp_payload_length = caead->len; 1077 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 1078 crp->crp_digest_start = crp->crp_payload_output_start + 1079 caead->len; 1080 else 1081 crp->crp_digest_start = crp->crp_payload_start + caead->len; 1082 1083 switch (cse->mode) { 1084 case CSP_MODE_AEAD: 1085 case CSP_MODE_ETA: 1086 switch (caead->op) { 1087 case COP_ENCRYPT: 1088 crp->crp_op = CRYPTO_OP_ENCRYPT | 1089 CRYPTO_OP_COMPUTE_DIGEST; 1090 break; 1091 case COP_DECRYPT: 1092 crp->crp_op = CRYPTO_OP_DECRYPT | 1093 CRYPTO_OP_VERIFY_DIGEST; 1094 break; 1095 default: 1096 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1097 error = EINVAL; 1098 goto bail; 1099 } 1100 break; 1101 default: 1102 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1103 error = EINVAL; 1104 goto bail; 1105 } 1106 1107 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 1108 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 1109 cse->hashsize); 1110 if (cod->obuf != NULL) 1111 crypto_use_output_buf(crp, cod->obuf, caead->len + 1112 cse->hashsize); 1113 crp->crp_callback = cryptodev_cb; 1114 crp->crp_opaque = cod; 1115 1116 if (caead->iv) { 1117 /* 1118 * Permit a 16-byte IV for AES-XTS, but only use the 1119 * first 8 bytes as a block number. 1120 */ 1121 if (cse->mode == CSP_MODE_ETA && 1122 caead->ivlen == AES_BLOCK_LEN && 1123 cse->ivsize == AES_XTS_IV_LEN) 1124 caead->ivlen = AES_XTS_IV_LEN; 1125 1126 if (caead->ivlen != cse->ivsize) { 1127 error = EINVAL; 1128 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1129 goto bail; 1130 } 1131 1132 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 1133 if (error) { 1134 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1135 goto bail; 1136 } 1137 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1138 } else { 1139 crp->crp_iv_start = crp->crp_payload_start; 1140 crp->crp_payload_start += cse->ivsize; 1141 crp->crp_payload_length -= cse->ivsize; 1142 dst += cse->ivsize; 1143 } 1144 1145 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1146 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1147 cse->hashsize); 1148 if (error) { 1149 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1150 goto bail; 1151 } 1152 } 1153 again: 1154 /* 1155 * Let the dispatch run unlocked, then, interlock against the 1156 * callback before checking if the operation completed and going 1157 * to sleep. This insures drivers don't inherit our lock which 1158 * results in a lock order reversal between crypto_dispatch forced 1159 * entry and the crypto_done callback into us. 1160 */ 1161 error = crypto_dispatch(crp); 1162 if (error != 0) { 1163 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1164 goto bail; 1165 } 1166 1167 mtx_lock(&cse->lock); 1168 while (!cod->done) 1169 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1170 mtx_unlock(&cse->lock); 1171 1172 if (crp->crp_etype == EAGAIN) { 1173 crp->crp_etype = 0; 1174 crp->crp_flags &= ~CRYPTO_F_DONE; 1175 cod->done = false; 1176 goto again; 1177 } 1178 1179 if (crp->crp_etype != 0) { 1180 error = crp->crp_etype; 1181 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1182 goto bail; 1183 } 1184 1185 if (caead->dst != NULL) { 1186 error = copyout(cod->obuf != NULL ? cod->obuf : 1187 cod->buf + crp->crp_payload_start, dst, 1188 crp->crp_payload_length); 1189 if (error) { 1190 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1191 goto bail; 1192 } 1193 } 1194 1195 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1196 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1197 crp->crp_digest_start, caead->tag, cse->hashsize); 1198 if (error) { 1199 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1200 goto bail; 1201 } 1202 } 1203 1204 bail: 1205 crypto_freereq(crp); 1206 cod_free(cod); 1207 1208 return (error); 1209 } 1210 1211 static void 1212 cryptodevkey_cb(struct cryptkop *krp) 1213 { 1214 1215 wakeup_one(krp); 1216 } 1217 1218 static int 1219 cryptodev_key(struct crypt_kop *kop) 1220 { 1221 struct cryptkop *krp = NULL; 1222 int error = EINVAL; 1223 int in, out, size, i; 1224 1225 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) { 1226 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1227 return (EFBIG); 1228 } 1229 1230 in = kop->crk_iparams; 1231 out = kop->crk_oparams; 1232 switch (kop->crk_op) { 1233 case CRK_MOD_EXP: 1234 if (in == 3 && out == 1) 1235 break; 1236 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1237 return (EINVAL); 1238 case CRK_MOD_EXP_CRT: 1239 if (in == 6 && out == 1) 1240 break; 1241 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1242 return (EINVAL); 1243 case CRK_DSA_SIGN: 1244 if (in == 5 && out == 2) 1245 break; 1246 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1247 return (EINVAL); 1248 case CRK_DSA_VERIFY: 1249 if (in == 7 && out == 0) 1250 break; 1251 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1252 return (EINVAL); 1253 case CRK_DH_COMPUTE_KEY: 1254 if (in == 3 && out == 1) 1255 break; 1256 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1257 return (EINVAL); 1258 default: 1259 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1260 return (EINVAL); 1261 } 1262 1263 krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO); 1264 krp->krp_op = kop->crk_op; 1265 krp->krp_status = kop->crk_status; 1266 krp->krp_iparams = kop->crk_iparams; 1267 krp->krp_oparams = kop->crk_oparams; 1268 krp->krp_crid = kop->crk_crid; 1269 krp->krp_status = 0; 1270 krp->krp_callback = cryptodevkey_cb; 1271 1272 for (i = 0; i < CRK_MAXPARAM; i++) { 1273 if (kop->crk_param[i].crp_nbits > 65536) { 1274 /* Limit is the same as in OpenBSD */ 1275 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1276 goto fail; 1277 } 1278 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 1279 } 1280 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1281 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1282 if (size == 0) 1283 continue; 1284 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK); 1285 if (i >= krp->krp_iparams) 1286 continue; 1287 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size); 1288 if (error) { 1289 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1290 goto fail; 1291 } 1292 } 1293 1294 error = crypto_kdispatch(krp); 1295 if (error) { 1296 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1297 goto fail; 1298 } 1299 error = tsleep(krp, PSOCK, "crydev", 0); 1300 if (error) { 1301 /* XXX can this happen? if so, how do we recover? */ 1302 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1303 goto fail; 1304 } 1305 1306 kop->crk_crid = krp->krp_hid; /* device that did the work */ 1307 if (krp->krp_status != 0) { 1308 error = krp->krp_status; 1309 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1310 goto fail; 1311 } 1312 1313 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) { 1314 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1315 if (size == 0) 1316 continue; 1317 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size); 1318 if (error) { 1319 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1320 goto fail; 1321 } 1322 } 1323 1324 fail: 1325 if (krp) { 1326 kop->crk_status = krp->krp_status; 1327 for (i = 0; i < CRK_MAXPARAM; i++) { 1328 if (krp->krp_param[i].crp_p) 1329 free(krp->krp_param[i].crp_p, M_XDATA); 1330 } 1331 free(krp, M_XDATA); 1332 } 1333 return (error); 1334 } 1335 1336 static int 1337 cryptodev_find(struct crypt_find_op *find) 1338 { 1339 device_t dev; 1340 size_t fnlen = sizeof find->name; 1341 1342 if (find->crid != -1) { 1343 dev = crypto_find_device_byhid(find->crid); 1344 if (dev == NULL) 1345 return (ENOENT); 1346 strncpy(find->name, device_get_nameunit(dev), fnlen); 1347 find->name[fnlen - 1] = '\x0'; 1348 } else { 1349 find->name[fnlen - 1] = '\x0'; 1350 find->crid = crypto_find_driver(find->name); 1351 if (find->crid == -1) 1352 return (ENOENT); 1353 } 1354 return (0); 1355 } 1356 1357 static void 1358 fcrypt_dtor(void *data) 1359 { 1360 struct fcrypt *fcr = data; 1361 struct csession *cse; 1362 1363 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1364 TAILQ_REMOVE(&fcr->csessions, cse, next); 1365 KASSERT(refcount_load(&cse->refs) == 1, 1366 ("%s: crypto session %p with %d refs", __func__, cse, 1367 refcount_load(&cse->refs))); 1368 cse_free(cse); 1369 } 1370 mtx_destroy(&fcr->lock); 1371 free(fcr, M_XDATA); 1372 } 1373 1374 static int 1375 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1376 { 1377 struct fcrypt *fcr; 1378 int error; 1379 1380 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1381 TAILQ_INIT(&fcr->csessions); 1382 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1383 error = devfs_set_cdevpriv(fcr, fcrypt_dtor); 1384 if (error) 1385 fcrypt_dtor(fcr); 1386 return (error); 1387 } 1388 1389 static int 1390 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, 1391 struct thread *td) 1392 { 1393 static struct timeval keywarn, featwarn; 1394 struct fcrypt *fcr; 1395 struct csession *cse; 1396 struct session2_op *sop; 1397 struct crypt_op *cop; 1398 struct crypt_aead *caead; 1399 struct crypt_kop *kop; 1400 uint32_t ses; 1401 int error = 0; 1402 union { 1403 struct session2_op sopc; 1404 #ifdef COMPAT_FREEBSD32 1405 struct crypt_op copc; 1406 struct crypt_aead aeadc; 1407 struct crypt_kop kopc; 1408 #endif 1409 } thunk; 1410 #ifdef COMPAT_FREEBSD32 1411 u_long cmd32; 1412 void *data32; 1413 1414 cmd32 = 0; 1415 data32 = NULL; 1416 switch (cmd) { 1417 case CIOCGSESSION32: 1418 cmd32 = cmd; 1419 data32 = data; 1420 cmd = CIOCGSESSION; 1421 data = (void *)&thunk.sopc; 1422 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 1423 break; 1424 case CIOCGSESSION232: 1425 cmd32 = cmd; 1426 data32 = data; 1427 cmd = CIOCGSESSION2; 1428 data = (void *)&thunk.sopc; 1429 session2_op_from_32((struct session2_op32 *)data32, 1430 &thunk.sopc); 1431 break; 1432 case CIOCCRYPT32: 1433 cmd32 = cmd; 1434 data32 = data; 1435 cmd = CIOCCRYPT; 1436 data = (void *)&thunk.copc; 1437 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 1438 break; 1439 case CIOCCRYPTAEAD32: 1440 cmd32 = cmd; 1441 data32 = data; 1442 cmd = CIOCCRYPTAEAD; 1443 data = (void *)&thunk.aeadc; 1444 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 1445 break; 1446 case CIOCKEY32: 1447 case CIOCKEY232: 1448 cmd32 = cmd; 1449 data32 = data; 1450 if (cmd == CIOCKEY32) 1451 cmd = CIOCKEY; 1452 else 1453 cmd = CIOCKEY2; 1454 data = (void *)&thunk.kopc; 1455 crypt_kop_from_32((struct crypt_kop32 *)data32, &thunk.kopc); 1456 break; 1457 } 1458 #endif 1459 1460 devfs_get_cdevpriv((void **)&fcr); 1461 1462 switch (cmd) { 1463 #ifdef COMPAT_FREEBSD12 1464 case CRIOGET: 1465 /* 1466 * NB: This may fail in cases that the old 1467 * implementation did not if the current process has 1468 * restricted filesystem access (e.g. running in a 1469 * jail that does not expose /dev/crypto or in 1470 * capability mode). 1471 */ 1472 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE, 1473 O_RDWR, 0); 1474 if (error == 0) 1475 *(uint32_t *)data = td->td_retval[0]; 1476 break; 1477 #endif 1478 case CIOCGSESSION: 1479 case CIOCGSESSION2: 1480 if (cmd == CIOCGSESSION) { 1481 session2_op_from_op((void *)data, &thunk.sopc); 1482 sop = &thunk.sopc; 1483 } else 1484 sop = (struct session2_op *)data; 1485 1486 error = cse_create(fcr, sop); 1487 if (cmd == CIOCGSESSION && error == 0) 1488 session2_op_to_op(sop, (void *)data); 1489 break; 1490 case CIOCFSESSION: 1491 ses = *(uint32_t *)data; 1492 if (!cse_delete(fcr, ses)) { 1493 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1494 return (EINVAL); 1495 } 1496 break; 1497 case CIOCCRYPT: 1498 cop = (struct crypt_op *)data; 1499 cse = cse_find(fcr, cop->ses); 1500 if (cse == NULL) { 1501 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1502 return (EINVAL); 1503 } 1504 error = cryptodev_op(cse, cop); 1505 cse_free(cse); 1506 break; 1507 case CIOCKEY: 1508 case CIOCKEY2: 1509 if (ratecheck(&keywarn, &warninterval)) 1510 gone_in(14, 1511 "Asymmetric crypto operations via /dev/crypto"); 1512 1513 if (!crypto_userasymcrypto) { 1514 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1515 return (EPERM); /* XXX compat? */ 1516 } 1517 kop = (struct crypt_kop *)data; 1518 if (cmd == CIOCKEY) { 1519 /* NB: crypto core enforces s/w driver use */ 1520 kop->crk_crid = 1521 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE; 1522 } 1523 mtx_lock(&Giant); 1524 error = cryptodev_key(kop); 1525 mtx_unlock(&Giant); 1526 break; 1527 case CIOCASYMFEAT: 1528 if (ratecheck(&featwarn, &warninterval)) 1529 gone_in(14, 1530 "Asymmetric crypto features via /dev/crypto"); 1531 1532 if (!crypto_userasymcrypto) { 1533 /* 1534 * NB: if user asym crypto operations are 1535 * not permitted return "no algorithms" 1536 * so well-behaved applications will just 1537 * fallback to doing them in software. 1538 */ 1539 *(int *)data = 0; 1540 } else { 1541 error = crypto_getfeat((int *)data); 1542 if (error) 1543 SDT_PROBE1(opencrypto, dev, ioctl, error, 1544 __LINE__); 1545 } 1546 break; 1547 case CIOCFINDDEV: 1548 error = cryptodev_find((struct crypt_find_op *)data); 1549 break; 1550 case CIOCCRYPTAEAD: 1551 caead = (struct crypt_aead *)data; 1552 cse = cse_find(fcr, caead->ses); 1553 if (cse == NULL) { 1554 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1555 return (EINVAL); 1556 } 1557 error = cryptodev_aead(cse, caead); 1558 cse_free(cse); 1559 break; 1560 default: 1561 error = EINVAL; 1562 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1563 break; 1564 } 1565 1566 #ifdef COMPAT_FREEBSD32 1567 switch (cmd32) { 1568 case CIOCGSESSION32: 1569 if (error == 0) 1570 session_op_to_32((void *)data, data32); 1571 break; 1572 case CIOCGSESSION232: 1573 if (error == 0) 1574 session2_op_to_32((void *)data, data32); 1575 break; 1576 case CIOCCRYPT32: 1577 if (error == 0) 1578 crypt_op_to_32((void *)data, data32); 1579 break; 1580 case CIOCCRYPTAEAD32: 1581 if (error == 0) 1582 crypt_aead_to_32((void *)data, data32); 1583 break; 1584 case CIOCKEY32: 1585 case CIOCKEY232: 1586 crypt_kop_to_32((void *)data, data32); 1587 break; 1588 } 1589 #endif 1590 return (error); 1591 } 1592 1593 static struct cdevsw crypto_cdevsw = { 1594 .d_version = D_VERSION, 1595 .d_open = crypto_open, 1596 .d_ioctl = crypto_ioctl, 1597 .d_name = "crypto", 1598 }; 1599 static struct cdev *crypto_dev; 1600 1601 /* 1602 * Initialization code, both for static and dynamic loading. 1603 */ 1604 static int 1605 cryptodev_modevent(module_t mod, int type, void *unused) 1606 { 1607 switch (type) { 1608 case MOD_LOAD: 1609 if (bootverbose) 1610 printf("crypto: <crypto device>\n"); 1611 crypto_dev = make_dev(&crypto_cdevsw, 0, 1612 UID_ROOT, GID_WHEEL, 0666, 1613 "crypto"); 1614 return 0; 1615 case MOD_UNLOAD: 1616 /*XXX disallow if active sessions */ 1617 destroy_dev(crypto_dev); 1618 return 0; 1619 } 1620 return EINVAL; 1621 } 1622 1623 static moduledata_t cryptodev_mod = { 1624 "cryptodev", 1625 cryptodev_modevent, 1626 0 1627 }; 1628 MODULE_VERSION(cryptodev, 1); 1629 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1630 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1631 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1632