1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 30 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 34 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Effort sponsored in part by the Defense Advanced Research Projects 37 * Agency (DARPA) and Air Force Research Laboratory, Air Force 38 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/sysctl.h> 51 #include <sys/file.h> 52 #include <sys/filedesc.h> 53 #include <sys/errno.h> 54 #include <sys/random.h> 55 #include <sys/conf.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/fcntl.h> 59 #include <sys/bus.h> 60 #include <sys/user.h> 61 #include <sys/sdt.h> 62 63 #include <opencrypto/cryptodev.h> 64 #include <opencrypto/xform.h> 65 66 SDT_PROVIDER_DECLARE(opencrypto); 67 68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 69 70 #ifdef COMPAT_FREEBSD32 71 #include <sys/mount.h> 72 #include <compat/freebsd32/freebsd32.h> 73 74 struct session_op32 { 75 uint32_t cipher; 76 uint32_t mac; 77 uint32_t keylen; 78 uint32_t key; 79 int mackeylen; 80 uint32_t mackey; 81 uint32_t ses; 82 }; 83 84 struct session2_op32 { 85 uint32_t cipher; 86 uint32_t mac; 87 uint32_t keylen; 88 uint32_t key; 89 int mackeylen; 90 uint32_t mackey; 91 uint32_t ses; 92 int crid; 93 int pad[4]; 94 }; 95 96 struct crypt_op32 { 97 uint32_t ses; 98 uint16_t op; 99 uint16_t flags; 100 u_int len; 101 uint32_t src, dst; 102 uint32_t mac; 103 uint32_t iv; 104 }; 105 106 struct crypt_aead32 { 107 uint32_t ses; 108 uint16_t op; 109 uint16_t flags; 110 u_int len; 111 u_int aadlen; 112 u_int ivlen; 113 uint32_t src; 114 uint32_t dst; 115 uint32_t aad; 116 uint32_t tag; 117 uint32_t iv; 118 }; 119 120 struct crparam32 { 121 uint32_t crp_p; 122 u_int crp_nbits; 123 }; 124 125 struct crypt_kop32 { 126 u_int crk_op; 127 u_int crk_status; 128 u_short crk_iparams; 129 u_short crk_oparams; 130 u_int crk_crid; 131 struct crparam32 crk_param[CRK_MAXPARAM]; 132 }; 133 134 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 135 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 136 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32) 137 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 138 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32) 139 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 140 141 static void 142 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 143 { 144 145 memset(to, 0, sizeof(*to)); 146 CP(*from, *to, cipher); 147 CP(*from, *to, mac); 148 CP(*from, *to, keylen); 149 PTRIN_CP(*from, *to, key); 150 CP(*from, *to, mackeylen); 151 PTRIN_CP(*from, *to, mackey); 152 CP(*from, *to, ses); 153 to->crid = CRYPTOCAP_F_HARDWARE; 154 } 155 156 static void 157 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 158 { 159 160 session_op_from_32((const struct session_op32 *)from, to); 161 CP(*from, *to, crid); 162 } 163 164 static void 165 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 166 { 167 168 CP(*from, *to, cipher); 169 CP(*from, *to, mac); 170 CP(*from, *to, keylen); 171 PTROUT_CP(*from, *to, key); 172 CP(*from, *to, mackeylen); 173 PTROUT_CP(*from, *to, mackey); 174 CP(*from, *to, ses); 175 } 176 177 static void 178 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 179 { 180 181 session_op_to_32(from, (struct session_op32 *)to); 182 CP(*from, *to, crid); 183 } 184 185 static void 186 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 187 { 188 189 CP(*from, *to, ses); 190 CP(*from, *to, op); 191 CP(*from, *to, flags); 192 CP(*from, *to, len); 193 PTRIN_CP(*from, *to, src); 194 PTRIN_CP(*from, *to, dst); 195 PTRIN_CP(*from, *to, mac); 196 PTRIN_CP(*from, *to, iv); 197 } 198 199 static void 200 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 201 { 202 203 CP(*from, *to, ses); 204 CP(*from, *to, op); 205 CP(*from, *to, flags); 206 CP(*from, *to, len); 207 PTROUT_CP(*from, *to, src); 208 PTROUT_CP(*from, *to, dst); 209 PTROUT_CP(*from, *to, mac); 210 PTROUT_CP(*from, *to, iv); 211 } 212 213 static void 214 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 215 { 216 217 CP(*from, *to, ses); 218 CP(*from, *to, op); 219 CP(*from, *to, flags); 220 CP(*from, *to, len); 221 CP(*from, *to, aadlen); 222 CP(*from, *to, ivlen); 223 PTRIN_CP(*from, *to, src); 224 PTRIN_CP(*from, *to, dst); 225 PTRIN_CP(*from, *to, aad); 226 PTRIN_CP(*from, *to, tag); 227 PTRIN_CP(*from, *to, iv); 228 } 229 230 static void 231 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 232 { 233 234 CP(*from, *to, ses); 235 CP(*from, *to, op); 236 CP(*from, *to, flags); 237 CP(*from, *to, len); 238 CP(*from, *to, aadlen); 239 CP(*from, *to, ivlen); 240 PTROUT_CP(*from, *to, src); 241 PTROUT_CP(*from, *to, dst); 242 PTROUT_CP(*from, *to, aad); 243 PTROUT_CP(*from, *to, tag); 244 PTROUT_CP(*from, *to, iv); 245 } 246 247 static void 248 crparam_from_32(const struct crparam32 *from, struct crparam *to) 249 { 250 251 PTRIN_CP(*from, *to, crp_p); 252 CP(*from, *to, crp_nbits); 253 } 254 255 static void 256 crparam_to_32(const struct crparam *from, struct crparam32 *to) 257 { 258 259 PTROUT_CP(*from, *to, crp_p); 260 CP(*from, *to, crp_nbits); 261 } 262 263 static void 264 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to) 265 { 266 int i; 267 268 CP(*from, *to, crk_op); 269 CP(*from, *to, crk_status); 270 CP(*from, *to, crk_iparams); 271 CP(*from, *to, crk_oparams); 272 CP(*from, *to, crk_crid); 273 for (i = 0; i < CRK_MAXPARAM; i++) 274 crparam_from_32(&from->crk_param[i], &to->crk_param[i]); 275 } 276 277 static void 278 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to) 279 { 280 int i; 281 282 CP(*from, *to, crk_op); 283 CP(*from, *to, crk_status); 284 CP(*from, *to, crk_iparams); 285 CP(*from, *to, crk_oparams); 286 CP(*from, *to, crk_crid); 287 for (i = 0; i < CRK_MAXPARAM; i++) 288 crparam_to_32(&from->crk_param[i], &to->crk_param[i]); 289 } 290 #endif 291 292 static void 293 session2_op_from_op(const struct session_op *from, struct session2_op *to) 294 { 295 296 memset(to, 0, sizeof(*to)); 297 memcpy(to, from, sizeof(*from)); 298 to->crid = CRYPTOCAP_F_HARDWARE; 299 } 300 301 static void 302 session2_op_to_op(const struct session2_op *from, struct session_op *to) 303 { 304 305 memcpy(to, from, sizeof(*to)); 306 } 307 308 struct csession { 309 TAILQ_ENTRY(csession) next; 310 crypto_session_t cses; 311 volatile u_int refs; 312 uint32_t ses; 313 struct mtx lock; /* for op submission */ 314 315 struct enc_xform *txform; 316 int hashsize; 317 int ivsize; 318 int mode; 319 320 void *key; 321 void *mackey; 322 }; 323 324 struct cryptop_data { 325 struct csession *cse; 326 327 char *buf; 328 char *obuf; 329 char *aad; 330 bool done; 331 }; 332 333 struct fcrypt { 334 TAILQ_HEAD(csessionlist, csession) csessions; 335 int sesn; 336 struct mtx lock; 337 }; 338 339 static bool use_outputbuffers; 340 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 341 &use_outputbuffers, 0, 342 "Use separate output buffers for /dev/crypto requests."); 343 344 static bool use_separate_aad; 345 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 346 &use_separate_aad, 0, 347 "Use separate AAD buffer for /dev/crypto requests."); 348 349 static struct timeval warninterval = { .tv_sec = 60, .tv_usec = 0 }; 350 SYSCTL_TIMEVAL_SEC(_kern, OID_AUTO, cryptodev_warn_interval, CTLFLAG_RW, 351 &warninterval, 352 "Delay in seconds between warnings of deprecated /dev/crypto algorithms"); 353 354 static int cryptof_ioctl(struct file *, u_long, void *, struct ucred *, 355 struct thread *); 356 static int cryptof_stat(struct file *, struct stat *, struct ucred *, 357 struct thread *); 358 static int cryptof_close(struct file *, struct thread *); 359 static int cryptof_fill_kinfo(struct file *, struct kinfo_file *, 360 struct filedesc *); 361 362 static struct fileops cryptofops = { 363 .fo_read = invfo_rdwr, 364 .fo_write = invfo_rdwr, 365 .fo_truncate = invfo_truncate, 366 .fo_ioctl = cryptof_ioctl, 367 .fo_poll = invfo_poll, 368 .fo_kqfilter = invfo_kqfilter, 369 .fo_stat = cryptof_stat, 370 .fo_close = cryptof_close, 371 .fo_chmod = invfo_chmod, 372 .fo_chown = invfo_chown, 373 .fo_sendfile = invfo_sendfile, 374 .fo_fill_kinfo = cryptof_fill_kinfo, 375 }; 376 377 /* 378 * Check a crypto identifier to see if it requested 379 * a software device/driver. This can be done either 380 * by device name/class or through search constraints. 381 */ 382 static int 383 checkforsoftware(int *cridp) 384 { 385 int crid; 386 387 crid = *cridp; 388 389 if (!crypto_devallowsoft) { 390 if (crid & CRYPTOCAP_F_SOFTWARE) { 391 if (crid & CRYPTOCAP_F_HARDWARE) { 392 *cridp = CRYPTOCAP_F_HARDWARE; 393 return 0; 394 } 395 return EINVAL; 396 } 397 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 398 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 399 return EINVAL; 400 } 401 return 0; 402 } 403 404 static int 405 cse_create(struct fcrypt *fcr, struct session2_op *sop) 406 { 407 struct crypto_session_params csp; 408 struct csession *cse; 409 struct enc_xform *txform; 410 struct auth_hash *thash; 411 void *key = NULL; 412 void *mackey = NULL; 413 crypto_session_t cses; 414 int crid, error; 415 416 switch (sop->cipher) { 417 case 0: 418 txform = NULL; 419 break; 420 case CRYPTO_AES_CBC: 421 txform = &enc_xform_rijndael128; 422 break; 423 case CRYPTO_AES_XTS: 424 txform = &enc_xform_aes_xts; 425 break; 426 case CRYPTO_NULL_CBC: 427 txform = &enc_xform_null; 428 break; 429 case CRYPTO_CAMELLIA_CBC: 430 txform = &enc_xform_camellia; 431 break; 432 case CRYPTO_AES_ICM: 433 txform = &enc_xform_aes_icm; 434 break; 435 case CRYPTO_AES_NIST_GCM_16: 436 txform = &enc_xform_aes_nist_gcm; 437 break; 438 case CRYPTO_CHACHA20: 439 txform = &enc_xform_chacha20; 440 break; 441 case CRYPTO_AES_CCM_16: 442 txform = &enc_xform_ccm; 443 break; 444 default: 445 CRYPTDEB("invalid cipher"); 446 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 447 return (EINVAL); 448 } 449 450 switch (sop->mac) { 451 case 0: 452 thash = NULL; 453 break; 454 case CRYPTO_POLY1305: 455 thash = &auth_hash_poly1305; 456 break; 457 case CRYPTO_SHA1_HMAC: 458 thash = &auth_hash_hmac_sha1; 459 break; 460 case CRYPTO_SHA2_224_HMAC: 461 thash = &auth_hash_hmac_sha2_224; 462 break; 463 case CRYPTO_SHA2_256_HMAC: 464 thash = &auth_hash_hmac_sha2_256; 465 break; 466 case CRYPTO_SHA2_384_HMAC: 467 thash = &auth_hash_hmac_sha2_384; 468 break; 469 case CRYPTO_SHA2_512_HMAC: 470 thash = &auth_hash_hmac_sha2_512; 471 break; 472 case CRYPTO_RIPEMD160_HMAC: 473 thash = &auth_hash_hmac_ripemd_160; 474 break; 475 #ifdef COMPAT_FREEBSD12 476 case CRYPTO_AES_128_NIST_GMAC: 477 case CRYPTO_AES_192_NIST_GMAC: 478 case CRYPTO_AES_256_NIST_GMAC: 479 /* Should always be paired with GCM. */ 480 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 481 CRYPTDEB("GMAC without GCM"); 482 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 483 return (EINVAL); 484 } 485 break; 486 #endif 487 case CRYPTO_AES_NIST_GMAC: 488 switch (sop->mackeylen * 8) { 489 case 128: 490 thash = &auth_hash_nist_gmac_aes_128; 491 break; 492 case 192: 493 thash = &auth_hash_nist_gmac_aes_192; 494 break; 495 case 256: 496 thash = &auth_hash_nist_gmac_aes_256; 497 break; 498 default: 499 CRYPTDEB("invalid GMAC key length"); 500 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 501 return (EINVAL); 502 } 503 break; 504 case CRYPTO_AES_CCM_CBC_MAC: 505 switch (sop->mackeylen) { 506 case 16: 507 thash = &auth_hash_ccm_cbc_mac_128; 508 break; 509 case 24: 510 thash = &auth_hash_ccm_cbc_mac_192; 511 break; 512 case 32: 513 thash = &auth_hash_ccm_cbc_mac_256; 514 break; 515 default: 516 CRYPTDEB("Invalid CBC MAC key size %d", sop->keylen); 517 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 518 return (EINVAL); 519 } 520 break; 521 case CRYPTO_SHA1: 522 thash = &auth_hash_sha1; 523 break; 524 case CRYPTO_SHA2_224: 525 thash = &auth_hash_sha2_224; 526 break; 527 case CRYPTO_SHA2_256: 528 thash = &auth_hash_sha2_256; 529 break; 530 case CRYPTO_SHA2_384: 531 thash = &auth_hash_sha2_384; 532 break; 533 case CRYPTO_SHA2_512: 534 thash = &auth_hash_sha2_512; 535 break; 536 537 case CRYPTO_NULL_HMAC: 538 thash = &auth_hash_null; 539 break; 540 541 case CRYPTO_BLAKE2B: 542 thash = &auth_hash_blake2b; 543 break; 544 case CRYPTO_BLAKE2S: 545 thash = &auth_hash_blake2s; 546 break; 547 548 default: 549 CRYPTDEB("invalid mac"); 550 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 551 return (EINVAL); 552 } 553 554 if (txform == NULL && thash == NULL) { 555 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 556 return (EINVAL); 557 } 558 559 memset(&csp, 0, sizeof(csp)); 560 if (use_outputbuffers) 561 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 562 563 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { 564 switch (sop->mac) { 565 #ifdef COMPAT_FREEBSD12 566 case CRYPTO_AES_128_NIST_GMAC: 567 case CRYPTO_AES_192_NIST_GMAC: 568 case CRYPTO_AES_256_NIST_GMAC: 569 if (sop->keylen != sop->mackeylen) { 570 SDT_PROBE1(opencrypto, dev, ioctl, error, 571 __LINE__); 572 return (EINVAL); 573 } 574 break; 575 #endif 576 case 0: 577 break; 578 default: 579 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 580 return (EINVAL); 581 } 582 csp.csp_mode = CSP_MODE_AEAD; 583 } else if (sop->cipher == CRYPTO_AES_CCM_16) { 584 switch (sop->mac) { 585 #ifdef COMPAT_FREEBSD12 586 case CRYPTO_AES_CCM_CBC_MAC: 587 if (sop->keylen != sop->mackeylen) { 588 SDT_PROBE1(opencrypto, dev, ioctl, error, 589 __LINE__); 590 return (EINVAL); 591 } 592 thash = NULL; 593 break; 594 #endif 595 case 0: 596 break; 597 default: 598 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 599 return (EINVAL); 600 } 601 csp.csp_mode = CSP_MODE_AEAD; 602 } else if (txform != NULL && thash != NULL) 603 csp.csp_mode = CSP_MODE_ETA; 604 else if (txform != NULL) 605 csp.csp_mode = CSP_MODE_CIPHER; 606 else 607 csp.csp_mode = CSP_MODE_DIGEST; 608 609 switch (csp.csp_mode) { 610 case CSP_MODE_AEAD: 611 case CSP_MODE_ETA: 612 if (use_separate_aad) 613 csp.csp_flags |= CSP_F_SEPARATE_AAD; 614 break; 615 } 616 617 if (txform != NULL) { 618 csp.csp_cipher_alg = txform->type; 619 csp.csp_cipher_klen = sop->keylen; 620 if (sop->keylen > txform->maxkey || 621 sop->keylen < txform->minkey) { 622 CRYPTDEB("invalid cipher parameters"); 623 error = EINVAL; 624 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 625 goto bail; 626 } 627 628 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 629 error = copyin(sop->key, key, csp.csp_cipher_klen); 630 if (error) { 631 CRYPTDEB("invalid key"); 632 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 633 goto bail; 634 } 635 csp.csp_cipher_key = key; 636 csp.csp_ivlen = txform->ivsize; 637 } 638 639 if (thash != NULL) { 640 csp.csp_auth_alg = thash->type; 641 csp.csp_auth_klen = sop->mackeylen; 642 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { 643 CRYPTDEB("invalid mac key length"); 644 error = EINVAL; 645 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 646 goto bail; 647 } 648 649 if (csp.csp_auth_klen != 0) { 650 mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK); 651 error = copyin(sop->mackey, mackey, csp.csp_auth_klen); 652 if (error) { 653 CRYPTDEB("invalid mac key"); 654 SDT_PROBE1(opencrypto, dev, ioctl, error, 655 __LINE__); 656 goto bail; 657 } 658 csp.csp_auth_key = mackey; 659 } 660 661 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 662 csp.csp_ivlen = AES_GCM_IV_LEN; 663 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 664 csp.csp_ivlen = AES_CCM_IV_LEN; 665 } 666 667 crid = sop->crid; 668 error = checkforsoftware(&crid); 669 if (error) { 670 CRYPTDEB("checkforsoftware"); 671 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 672 goto bail; 673 } 674 error = crypto_newsession(&cses, &csp, crid); 675 if (error) { 676 CRYPTDEB("crypto_newsession"); 677 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 678 goto bail; 679 } 680 681 cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO); 682 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 683 refcount_init(&cse->refs, 1); 684 cse->key = key; 685 cse->mackey = mackey; 686 cse->mode = csp.csp_mode; 687 cse->cses = cses; 688 cse->txform = txform; 689 if (thash != NULL) 690 cse->hashsize = thash->hashsize; 691 else if (csp.csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) 692 cse->hashsize = AES_GMAC_HASH_LEN; 693 else if (csp.csp_cipher_alg == CRYPTO_AES_CCM_16) 694 cse->hashsize = AES_CBC_MAC_HASH_LEN; 695 cse->ivsize = csp.csp_ivlen; 696 697 mtx_lock(&fcr->lock); 698 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 699 cse->ses = fcr->sesn++; 700 mtx_unlock(&fcr->lock); 701 702 sop->ses = cse->ses; 703 704 /* return hardware/driver id */ 705 sop->crid = crypto_ses2hid(cse->cses); 706 bail: 707 if (error) { 708 free(key, M_XDATA); 709 free(mackey, M_XDATA); 710 } 711 return (error); 712 } 713 714 static struct csession * 715 cse_find(struct fcrypt *fcr, u_int ses) 716 { 717 struct csession *cse; 718 719 mtx_lock(&fcr->lock); 720 TAILQ_FOREACH(cse, &fcr->csessions, next) { 721 if (cse->ses == ses) { 722 refcount_acquire(&cse->refs); 723 mtx_unlock(&fcr->lock); 724 return (cse); 725 } 726 } 727 mtx_unlock(&fcr->lock); 728 return (NULL); 729 } 730 731 static void 732 cse_free(struct csession *cse) 733 { 734 735 if (!refcount_release(&cse->refs)) 736 return; 737 crypto_freesession(cse->cses); 738 mtx_destroy(&cse->lock); 739 if (cse->key) 740 free(cse->key, M_XDATA); 741 if (cse->mackey) 742 free(cse->mackey, M_XDATA); 743 free(cse, M_XDATA); 744 } 745 746 static bool 747 cse_delete(struct fcrypt *fcr, u_int ses) 748 { 749 struct csession *cse; 750 751 mtx_lock(&fcr->lock); 752 TAILQ_FOREACH(cse, &fcr->csessions, next) { 753 if (cse->ses == ses) { 754 TAILQ_REMOVE(&fcr->csessions, cse, next); 755 mtx_unlock(&fcr->lock); 756 cse_free(cse); 757 return (true); 758 } 759 } 760 mtx_unlock(&fcr->lock); 761 return (false); 762 } 763 764 static struct cryptop_data * 765 cod_alloc(struct csession *cse, size_t aad_len, size_t len, struct thread *td) 766 { 767 struct cryptop_data *cod; 768 769 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 770 771 cod->cse = cse; 772 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 773 if (aad_len != 0) 774 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 775 cod->buf = malloc(len, M_XDATA, M_WAITOK); 776 } else 777 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 778 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 779 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 780 return (cod); 781 } 782 783 static void 784 cod_free(struct cryptop_data *cod) 785 { 786 787 free(cod->aad, M_XDATA); 788 free(cod->obuf, M_XDATA); 789 free(cod->buf, M_XDATA); 790 free(cod, M_XDATA); 791 } 792 793 static int 794 cryptodev_cb(struct cryptop *crp) 795 { 796 struct cryptop_data *cod = crp->crp_opaque; 797 798 /* 799 * Lock to ensure the wakeup() is not missed by the loops 800 * waiting on cod->done in cryptodev_op() and 801 * cryptodev_aead(). 802 */ 803 mtx_lock(&cod->cse->lock); 804 cod->done = true; 805 mtx_unlock(&cod->cse->lock); 806 wakeup(cod); 807 return (0); 808 } 809 810 static int 811 cryptodev_op(struct csession *cse, const struct crypt_op *cop, 812 struct ucred *active_cred, struct thread *td) 813 { 814 struct cryptop_data *cod = NULL; 815 struct cryptop *crp = NULL; 816 char *dst; 817 int error; 818 819 if (cop->len > 256*1024-4) { 820 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 821 return (E2BIG); 822 } 823 824 if (cse->txform) { 825 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { 826 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 827 return (EINVAL); 828 } 829 } 830 831 if (cop->mac && cse->hashsize == 0) { 832 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 833 return (EINVAL); 834 } 835 836 /* 837 * The COP_F_CIPHER_FIRST flag predates explicit session 838 * modes, but the only way it was used was for EtA so allow it 839 * as long as it is consistent with EtA. 840 */ 841 if (cop->flags & COP_F_CIPHER_FIRST) { 842 if (cop->op != COP_ENCRYPT) { 843 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 844 return (EINVAL); 845 } 846 } 847 848 cod = cod_alloc(cse, 0, cop->len + cse->hashsize, td); 849 dst = cop->dst; 850 851 crp = crypto_getreq(cse->cses, M_WAITOK); 852 853 error = copyin(cop->src, cod->buf, cop->len); 854 if (error) { 855 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 856 goto bail; 857 } 858 crp->crp_payload_start = 0; 859 crp->crp_payload_length = cop->len; 860 if (cse->hashsize) 861 crp->crp_digest_start = cop->len; 862 863 switch (cse->mode) { 864 case CSP_MODE_COMPRESS: 865 switch (cop->op) { 866 case COP_ENCRYPT: 867 crp->crp_op = CRYPTO_OP_COMPRESS; 868 break; 869 case COP_DECRYPT: 870 crp->crp_op = CRYPTO_OP_DECOMPRESS; 871 break; 872 default: 873 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 874 error = EINVAL; 875 goto bail; 876 } 877 break; 878 case CSP_MODE_CIPHER: 879 switch (cop->op) { 880 case COP_ENCRYPT: 881 crp->crp_op = CRYPTO_OP_ENCRYPT; 882 break; 883 case COP_DECRYPT: 884 crp->crp_op = CRYPTO_OP_DECRYPT; 885 break; 886 default: 887 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 888 error = EINVAL; 889 goto bail; 890 } 891 break; 892 case CSP_MODE_DIGEST: 893 switch (cop->op) { 894 case 0: 895 case COP_ENCRYPT: 896 case COP_DECRYPT: 897 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 898 if (cod->obuf != NULL) 899 crp->crp_digest_start = 0; 900 break; 901 default: 902 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 903 error = EINVAL; 904 goto bail; 905 } 906 break; 907 case CSP_MODE_ETA: 908 switch (cop->op) { 909 case COP_ENCRYPT: 910 crp->crp_op = CRYPTO_OP_ENCRYPT | 911 CRYPTO_OP_COMPUTE_DIGEST; 912 break; 913 case COP_DECRYPT: 914 crp->crp_op = CRYPTO_OP_DECRYPT | 915 CRYPTO_OP_VERIFY_DIGEST; 916 break; 917 default: 918 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 919 error = EINVAL; 920 goto bail; 921 } 922 break; 923 default: 924 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 925 error = EINVAL; 926 goto bail; 927 } 928 929 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 930 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 931 if (cod->obuf) 932 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 933 crp->crp_callback = cryptodev_cb; 934 crp->crp_opaque = cod; 935 936 if (cop->iv) { 937 if (cse->ivsize == 0) { 938 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 939 error = EINVAL; 940 goto bail; 941 } 942 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 943 if (error) { 944 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 945 goto bail; 946 } 947 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 948 } else if (cse->ivsize != 0) { 949 crp->crp_iv_start = 0; 950 crp->crp_payload_start += cse->ivsize; 951 crp->crp_payload_length -= cse->ivsize; 952 dst += cse->ivsize; 953 } 954 955 if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 956 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 957 cse->hashsize); 958 if (error) { 959 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 960 goto bail; 961 } 962 } 963 again: 964 /* 965 * Let the dispatch run unlocked, then, interlock against the 966 * callback before checking if the operation completed and going 967 * to sleep. This insures drivers don't inherit our lock which 968 * results in a lock order reversal between crypto_dispatch forced 969 * entry and the crypto_done callback into us. 970 */ 971 error = crypto_dispatch(crp); 972 if (error != 0) { 973 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 974 goto bail; 975 } 976 977 mtx_lock(&cse->lock); 978 while (!cod->done) 979 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 980 mtx_unlock(&cse->lock); 981 982 if (crp->crp_etype == EAGAIN) { 983 crp->crp_etype = 0; 984 crp->crp_flags &= ~CRYPTO_F_DONE; 985 cod->done = false; 986 goto again; 987 } 988 989 if (crp->crp_etype != 0) { 990 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 991 error = crp->crp_etype; 992 goto bail; 993 } 994 995 if (cop->dst != NULL) { 996 error = copyout(cod->obuf != NULL ? cod->obuf : 997 cod->buf + crp->crp_payload_start, dst, 998 crp->crp_payload_length); 999 if (error) { 1000 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1001 goto bail; 1002 } 1003 } 1004 1005 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1006 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1007 crp->crp_digest_start, cop->mac, cse->hashsize); 1008 if (error) { 1009 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1010 goto bail; 1011 } 1012 } 1013 1014 bail: 1015 crypto_freereq(crp); 1016 cod_free(cod); 1017 1018 return (error); 1019 } 1020 1021 static int 1022 cryptodev_aead(struct csession *cse, struct crypt_aead *caead, 1023 struct ucred *active_cred, struct thread *td) 1024 { 1025 struct cryptop_data *cod = NULL; 1026 struct cryptop *crp = NULL; 1027 char *dst; 1028 int error; 1029 1030 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 1031 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1032 return (E2BIG); 1033 } 1034 1035 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || 1036 (caead->len % cse->txform->blocksize) != 0) { 1037 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1038 return (EINVAL); 1039 } 1040 1041 /* 1042 * The COP_F_CIPHER_FIRST flag predates explicit session 1043 * modes, but the only way it was used was for EtA so allow it 1044 * as long as it is consistent with EtA. 1045 */ 1046 if (caead->flags & COP_F_CIPHER_FIRST) { 1047 if (caead->op != COP_ENCRYPT) { 1048 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1049 return (EINVAL); 1050 } 1051 } 1052 1053 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize, td); 1054 dst = caead->dst; 1055 1056 crp = crypto_getreq(cse->cses, M_WAITOK); 1057 1058 if (cod->aad != NULL) 1059 error = copyin(caead->aad, cod->aad, caead->aadlen); 1060 else 1061 error = copyin(caead->aad, cod->buf, caead->aadlen); 1062 if (error) { 1063 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1064 goto bail; 1065 } 1066 crp->crp_aad = cod->aad; 1067 crp->crp_aad_start = 0; 1068 crp->crp_aad_length = caead->aadlen; 1069 1070 if (cod->aad != NULL) 1071 crp->crp_payload_start = 0; 1072 else 1073 crp->crp_payload_start = caead->aadlen; 1074 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 1075 caead->len); 1076 if (error) { 1077 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1078 goto bail; 1079 } 1080 crp->crp_payload_length = caead->len; 1081 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 1082 crp->crp_digest_start = crp->crp_payload_output_start + 1083 caead->len; 1084 else 1085 crp->crp_digest_start = crp->crp_payload_start + caead->len; 1086 1087 switch (cse->mode) { 1088 case CSP_MODE_AEAD: 1089 case CSP_MODE_ETA: 1090 switch (caead->op) { 1091 case COP_ENCRYPT: 1092 crp->crp_op = CRYPTO_OP_ENCRYPT | 1093 CRYPTO_OP_COMPUTE_DIGEST; 1094 break; 1095 case COP_DECRYPT: 1096 crp->crp_op = CRYPTO_OP_DECRYPT | 1097 CRYPTO_OP_VERIFY_DIGEST; 1098 break; 1099 default: 1100 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1101 error = EINVAL; 1102 goto bail; 1103 } 1104 break; 1105 default: 1106 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1107 error = EINVAL; 1108 goto bail; 1109 } 1110 1111 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 1112 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 1113 cse->hashsize); 1114 if (cod->obuf != NULL) 1115 crypto_use_output_buf(crp, cod->obuf, caead->len + 1116 cse->hashsize); 1117 crp->crp_callback = cryptodev_cb; 1118 crp->crp_opaque = cod; 1119 1120 if (caead->iv) { 1121 /* 1122 * Permit a 16-byte IV for AES-XTS, but only use the 1123 * first 8 bytes as a block number. 1124 */ 1125 if (cse->mode == CSP_MODE_ETA && 1126 caead->ivlen == AES_BLOCK_LEN && 1127 cse->ivsize == AES_XTS_IV_LEN) 1128 caead->ivlen = AES_XTS_IV_LEN; 1129 1130 if (caead->ivlen != cse->ivsize) { 1131 error = EINVAL; 1132 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1133 goto bail; 1134 } 1135 1136 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 1137 if (error) { 1138 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1139 goto bail; 1140 } 1141 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1142 } else { 1143 crp->crp_iv_start = crp->crp_payload_start; 1144 crp->crp_payload_start += cse->ivsize; 1145 crp->crp_payload_length -= cse->ivsize; 1146 dst += cse->ivsize; 1147 } 1148 1149 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1150 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1151 cse->hashsize); 1152 if (error) { 1153 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1154 goto bail; 1155 } 1156 } 1157 again: 1158 /* 1159 * Let the dispatch run unlocked, then, interlock against the 1160 * callback before checking if the operation completed and going 1161 * to sleep. This insures drivers don't inherit our lock which 1162 * results in a lock order reversal between crypto_dispatch forced 1163 * entry and the crypto_done callback into us. 1164 */ 1165 error = crypto_dispatch(crp); 1166 if (error != 0) { 1167 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1168 goto bail; 1169 } 1170 1171 mtx_lock(&cse->lock); 1172 while (!cod->done) 1173 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1174 mtx_unlock(&cse->lock); 1175 1176 if (crp->crp_etype == EAGAIN) { 1177 crp->crp_etype = 0; 1178 crp->crp_flags &= ~CRYPTO_F_DONE; 1179 cod->done = false; 1180 goto again; 1181 } 1182 1183 if (crp->crp_etype != 0) { 1184 error = crp->crp_etype; 1185 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1186 goto bail; 1187 } 1188 1189 if (caead->dst != NULL) { 1190 error = copyout(cod->obuf != NULL ? cod->obuf : 1191 cod->buf + crp->crp_payload_start, dst, 1192 crp->crp_payload_length); 1193 if (error) { 1194 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1195 goto bail; 1196 } 1197 } 1198 1199 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1200 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1201 crp->crp_digest_start, caead->tag, cse->hashsize); 1202 if (error) { 1203 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1204 goto bail; 1205 } 1206 } 1207 1208 bail: 1209 crypto_freereq(crp); 1210 cod_free(cod); 1211 1212 return (error); 1213 } 1214 1215 static void 1216 cryptodevkey_cb(struct cryptkop *krp) 1217 { 1218 1219 wakeup_one(krp); 1220 } 1221 1222 static int 1223 cryptodev_key(struct crypt_kop *kop) 1224 { 1225 struct cryptkop *krp = NULL; 1226 int error = EINVAL; 1227 int in, out, size, i; 1228 1229 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) { 1230 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1231 return (EFBIG); 1232 } 1233 1234 in = kop->crk_iparams; 1235 out = kop->crk_oparams; 1236 switch (kop->crk_op) { 1237 case CRK_MOD_EXP: 1238 if (in == 3 && out == 1) 1239 break; 1240 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1241 return (EINVAL); 1242 case CRK_MOD_EXP_CRT: 1243 if (in == 6 && out == 1) 1244 break; 1245 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1246 return (EINVAL); 1247 case CRK_DSA_SIGN: 1248 if (in == 5 && out == 2) 1249 break; 1250 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1251 return (EINVAL); 1252 case CRK_DSA_VERIFY: 1253 if (in == 7 && out == 0) 1254 break; 1255 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1256 return (EINVAL); 1257 case CRK_DH_COMPUTE_KEY: 1258 if (in == 3 && out == 1) 1259 break; 1260 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1261 return (EINVAL); 1262 default: 1263 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1264 return (EINVAL); 1265 } 1266 1267 krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO); 1268 krp->krp_op = kop->crk_op; 1269 krp->krp_status = kop->crk_status; 1270 krp->krp_iparams = kop->crk_iparams; 1271 krp->krp_oparams = kop->crk_oparams; 1272 krp->krp_crid = kop->crk_crid; 1273 krp->krp_status = 0; 1274 krp->krp_callback = cryptodevkey_cb; 1275 1276 for (i = 0; i < CRK_MAXPARAM; i++) { 1277 if (kop->crk_param[i].crp_nbits > 65536) { 1278 /* Limit is the same as in OpenBSD */ 1279 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1280 goto fail; 1281 } 1282 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 1283 } 1284 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1285 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1286 if (size == 0) 1287 continue; 1288 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK); 1289 if (i >= krp->krp_iparams) 1290 continue; 1291 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size); 1292 if (error) { 1293 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1294 goto fail; 1295 } 1296 } 1297 1298 error = crypto_kdispatch(krp); 1299 if (error) { 1300 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1301 goto fail; 1302 } 1303 error = tsleep(krp, PSOCK, "crydev", 0); 1304 if (error) { 1305 /* XXX can this happen? if so, how do we recover? */ 1306 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1307 goto fail; 1308 } 1309 1310 kop->crk_crid = krp->krp_hid; /* device that did the work */ 1311 if (krp->krp_status != 0) { 1312 error = krp->krp_status; 1313 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1314 goto fail; 1315 } 1316 1317 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) { 1318 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1319 if (size == 0) 1320 continue; 1321 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size); 1322 if (error) { 1323 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1324 goto fail; 1325 } 1326 } 1327 1328 fail: 1329 if (krp) { 1330 kop->crk_status = krp->krp_status; 1331 for (i = 0; i < CRK_MAXPARAM; i++) { 1332 if (krp->krp_param[i].crp_p) 1333 free(krp->krp_param[i].crp_p, M_XDATA); 1334 } 1335 free(krp, M_XDATA); 1336 } 1337 return (error); 1338 } 1339 1340 static int 1341 cryptodev_find(struct crypt_find_op *find) 1342 { 1343 device_t dev; 1344 size_t fnlen = sizeof find->name; 1345 1346 if (find->crid != -1) { 1347 dev = crypto_find_device_byhid(find->crid); 1348 if (dev == NULL) 1349 return (ENOENT); 1350 strncpy(find->name, device_get_nameunit(dev), fnlen); 1351 find->name[fnlen - 1] = '\x0'; 1352 } else { 1353 find->name[fnlen - 1] = '\x0'; 1354 find->crid = crypto_find_driver(find->name); 1355 if (find->crid == -1) 1356 return (ENOENT); 1357 } 1358 return (0); 1359 } 1360 1361 static int 1362 cryptof_ioctl(struct file *fp, u_long cmd, void *data, 1363 struct ucred *active_cred, struct thread *td) 1364 { 1365 static struct timeval keywarn, featwarn; 1366 struct fcrypt *fcr = fp->f_data; 1367 struct csession *cse; 1368 struct session2_op *sop; 1369 struct crypt_op *cop; 1370 struct crypt_aead *caead; 1371 struct crypt_kop *kop; 1372 uint32_t ses; 1373 int error = 0; 1374 union { 1375 struct session2_op sopc; 1376 #ifdef COMPAT_FREEBSD32 1377 struct crypt_op copc; 1378 struct crypt_aead aeadc; 1379 struct crypt_kop kopc; 1380 #endif 1381 } thunk; 1382 #ifdef COMPAT_FREEBSD32 1383 u_long cmd32; 1384 void *data32; 1385 1386 cmd32 = 0; 1387 data32 = NULL; 1388 switch (cmd) { 1389 case CIOCGSESSION32: 1390 cmd32 = cmd; 1391 data32 = data; 1392 cmd = CIOCGSESSION; 1393 data = &thunk.sopc; 1394 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 1395 break; 1396 case CIOCGSESSION232: 1397 cmd32 = cmd; 1398 data32 = data; 1399 cmd = CIOCGSESSION2; 1400 data = &thunk.sopc; 1401 session2_op_from_32((struct session2_op32 *)data32, 1402 &thunk.sopc); 1403 break; 1404 case CIOCCRYPT32: 1405 cmd32 = cmd; 1406 data32 = data; 1407 cmd = CIOCCRYPT; 1408 data = &thunk.copc; 1409 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 1410 break; 1411 case CIOCCRYPTAEAD32: 1412 cmd32 = cmd; 1413 data32 = data; 1414 cmd = CIOCCRYPTAEAD; 1415 data = &thunk.aeadc; 1416 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 1417 break; 1418 case CIOCKEY32: 1419 case CIOCKEY232: 1420 cmd32 = cmd; 1421 data32 = data; 1422 if (cmd == CIOCKEY32) 1423 cmd = CIOCKEY; 1424 else 1425 cmd = CIOCKEY2; 1426 data = &thunk.kopc; 1427 crypt_kop_from_32((struct crypt_kop32 *)data32, &thunk.kopc); 1428 break; 1429 } 1430 #endif 1431 1432 switch (cmd) { 1433 case CIOCGSESSION: 1434 case CIOCGSESSION2: 1435 if (cmd == CIOCGSESSION) { 1436 session2_op_from_op(data, &thunk.sopc); 1437 sop = &thunk.sopc; 1438 } else 1439 sop = (struct session2_op *)data; 1440 1441 error = cse_create(fcr, sop); 1442 if (cmd == CIOCGSESSION && error == 0) 1443 session2_op_to_op(sop, data); 1444 break; 1445 case CIOCFSESSION: 1446 ses = *(uint32_t *)data; 1447 if (!cse_delete(fcr, ses)) { 1448 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1449 return (EINVAL); 1450 } 1451 break; 1452 case CIOCCRYPT: 1453 cop = (struct crypt_op *)data; 1454 cse = cse_find(fcr, cop->ses); 1455 if (cse == NULL) { 1456 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1457 return (EINVAL); 1458 } 1459 error = cryptodev_op(cse, cop, active_cred, td); 1460 cse_free(cse); 1461 break; 1462 case CIOCKEY: 1463 case CIOCKEY2: 1464 if (ratecheck(&keywarn, &warninterval)) 1465 gone_in(14, 1466 "Asymmetric crypto operations via /dev/crypto"); 1467 1468 if (!crypto_userasymcrypto) { 1469 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1470 return (EPERM); /* XXX compat? */ 1471 } 1472 kop = (struct crypt_kop *)data; 1473 if (cmd == CIOCKEY) { 1474 /* NB: crypto core enforces s/w driver use */ 1475 kop->crk_crid = 1476 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE; 1477 } 1478 mtx_lock(&Giant); 1479 error = cryptodev_key(kop); 1480 mtx_unlock(&Giant); 1481 break; 1482 case CIOCASYMFEAT: 1483 if (ratecheck(&featwarn, &warninterval)) 1484 gone_in(14, 1485 "Asymmetric crypto features via /dev/crypto"); 1486 1487 if (!crypto_userasymcrypto) { 1488 /* 1489 * NB: if user asym crypto operations are 1490 * not permitted return "no algorithms" 1491 * so well-behaved applications will just 1492 * fallback to doing them in software. 1493 */ 1494 *(int *)data = 0; 1495 } else { 1496 error = crypto_getfeat((int *)data); 1497 if (error) 1498 SDT_PROBE1(opencrypto, dev, ioctl, error, 1499 __LINE__); 1500 } 1501 break; 1502 case CIOCFINDDEV: 1503 error = cryptodev_find((struct crypt_find_op *)data); 1504 break; 1505 case CIOCCRYPTAEAD: 1506 caead = (struct crypt_aead *)data; 1507 cse = cse_find(fcr, caead->ses); 1508 if (cse == NULL) { 1509 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1510 return (EINVAL); 1511 } 1512 error = cryptodev_aead(cse, caead, active_cred, td); 1513 cse_free(cse); 1514 break; 1515 default: 1516 error = EINVAL; 1517 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1518 break; 1519 } 1520 1521 #ifdef COMPAT_FREEBSD32 1522 switch (cmd32) { 1523 case CIOCGSESSION32: 1524 if (error == 0) 1525 session_op_to_32(data, data32); 1526 break; 1527 case CIOCGSESSION232: 1528 if (error == 0) 1529 session2_op_to_32(data, data32); 1530 break; 1531 case CIOCCRYPT32: 1532 if (error == 0) 1533 crypt_op_to_32(data, data32); 1534 break; 1535 case CIOCCRYPTAEAD32: 1536 if (error == 0) 1537 crypt_aead_to_32(data, data32); 1538 break; 1539 case CIOCKEY32: 1540 case CIOCKEY232: 1541 crypt_kop_to_32(data, data32); 1542 break; 1543 } 1544 #endif 1545 return (error); 1546 } 1547 1548 /* ARGSUSED */ 1549 static int 1550 cryptof_stat(struct file *fp, struct stat *sb, struct ucred *active_cred, 1551 struct thread *td) 1552 { 1553 1554 return (EOPNOTSUPP); 1555 } 1556 1557 /* ARGSUSED */ 1558 static int 1559 cryptof_close(struct file *fp, struct thread *td) 1560 { 1561 struct fcrypt *fcr = fp->f_data; 1562 struct csession *cse; 1563 1564 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1565 TAILQ_REMOVE(&fcr->csessions, cse, next); 1566 KASSERT(cse->refs == 1, 1567 ("%s: crypto session %p with %d refs", __func__, cse, 1568 cse->refs)); 1569 cse_free(cse); 1570 } 1571 free(fcr, M_XDATA); 1572 fp->f_data = NULL; 1573 return 0; 1574 } 1575 1576 static int 1577 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif, 1578 struct filedesc *fdp) 1579 { 1580 1581 kif->kf_type = KF_TYPE_CRYPTO; 1582 return (0); 1583 } 1584 1585 static int 1586 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, 1587 struct thread *td) 1588 { 1589 struct file *f; 1590 struct fcrypt *fcr; 1591 int fd, error; 1592 1593 switch (cmd) { 1594 case CRIOGET: 1595 error = falloc_noinstall(td, &f); 1596 if (error) 1597 break; 1598 1599 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1600 TAILQ_INIT(&fcr->csessions); 1601 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1602 1603 finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops); 1604 error = finstall(td, f, &fd, 0, NULL); 1605 if (error) { 1606 mtx_destroy(&fcr->lock); 1607 free(fcr, M_XDATA); 1608 } else 1609 *(uint32_t *)data = fd; 1610 fdrop(f, td); 1611 break; 1612 case CRIOFINDDEV: 1613 error = cryptodev_find((struct crypt_find_op *)data); 1614 break; 1615 case CRIOASYMFEAT: 1616 error = crypto_getfeat((int *)data); 1617 break; 1618 default: 1619 error = EINVAL; 1620 break; 1621 } 1622 return (error); 1623 } 1624 1625 static struct cdevsw crypto_cdevsw = { 1626 .d_version = D_VERSION, 1627 .d_ioctl = cryptoioctl, 1628 .d_name = "crypto", 1629 }; 1630 static struct cdev *crypto_dev; 1631 1632 /* 1633 * Initialization code, both for static and dynamic loading. 1634 */ 1635 static int 1636 cryptodev_modevent(module_t mod, int type, void *unused) 1637 { 1638 switch (type) { 1639 case MOD_LOAD: 1640 if (bootverbose) 1641 printf("crypto: <crypto device>\n"); 1642 crypto_dev = make_dev(&crypto_cdevsw, 0, 1643 UID_ROOT, GID_WHEEL, 0666, 1644 "crypto"); 1645 return 0; 1646 case MOD_UNLOAD: 1647 /*XXX disallow if active sessions */ 1648 destroy_dev(crypto_dev); 1649 return 0; 1650 } 1651 return EINVAL; 1652 } 1653 1654 static moduledata_t cryptodev_mod = { 1655 "cryptodev", 1656 cryptodev_modevent, 1657 0 1658 }; 1659 MODULE_VERSION(cryptodev, 1); 1660 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1661 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1662 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1663