1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 30 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 34 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Effort sponsored in part by the Defense Advanced Research Projects 37 * Agency (DARPA) and Air Force Research Laboratory, Air Force 38 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/sysctl.h> 51 #include <sys/file.h> 52 #include <sys/filedesc.h> 53 #include <sys/errno.h> 54 #include <sys/random.h> 55 #include <sys/conf.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/fcntl.h> 59 #include <sys/bus.h> 60 #include <sys/user.h> 61 #include <sys/sdt.h> 62 63 #include <opencrypto/cryptodev.h> 64 #include <opencrypto/xform.h> 65 66 SDT_PROVIDER_DECLARE(opencrypto); 67 68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 69 70 #ifdef COMPAT_FREEBSD32 71 #include <sys/mount.h> 72 #include <compat/freebsd32/freebsd32.h> 73 74 struct session_op32 { 75 uint32_t cipher; 76 uint32_t mac; 77 uint32_t keylen; 78 uint32_t key; 79 int mackeylen; 80 uint32_t mackey; 81 uint32_t ses; 82 }; 83 84 struct session2_op32 { 85 uint32_t cipher; 86 uint32_t mac; 87 uint32_t keylen; 88 uint32_t key; 89 int mackeylen; 90 uint32_t mackey; 91 uint32_t ses; 92 int crid; 93 int pad[4]; 94 }; 95 96 struct crypt_op32 { 97 uint32_t ses; 98 uint16_t op; 99 uint16_t flags; 100 u_int len; 101 uint32_t src, dst; 102 uint32_t mac; 103 uint32_t iv; 104 }; 105 106 struct crypt_aead32 { 107 uint32_t ses; 108 uint16_t op; 109 uint16_t flags; 110 u_int len; 111 u_int aadlen; 112 u_int ivlen; 113 uint32_t src; 114 uint32_t dst; 115 uint32_t aad; 116 uint32_t tag; 117 uint32_t iv; 118 }; 119 120 struct crparam32 { 121 uint32_t crp_p; 122 u_int crp_nbits; 123 }; 124 125 struct crypt_kop32 { 126 u_int crk_op; 127 u_int crk_status; 128 u_short crk_iparams; 129 u_short crk_oparams; 130 u_int crk_crid; 131 struct crparam32 crk_param[CRK_MAXPARAM]; 132 }; 133 134 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 135 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 136 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32) 137 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 138 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32) 139 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 140 141 static void 142 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 143 { 144 145 memset(to, 0, sizeof(*to)); 146 CP(*from, *to, cipher); 147 CP(*from, *to, mac); 148 CP(*from, *to, keylen); 149 PTRIN_CP(*from, *to, key); 150 CP(*from, *to, mackeylen); 151 PTRIN_CP(*from, *to, mackey); 152 CP(*from, *to, ses); 153 to->crid = CRYPTOCAP_F_HARDWARE; 154 } 155 156 static void 157 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 158 { 159 160 session_op_from_32((const struct session_op32 *)from, to); 161 CP(*from, *to, crid); 162 } 163 164 static void 165 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 166 { 167 168 CP(*from, *to, cipher); 169 CP(*from, *to, mac); 170 CP(*from, *to, keylen); 171 PTROUT_CP(*from, *to, key); 172 CP(*from, *to, mackeylen); 173 PTROUT_CP(*from, *to, mackey); 174 CP(*from, *to, ses); 175 } 176 177 static void 178 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 179 { 180 181 session_op_to_32(from, (struct session_op32 *)to); 182 CP(*from, *to, crid); 183 } 184 185 static void 186 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 187 { 188 189 CP(*from, *to, ses); 190 CP(*from, *to, op); 191 CP(*from, *to, flags); 192 CP(*from, *to, len); 193 PTRIN_CP(*from, *to, src); 194 PTRIN_CP(*from, *to, dst); 195 PTRIN_CP(*from, *to, mac); 196 PTRIN_CP(*from, *to, iv); 197 } 198 199 static void 200 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 201 { 202 203 CP(*from, *to, ses); 204 CP(*from, *to, op); 205 CP(*from, *to, flags); 206 CP(*from, *to, len); 207 PTROUT_CP(*from, *to, src); 208 PTROUT_CP(*from, *to, dst); 209 PTROUT_CP(*from, *to, mac); 210 PTROUT_CP(*from, *to, iv); 211 } 212 213 static void 214 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 215 { 216 217 CP(*from, *to, ses); 218 CP(*from, *to, op); 219 CP(*from, *to, flags); 220 CP(*from, *to, len); 221 CP(*from, *to, aadlen); 222 CP(*from, *to, ivlen); 223 PTRIN_CP(*from, *to, src); 224 PTRIN_CP(*from, *to, dst); 225 PTRIN_CP(*from, *to, aad); 226 PTRIN_CP(*from, *to, tag); 227 PTRIN_CP(*from, *to, iv); 228 } 229 230 static void 231 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 232 { 233 234 CP(*from, *to, ses); 235 CP(*from, *to, op); 236 CP(*from, *to, flags); 237 CP(*from, *to, len); 238 CP(*from, *to, aadlen); 239 CP(*from, *to, ivlen); 240 PTROUT_CP(*from, *to, src); 241 PTROUT_CP(*from, *to, dst); 242 PTROUT_CP(*from, *to, aad); 243 PTROUT_CP(*from, *to, tag); 244 PTROUT_CP(*from, *to, iv); 245 } 246 247 static void 248 crparam_from_32(const struct crparam32 *from, struct crparam *to) 249 { 250 251 PTRIN_CP(*from, *to, crp_p); 252 CP(*from, *to, crp_nbits); 253 } 254 255 static void 256 crparam_to_32(const struct crparam *from, struct crparam32 *to) 257 { 258 259 PTROUT_CP(*from, *to, crp_p); 260 CP(*from, *to, crp_nbits); 261 } 262 263 static void 264 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to) 265 { 266 int i; 267 268 CP(*from, *to, crk_op); 269 CP(*from, *to, crk_status); 270 CP(*from, *to, crk_iparams); 271 CP(*from, *to, crk_oparams); 272 CP(*from, *to, crk_crid); 273 for (i = 0; i < CRK_MAXPARAM; i++) 274 crparam_from_32(&from->crk_param[i], &to->crk_param[i]); 275 } 276 277 static void 278 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to) 279 { 280 int i; 281 282 CP(*from, *to, crk_op); 283 CP(*from, *to, crk_status); 284 CP(*from, *to, crk_iparams); 285 CP(*from, *to, crk_oparams); 286 CP(*from, *to, crk_crid); 287 for (i = 0; i < CRK_MAXPARAM; i++) 288 crparam_to_32(&from->crk_param[i], &to->crk_param[i]); 289 } 290 #endif 291 292 static void 293 session2_op_from_op(const struct session_op *from, struct session2_op *to) 294 { 295 296 memset(to, 0, sizeof(*to)); 297 memcpy(to, from, sizeof(*from)); 298 to->crid = CRYPTOCAP_F_HARDWARE; 299 } 300 301 static void 302 session2_op_to_op(const struct session2_op *from, struct session_op *to) 303 { 304 305 memcpy(to, from, sizeof(*to)); 306 } 307 308 struct csession { 309 TAILQ_ENTRY(csession) next; 310 crypto_session_t cses; 311 volatile u_int refs; 312 uint32_t ses; 313 struct mtx lock; /* for op submission */ 314 315 struct enc_xform *txform; 316 int hashsize; 317 int ivsize; 318 int mode; 319 320 void *key; 321 void *mackey; 322 }; 323 324 struct cryptop_data { 325 struct csession *cse; 326 327 char *buf; 328 char *obuf; 329 char *aad; 330 bool done; 331 }; 332 333 struct fcrypt { 334 TAILQ_HEAD(csessionlist, csession) csessions; 335 int sesn; 336 struct mtx lock; 337 }; 338 339 static bool use_outputbuffers; 340 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 341 &use_outputbuffers, 0, 342 "Use separate output buffers for /dev/crypto requests."); 343 344 static bool use_separate_aad; 345 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 346 &use_separate_aad, 0, 347 "Use separate AAD buffer for /dev/crypto requests."); 348 349 static struct timeval warninterval = { .tv_sec = 60, .tv_usec = 0 }; 350 SYSCTL_TIMEVAL_SEC(_kern, OID_AUTO, cryptodev_warn_interval, CTLFLAG_RW, 351 &warninterval, 352 "Delay in seconds between warnings of deprecated /dev/crypto algorithms"); 353 354 static int cryptof_ioctl(struct file *, u_long, void *, 355 struct ucred *, struct thread *); 356 static int cryptof_stat(struct file *, struct stat *, 357 struct ucred *, struct thread *); 358 static int cryptof_close(struct file *, struct thread *); 359 static int cryptof_fill_kinfo(struct file *, struct kinfo_file *, 360 struct filedesc *); 361 362 static struct fileops cryptofops = { 363 .fo_read = invfo_rdwr, 364 .fo_write = invfo_rdwr, 365 .fo_truncate = invfo_truncate, 366 .fo_ioctl = cryptof_ioctl, 367 .fo_poll = invfo_poll, 368 .fo_kqfilter = invfo_kqfilter, 369 .fo_stat = cryptof_stat, 370 .fo_close = cryptof_close, 371 .fo_chmod = invfo_chmod, 372 .fo_chown = invfo_chown, 373 .fo_sendfile = invfo_sendfile, 374 .fo_fill_kinfo = cryptof_fill_kinfo, 375 }; 376 377 static struct csession *csefind(struct fcrypt *, u_int); 378 static bool csedelete(struct fcrypt *, u_int); 379 static struct csession *csecreate(struct fcrypt *, crypto_session_t, 380 struct crypto_session_params *, struct enc_xform *, void *, 381 struct auth_hash *, void *); 382 static void csefree(struct csession *); 383 384 static int cryptodev_op(struct csession *, struct crypt_op *, 385 struct ucred *, struct thread *td); 386 static int cryptodev_aead(struct csession *, struct crypt_aead *, 387 struct ucred *, struct thread *); 388 static int cryptodev_key(struct crypt_kop *); 389 static int cryptodev_find(struct crypt_find_op *); 390 391 /* 392 * Check a crypto identifier to see if it requested 393 * a software device/driver. This can be done either 394 * by device name/class or through search constraints. 395 */ 396 static int 397 checkforsoftware(int *cridp) 398 { 399 int crid; 400 401 crid = *cridp; 402 403 if (!crypto_devallowsoft) { 404 if (crid & CRYPTOCAP_F_SOFTWARE) { 405 if (crid & CRYPTOCAP_F_HARDWARE) { 406 *cridp = CRYPTOCAP_F_HARDWARE; 407 return 0; 408 } 409 return EINVAL; 410 } 411 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 412 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 413 return EINVAL; 414 } 415 return 0; 416 } 417 418 /* ARGSUSED */ 419 static int 420 cryptof_ioctl( 421 struct file *fp, 422 u_long cmd, 423 void *data, 424 struct ucred *active_cred, 425 struct thread *td) 426 { 427 static struct timeval keywarn, featwarn; 428 struct crypto_session_params csp; 429 struct fcrypt *fcr = fp->f_data; 430 struct csession *cse; 431 struct session2_op *sop; 432 struct crypt_op *cop; 433 struct crypt_aead *caead; 434 struct enc_xform *txform = NULL; 435 struct auth_hash *thash = NULL; 436 void *key = NULL; 437 void *mackey = NULL; 438 struct crypt_kop *kop; 439 crypto_session_t cses; 440 uint32_t ses; 441 int error = 0, crid; 442 union { 443 struct session2_op sopc; 444 #ifdef COMPAT_FREEBSD32 445 struct crypt_op copc; 446 struct crypt_aead aeadc; 447 struct crypt_kop kopc; 448 #endif 449 } thunk; 450 #ifdef COMPAT_FREEBSD32 451 u_long cmd32; 452 void *data32; 453 454 cmd32 = 0; 455 data32 = NULL; 456 switch (cmd) { 457 case CIOCGSESSION32: 458 cmd32 = cmd; 459 data32 = data; 460 cmd = CIOCGSESSION; 461 data = &thunk.sopc; 462 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 463 break; 464 case CIOCGSESSION232: 465 cmd32 = cmd; 466 data32 = data; 467 cmd = CIOCGSESSION2; 468 data = &thunk.sopc; 469 session2_op_from_32((struct session2_op32 *)data32, 470 &thunk.sopc); 471 break; 472 case CIOCCRYPT32: 473 cmd32 = cmd; 474 data32 = data; 475 cmd = CIOCCRYPT; 476 data = &thunk.copc; 477 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 478 break; 479 case CIOCCRYPTAEAD32: 480 cmd32 = cmd; 481 data32 = data; 482 cmd = CIOCCRYPTAEAD; 483 data = &thunk.aeadc; 484 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 485 break; 486 case CIOCKEY32: 487 case CIOCKEY232: 488 cmd32 = cmd; 489 data32 = data; 490 if (cmd == CIOCKEY32) 491 cmd = CIOCKEY; 492 else 493 cmd = CIOCKEY2; 494 data = &thunk.kopc; 495 crypt_kop_from_32((struct crypt_kop32 *)data32, &thunk.kopc); 496 break; 497 } 498 #endif 499 500 switch (cmd) { 501 case CIOCGSESSION: 502 case CIOCGSESSION2: 503 if (cmd == CIOCGSESSION) { 504 session2_op_from_op(data, &thunk.sopc); 505 sop = &thunk.sopc; 506 } else 507 sop = (struct session2_op *)data; 508 509 switch (sop->cipher) { 510 case 0: 511 break; 512 case CRYPTO_AES_CBC: 513 txform = &enc_xform_rijndael128; 514 break; 515 case CRYPTO_AES_XTS: 516 txform = &enc_xform_aes_xts; 517 break; 518 case CRYPTO_NULL_CBC: 519 txform = &enc_xform_null; 520 break; 521 case CRYPTO_CAMELLIA_CBC: 522 txform = &enc_xform_camellia; 523 break; 524 case CRYPTO_AES_ICM: 525 txform = &enc_xform_aes_icm; 526 break; 527 case CRYPTO_AES_NIST_GCM_16: 528 txform = &enc_xform_aes_nist_gcm; 529 break; 530 case CRYPTO_CHACHA20: 531 txform = &enc_xform_chacha20; 532 break; 533 case CRYPTO_AES_CCM_16: 534 txform = &enc_xform_ccm; 535 break; 536 537 default: 538 CRYPTDEB("invalid cipher"); 539 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 540 return (EINVAL); 541 } 542 543 switch (sop->mac) { 544 case 0: 545 break; 546 case CRYPTO_POLY1305: 547 thash = &auth_hash_poly1305; 548 break; 549 case CRYPTO_SHA1_HMAC: 550 thash = &auth_hash_hmac_sha1; 551 break; 552 case CRYPTO_SHA2_224_HMAC: 553 thash = &auth_hash_hmac_sha2_224; 554 break; 555 case CRYPTO_SHA2_256_HMAC: 556 thash = &auth_hash_hmac_sha2_256; 557 break; 558 case CRYPTO_SHA2_384_HMAC: 559 thash = &auth_hash_hmac_sha2_384; 560 break; 561 case CRYPTO_SHA2_512_HMAC: 562 thash = &auth_hash_hmac_sha2_512; 563 break; 564 case CRYPTO_RIPEMD160_HMAC: 565 thash = &auth_hash_hmac_ripemd_160; 566 break; 567 #ifdef COMPAT_FREEBSD12 568 case CRYPTO_AES_128_NIST_GMAC: 569 case CRYPTO_AES_192_NIST_GMAC: 570 case CRYPTO_AES_256_NIST_GMAC: 571 /* Should always be paired with GCM. */ 572 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 573 CRYPTDEB("GMAC without GCM"); 574 SDT_PROBE1(opencrypto, dev, ioctl, error, 575 __LINE__); 576 return (EINVAL); 577 } 578 break; 579 #endif 580 case CRYPTO_AES_NIST_GMAC: 581 switch (sop->mackeylen * 8) { 582 case 128: 583 thash = &auth_hash_nist_gmac_aes_128; 584 break; 585 case 192: 586 thash = &auth_hash_nist_gmac_aes_192; 587 break; 588 case 256: 589 thash = &auth_hash_nist_gmac_aes_256; 590 break; 591 default: 592 CRYPTDEB("invalid GMAC key length"); 593 SDT_PROBE1(opencrypto, dev, ioctl, error, 594 __LINE__); 595 return (EINVAL); 596 } 597 break; 598 case CRYPTO_AES_CCM_CBC_MAC: 599 switch (sop->mackeylen) { 600 case 16: 601 thash = &auth_hash_ccm_cbc_mac_128; 602 break; 603 case 24: 604 thash = &auth_hash_ccm_cbc_mac_192; 605 break; 606 case 32: 607 thash = &auth_hash_ccm_cbc_mac_256; 608 break; 609 default: 610 CRYPTDEB("Invalid CBC MAC key size %d", 611 sop->keylen); 612 SDT_PROBE1(opencrypto, dev, ioctl, 613 error, __LINE__); 614 return (EINVAL); 615 } 616 break; 617 case CRYPTO_SHA1: 618 thash = &auth_hash_sha1; 619 break; 620 case CRYPTO_SHA2_224: 621 thash = &auth_hash_sha2_224; 622 break; 623 case CRYPTO_SHA2_256: 624 thash = &auth_hash_sha2_256; 625 break; 626 case CRYPTO_SHA2_384: 627 thash = &auth_hash_sha2_384; 628 break; 629 case CRYPTO_SHA2_512: 630 thash = &auth_hash_sha2_512; 631 break; 632 633 case CRYPTO_NULL_HMAC: 634 thash = &auth_hash_null; 635 break; 636 637 case CRYPTO_BLAKE2B: 638 thash = &auth_hash_blake2b; 639 break; 640 case CRYPTO_BLAKE2S: 641 thash = &auth_hash_blake2s; 642 break; 643 644 default: 645 CRYPTDEB("invalid mac"); 646 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 647 return (EINVAL); 648 } 649 650 if (txform == NULL && thash == NULL) { 651 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 652 return (EINVAL); 653 } 654 655 memset(&csp, 0, sizeof(csp)); 656 if (use_outputbuffers) 657 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 658 659 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { 660 switch (sop->mac) { 661 #ifdef COMPAT_FREEBSD12 662 case CRYPTO_AES_128_NIST_GMAC: 663 case CRYPTO_AES_192_NIST_GMAC: 664 case CRYPTO_AES_256_NIST_GMAC: 665 if (sop->keylen != sop->mackeylen) { 666 SDT_PROBE1(opencrypto, dev, ioctl, 667 error, __LINE__); 668 return (EINVAL); 669 } 670 break; 671 #endif 672 case 0: 673 break; 674 default: 675 SDT_PROBE1(opencrypto, dev, ioctl, error, 676 __LINE__); 677 return (EINVAL); 678 } 679 csp.csp_mode = CSP_MODE_AEAD; 680 } else if (sop->cipher == CRYPTO_AES_CCM_16) { 681 switch (sop->mac) { 682 #ifdef COMPAT_FREEBSD12 683 case CRYPTO_AES_CCM_CBC_MAC: 684 if (sop->keylen != sop->mackeylen) { 685 SDT_PROBE1(opencrypto, dev, ioctl, 686 error, __LINE__); 687 return (EINVAL); 688 } 689 thash = NULL; 690 break; 691 #endif 692 case 0: 693 break; 694 default: 695 SDT_PROBE1(opencrypto, dev, ioctl, error, 696 __LINE__); 697 return (EINVAL); 698 } 699 csp.csp_mode = CSP_MODE_AEAD; 700 } else if (txform && thash) 701 csp.csp_mode = CSP_MODE_ETA; 702 else if (txform) 703 csp.csp_mode = CSP_MODE_CIPHER; 704 else 705 csp.csp_mode = CSP_MODE_DIGEST; 706 707 switch (csp.csp_mode) { 708 case CSP_MODE_AEAD: 709 case CSP_MODE_ETA: 710 if (use_separate_aad) 711 csp.csp_flags |= CSP_F_SEPARATE_AAD; 712 break; 713 } 714 715 if (txform) { 716 csp.csp_cipher_alg = txform->type; 717 csp.csp_cipher_klen = sop->keylen; 718 if (sop->keylen > txform->maxkey || 719 sop->keylen < txform->minkey) { 720 CRYPTDEB("invalid cipher parameters"); 721 error = EINVAL; 722 SDT_PROBE1(opencrypto, dev, ioctl, error, 723 __LINE__); 724 goto bail; 725 } 726 727 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 728 error = copyin(sop->key, key, csp.csp_cipher_klen); 729 if (error) { 730 CRYPTDEB("invalid key"); 731 SDT_PROBE1(opencrypto, dev, ioctl, error, 732 __LINE__); 733 goto bail; 734 } 735 csp.csp_cipher_key = key; 736 csp.csp_ivlen = txform->ivsize; 737 } 738 739 if (thash) { 740 csp.csp_auth_alg = thash->type; 741 csp.csp_auth_klen = sop->mackeylen; 742 if (sop->mackeylen > thash->keysize || 743 sop->mackeylen < 0) { 744 CRYPTDEB("invalid mac key length"); 745 error = EINVAL; 746 SDT_PROBE1(opencrypto, dev, ioctl, error, 747 __LINE__); 748 goto bail; 749 } 750 751 if (csp.csp_auth_klen) { 752 mackey = malloc(csp.csp_auth_klen, M_XDATA, 753 M_WAITOK); 754 error = copyin(sop->mackey, mackey, 755 csp.csp_auth_klen); 756 if (error) { 757 CRYPTDEB("invalid mac key"); 758 SDT_PROBE1(opencrypto, dev, ioctl, 759 error, __LINE__); 760 goto bail; 761 } 762 csp.csp_auth_key = mackey; 763 } 764 765 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 766 csp.csp_ivlen = AES_GCM_IV_LEN; 767 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 768 csp.csp_ivlen = AES_CCM_IV_LEN; 769 } 770 771 crid = sop->crid; 772 error = checkforsoftware(&crid); 773 if (error) { 774 CRYPTDEB("checkforsoftware"); 775 SDT_PROBE1(opencrypto, dev, ioctl, error, 776 __LINE__); 777 goto bail; 778 } 779 error = crypto_newsession(&cses, &csp, crid); 780 if (error) { 781 CRYPTDEB("crypto_newsession"); 782 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 783 goto bail; 784 } 785 786 cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey); 787 788 if (cse == NULL) { 789 crypto_freesession(cses); 790 error = EINVAL; 791 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 792 CRYPTDEB("csecreate"); 793 goto bail; 794 } 795 sop->ses = cse->ses; 796 797 /* return hardware/driver id */ 798 sop->crid = crypto_ses2hid(cse->cses); 799 bail: 800 if (error) { 801 free(key, M_XDATA); 802 free(mackey, M_XDATA); 803 } 804 805 if (cmd == CIOCGSESSION && error == 0) 806 session2_op_to_op(sop, data); 807 break; 808 case CIOCFSESSION: 809 ses = *(uint32_t *)data; 810 if (!csedelete(fcr, ses)) { 811 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 812 return (EINVAL); 813 } 814 break; 815 case CIOCCRYPT: 816 cop = (struct crypt_op *)data; 817 cse = csefind(fcr, cop->ses); 818 if (cse == NULL) { 819 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 820 return (EINVAL); 821 } 822 error = cryptodev_op(cse, cop, active_cred, td); 823 csefree(cse); 824 break; 825 case CIOCKEY: 826 case CIOCKEY2: 827 if (ratecheck(&keywarn, &warninterval)) 828 gone_in(14, 829 "Asymmetric crypto operations via /dev/crypto"); 830 831 if (!crypto_userasymcrypto) { 832 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 833 return (EPERM); /* XXX compat? */ 834 } 835 kop = (struct crypt_kop *)data; 836 if (cmd == CIOCKEY) { 837 /* NB: crypto core enforces s/w driver use */ 838 kop->crk_crid = 839 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE; 840 } 841 mtx_lock(&Giant); 842 error = cryptodev_key(kop); 843 mtx_unlock(&Giant); 844 break; 845 case CIOCASYMFEAT: 846 if (ratecheck(&featwarn, &warninterval)) 847 gone_in(14, 848 "Asymmetric crypto features via /dev/crypto"); 849 850 if (!crypto_userasymcrypto) { 851 /* 852 * NB: if user asym crypto operations are 853 * not permitted return "no algorithms" 854 * so well-behaved applications will just 855 * fallback to doing them in software. 856 */ 857 *(int *)data = 0; 858 } else { 859 error = crypto_getfeat((int *)data); 860 if (error) 861 SDT_PROBE1(opencrypto, dev, ioctl, error, 862 __LINE__); 863 } 864 break; 865 case CIOCFINDDEV: 866 error = cryptodev_find((struct crypt_find_op *)data); 867 break; 868 case CIOCCRYPTAEAD: 869 caead = (struct crypt_aead *)data; 870 cse = csefind(fcr, caead->ses); 871 if (cse == NULL) { 872 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 873 return (EINVAL); 874 } 875 error = cryptodev_aead(cse, caead, active_cred, td); 876 csefree(cse); 877 break; 878 default: 879 error = EINVAL; 880 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 881 break; 882 } 883 884 #ifdef COMPAT_FREEBSD32 885 switch (cmd32) { 886 case CIOCGSESSION32: 887 if (error == 0) 888 session_op_to_32(data, data32); 889 break; 890 case CIOCGSESSION232: 891 if (error == 0) 892 session2_op_to_32(data, data32); 893 break; 894 case CIOCCRYPT32: 895 if (error == 0) 896 crypt_op_to_32(data, data32); 897 break; 898 case CIOCCRYPTAEAD32: 899 if (error == 0) 900 crypt_aead_to_32(data, data32); 901 break; 902 case CIOCKEY32: 903 case CIOCKEY232: 904 crypt_kop_to_32(data, data32); 905 break; 906 } 907 #endif 908 return (error); 909 } 910 911 static int cryptodev_cb(struct cryptop *); 912 913 static struct cryptop_data * 914 cod_alloc(struct csession *cse, size_t aad_len, size_t len, struct thread *td) 915 { 916 struct cryptop_data *cod; 917 918 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 919 920 cod->cse = cse; 921 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 922 if (aad_len != 0) 923 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 924 cod->buf = malloc(len, M_XDATA, M_WAITOK); 925 } else 926 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 927 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 928 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 929 return (cod); 930 } 931 932 static void 933 cod_free(struct cryptop_data *cod) 934 { 935 936 free(cod->aad, M_XDATA); 937 free(cod->obuf, M_XDATA); 938 free(cod->buf, M_XDATA); 939 free(cod, M_XDATA); 940 } 941 942 static int 943 cryptodev_op( 944 struct csession *cse, 945 struct crypt_op *cop, 946 struct ucred *active_cred, 947 struct thread *td) 948 { 949 struct cryptop_data *cod = NULL; 950 struct cryptop *crp = NULL; 951 int error; 952 953 if (cop->len > 256*1024-4) { 954 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 955 return (E2BIG); 956 } 957 958 if (cse->txform) { 959 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { 960 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 961 return (EINVAL); 962 } 963 } 964 965 if (cop->mac && cse->hashsize == 0) { 966 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 967 return (EINVAL); 968 } 969 970 /* 971 * The COP_F_CIPHER_FIRST flag predates explicit session 972 * modes, but the only way it was used was for EtA so allow it 973 * as long as it is consistent with EtA. 974 */ 975 if (cop->flags & COP_F_CIPHER_FIRST) { 976 if (cop->op != COP_ENCRYPT) { 977 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 978 return (EINVAL); 979 } 980 } 981 982 cod = cod_alloc(cse, 0, cop->len + cse->hashsize, td); 983 984 crp = crypto_getreq(cse->cses, M_WAITOK); 985 986 error = copyin(cop->src, cod->buf, cop->len); 987 if (error) { 988 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 989 goto bail; 990 } 991 crp->crp_payload_start = 0; 992 crp->crp_payload_length = cop->len; 993 if (cse->hashsize) 994 crp->crp_digest_start = cop->len; 995 996 switch (cse->mode) { 997 case CSP_MODE_COMPRESS: 998 switch (cop->op) { 999 case COP_ENCRYPT: 1000 crp->crp_op = CRYPTO_OP_COMPRESS; 1001 break; 1002 case COP_DECRYPT: 1003 crp->crp_op = CRYPTO_OP_DECOMPRESS; 1004 break; 1005 default: 1006 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1007 error = EINVAL; 1008 goto bail; 1009 } 1010 break; 1011 case CSP_MODE_CIPHER: 1012 switch (cop->op) { 1013 case COP_ENCRYPT: 1014 crp->crp_op = CRYPTO_OP_ENCRYPT; 1015 break; 1016 case COP_DECRYPT: 1017 crp->crp_op = CRYPTO_OP_DECRYPT; 1018 break; 1019 default: 1020 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1021 error = EINVAL; 1022 goto bail; 1023 } 1024 break; 1025 case CSP_MODE_DIGEST: 1026 switch (cop->op) { 1027 case 0: 1028 case COP_ENCRYPT: 1029 case COP_DECRYPT: 1030 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 1031 if (cod->obuf != NULL) 1032 crp->crp_digest_start = 0; 1033 break; 1034 default: 1035 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1036 error = EINVAL; 1037 goto bail; 1038 } 1039 break; 1040 case CSP_MODE_ETA: 1041 switch (cop->op) { 1042 case COP_ENCRYPT: 1043 crp->crp_op = CRYPTO_OP_ENCRYPT | 1044 CRYPTO_OP_COMPUTE_DIGEST; 1045 break; 1046 case COP_DECRYPT: 1047 crp->crp_op = CRYPTO_OP_DECRYPT | 1048 CRYPTO_OP_VERIFY_DIGEST; 1049 break; 1050 default: 1051 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1052 error = EINVAL; 1053 goto bail; 1054 } 1055 break; 1056 default: 1057 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1058 error = EINVAL; 1059 goto bail; 1060 } 1061 1062 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 1063 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 1064 if (cod->obuf) 1065 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 1066 crp->crp_callback = cryptodev_cb; 1067 crp->crp_opaque = cod; 1068 1069 if (cop->iv) { 1070 if (cse->ivsize == 0) { 1071 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1072 error = EINVAL; 1073 goto bail; 1074 } 1075 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 1076 if (error) { 1077 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1078 goto bail; 1079 } 1080 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1081 } else if (cse->ivsize != 0) { 1082 crp->crp_iv_start = 0; 1083 crp->crp_payload_start += cse->ivsize; 1084 crp->crp_payload_length -= cse->ivsize; 1085 cop->dst += cse->ivsize; 1086 } 1087 1088 if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1089 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 1090 cse->hashsize); 1091 if (error) { 1092 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1093 goto bail; 1094 } 1095 } 1096 again: 1097 /* 1098 * Let the dispatch run unlocked, then, interlock against the 1099 * callback before checking if the operation completed and going 1100 * to sleep. This insures drivers don't inherit our lock which 1101 * results in a lock order reversal between crypto_dispatch forced 1102 * entry and the crypto_done callback into us. 1103 */ 1104 error = crypto_dispatch(crp); 1105 if (error != 0) { 1106 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1107 goto bail; 1108 } 1109 1110 mtx_lock(&cse->lock); 1111 while (!cod->done) 1112 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1113 mtx_unlock(&cse->lock); 1114 1115 if (crp->crp_etype == EAGAIN) { 1116 crp->crp_etype = 0; 1117 crp->crp_flags &= ~CRYPTO_F_DONE; 1118 cod->done = false; 1119 goto again; 1120 } 1121 1122 if (crp->crp_etype != 0) { 1123 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1124 error = crp->crp_etype; 1125 goto bail; 1126 } 1127 1128 if (cop->dst != NULL) { 1129 error = copyout(cod->obuf != NULL ? cod->obuf : 1130 cod->buf + crp->crp_payload_start, cop->dst, 1131 crp->crp_payload_length); 1132 if (error) { 1133 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1134 goto bail; 1135 } 1136 } 1137 1138 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1139 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1140 crp->crp_digest_start, cop->mac, cse->hashsize); 1141 if (error) { 1142 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1143 goto bail; 1144 } 1145 } 1146 1147 bail: 1148 crypto_freereq(crp); 1149 cod_free(cod); 1150 1151 return (error); 1152 } 1153 1154 static int 1155 cryptodev_aead( 1156 struct csession *cse, 1157 struct crypt_aead *caead, 1158 struct ucred *active_cred, 1159 struct thread *td) 1160 { 1161 struct cryptop_data *cod = NULL; 1162 struct cryptop *crp = NULL; 1163 int error; 1164 1165 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 1166 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1167 return (E2BIG); 1168 } 1169 1170 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || 1171 (caead->len % cse->txform->blocksize) != 0) { 1172 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1173 return (EINVAL); 1174 } 1175 1176 /* 1177 * The COP_F_CIPHER_FIRST flag predates explicit session 1178 * modes, but the only way it was used was for EtA so allow it 1179 * as long as it is consistent with EtA. 1180 */ 1181 if (caead->flags & COP_F_CIPHER_FIRST) { 1182 if (caead->op != COP_ENCRYPT) { 1183 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1184 return (EINVAL); 1185 } 1186 } 1187 1188 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize, td); 1189 1190 crp = crypto_getreq(cse->cses, M_WAITOK); 1191 1192 if (cod->aad != NULL) 1193 error = copyin(caead->aad, cod->aad, caead->aadlen); 1194 else 1195 error = copyin(caead->aad, cod->buf, caead->aadlen); 1196 if (error) { 1197 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1198 goto bail; 1199 } 1200 crp->crp_aad = cod->aad; 1201 crp->crp_aad_start = 0; 1202 crp->crp_aad_length = caead->aadlen; 1203 1204 if (cod->aad != NULL) 1205 crp->crp_payload_start = 0; 1206 else 1207 crp->crp_payload_start = caead->aadlen; 1208 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 1209 caead->len); 1210 if (error) { 1211 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1212 goto bail; 1213 } 1214 crp->crp_payload_length = caead->len; 1215 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 1216 crp->crp_digest_start = crp->crp_payload_output_start + 1217 caead->len; 1218 else 1219 crp->crp_digest_start = crp->crp_payload_start + caead->len; 1220 1221 switch (cse->mode) { 1222 case CSP_MODE_AEAD: 1223 case CSP_MODE_ETA: 1224 switch (caead->op) { 1225 case COP_ENCRYPT: 1226 crp->crp_op = CRYPTO_OP_ENCRYPT | 1227 CRYPTO_OP_COMPUTE_DIGEST; 1228 break; 1229 case COP_DECRYPT: 1230 crp->crp_op = CRYPTO_OP_DECRYPT | 1231 CRYPTO_OP_VERIFY_DIGEST; 1232 break; 1233 default: 1234 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1235 error = EINVAL; 1236 goto bail; 1237 } 1238 break; 1239 default: 1240 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1241 error = EINVAL; 1242 goto bail; 1243 } 1244 1245 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 1246 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 1247 cse->hashsize); 1248 if (cod->obuf != NULL) 1249 crypto_use_output_buf(crp, cod->obuf, caead->len + 1250 cse->hashsize); 1251 crp->crp_callback = cryptodev_cb; 1252 crp->crp_opaque = cod; 1253 1254 if (caead->iv) { 1255 /* 1256 * Permit a 16-byte IV for AES-XTS, but only use the 1257 * first 8 bytes as a block number. 1258 */ 1259 if (cse->mode == CSP_MODE_ETA && 1260 caead->ivlen == AES_BLOCK_LEN && 1261 cse->ivsize == AES_XTS_IV_LEN) 1262 caead->ivlen = AES_XTS_IV_LEN; 1263 1264 if (caead->ivlen != cse->ivsize) { 1265 error = EINVAL; 1266 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1267 goto bail; 1268 } 1269 1270 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 1271 if (error) { 1272 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1273 goto bail; 1274 } 1275 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1276 } else { 1277 crp->crp_iv_start = crp->crp_payload_start; 1278 crp->crp_payload_start += cse->ivsize; 1279 crp->crp_payload_length -= cse->ivsize; 1280 caead->dst += cse->ivsize; 1281 } 1282 1283 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1284 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1285 cse->hashsize); 1286 if (error) { 1287 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1288 goto bail; 1289 } 1290 } 1291 again: 1292 /* 1293 * Let the dispatch run unlocked, then, interlock against the 1294 * callback before checking if the operation completed and going 1295 * to sleep. This insures drivers don't inherit our lock which 1296 * results in a lock order reversal between crypto_dispatch forced 1297 * entry and the crypto_done callback into us. 1298 */ 1299 error = crypto_dispatch(crp); 1300 if (error != 0) { 1301 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1302 goto bail; 1303 } 1304 1305 mtx_lock(&cse->lock); 1306 while (!cod->done) 1307 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1308 mtx_unlock(&cse->lock); 1309 1310 if (crp->crp_etype == EAGAIN) { 1311 crp->crp_etype = 0; 1312 crp->crp_flags &= ~CRYPTO_F_DONE; 1313 cod->done = false; 1314 goto again; 1315 } 1316 1317 if (crp->crp_etype != 0) { 1318 error = crp->crp_etype; 1319 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1320 goto bail; 1321 } 1322 1323 if (caead->dst != NULL) { 1324 error = copyout(cod->obuf != NULL ? cod->obuf : 1325 cod->buf + crp->crp_payload_start, caead->dst, 1326 crp->crp_payload_length); 1327 if (error) { 1328 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1329 goto bail; 1330 } 1331 } 1332 1333 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1334 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1335 crp->crp_digest_start, caead->tag, cse->hashsize); 1336 if (error) { 1337 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1338 goto bail; 1339 } 1340 } 1341 1342 bail: 1343 crypto_freereq(crp); 1344 cod_free(cod); 1345 1346 return (error); 1347 } 1348 1349 static int 1350 cryptodev_cb(struct cryptop *crp) 1351 { 1352 struct cryptop_data *cod = crp->crp_opaque; 1353 1354 /* 1355 * Lock to ensure the wakeup() is not missed by the loops 1356 * waiting on cod->done in cryptodev_op() and 1357 * cryptodev_aead(). 1358 */ 1359 mtx_lock(&cod->cse->lock); 1360 cod->done = true; 1361 mtx_unlock(&cod->cse->lock); 1362 wakeup(cod); 1363 return (0); 1364 } 1365 1366 static void 1367 cryptodevkey_cb(struct cryptkop *krp) 1368 { 1369 1370 wakeup_one(krp); 1371 } 1372 1373 static int 1374 cryptodev_key(struct crypt_kop *kop) 1375 { 1376 struct cryptkop *krp = NULL; 1377 int error = EINVAL; 1378 int in, out, size, i; 1379 1380 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) { 1381 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1382 return (EFBIG); 1383 } 1384 1385 in = kop->crk_iparams; 1386 out = kop->crk_oparams; 1387 switch (kop->crk_op) { 1388 case CRK_MOD_EXP: 1389 if (in == 3 && out == 1) 1390 break; 1391 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1392 return (EINVAL); 1393 case CRK_MOD_EXP_CRT: 1394 if (in == 6 && out == 1) 1395 break; 1396 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1397 return (EINVAL); 1398 case CRK_DSA_SIGN: 1399 if (in == 5 && out == 2) 1400 break; 1401 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1402 return (EINVAL); 1403 case CRK_DSA_VERIFY: 1404 if (in == 7 && out == 0) 1405 break; 1406 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1407 return (EINVAL); 1408 case CRK_DH_COMPUTE_KEY: 1409 if (in == 3 && out == 1) 1410 break; 1411 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1412 return (EINVAL); 1413 default: 1414 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1415 return (EINVAL); 1416 } 1417 1418 krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO); 1419 krp->krp_op = kop->crk_op; 1420 krp->krp_status = kop->crk_status; 1421 krp->krp_iparams = kop->crk_iparams; 1422 krp->krp_oparams = kop->crk_oparams; 1423 krp->krp_crid = kop->crk_crid; 1424 krp->krp_status = 0; 1425 krp->krp_callback = cryptodevkey_cb; 1426 1427 for (i = 0; i < CRK_MAXPARAM; i++) { 1428 if (kop->crk_param[i].crp_nbits > 65536) { 1429 /* Limit is the same as in OpenBSD */ 1430 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1431 goto fail; 1432 } 1433 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 1434 } 1435 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1436 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1437 if (size == 0) 1438 continue; 1439 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK); 1440 if (i >= krp->krp_iparams) 1441 continue; 1442 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size); 1443 if (error) { 1444 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1445 goto fail; 1446 } 1447 } 1448 1449 error = crypto_kdispatch(krp); 1450 if (error) { 1451 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1452 goto fail; 1453 } 1454 error = tsleep(krp, PSOCK, "crydev", 0); 1455 if (error) { 1456 /* XXX can this happen? if so, how do we recover? */ 1457 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1458 goto fail; 1459 } 1460 1461 kop->crk_crid = krp->krp_hid; /* device that did the work */ 1462 if (krp->krp_status != 0) { 1463 error = krp->krp_status; 1464 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1465 goto fail; 1466 } 1467 1468 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) { 1469 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1470 if (size == 0) 1471 continue; 1472 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size); 1473 if (error) { 1474 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1475 goto fail; 1476 } 1477 } 1478 1479 fail: 1480 if (krp) { 1481 kop->crk_status = krp->krp_status; 1482 for (i = 0; i < CRK_MAXPARAM; i++) { 1483 if (krp->krp_param[i].crp_p) 1484 free(krp->krp_param[i].crp_p, M_XDATA); 1485 } 1486 free(krp, M_XDATA); 1487 } 1488 return (error); 1489 } 1490 1491 static int 1492 cryptodev_find(struct crypt_find_op *find) 1493 { 1494 device_t dev; 1495 size_t fnlen = sizeof find->name; 1496 1497 if (find->crid != -1) { 1498 dev = crypto_find_device_byhid(find->crid); 1499 if (dev == NULL) 1500 return (ENOENT); 1501 strncpy(find->name, device_get_nameunit(dev), fnlen); 1502 find->name[fnlen - 1] = '\x0'; 1503 } else { 1504 find->name[fnlen - 1] = '\x0'; 1505 find->crid = crypto_find_driver(find->name); 1506 if (find->crid == -1) 1507 return (ENOENT); 1508 } 1509 return (0); 1510 } 1511 1512 /* ARGSUSED */ 1513 static int 1514 cryptof_stat( 1515 struct file *fp, 1516 struct stat *sb, 1517 struct ucred *active_cred, 1518 struct thread *td) 1519 { 1520 1521 return (EOPNOTSUPP); 1522 } 1523 1524 /* ARGSUSED */ 1525 static int 1526 cryptof_close(struct file *fp, struct thread *td) 1527 { 1528 struct fcrypt *fcr = fp->f_data; 1529 struct csession *cse; 1530 1531 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1532 TAILQ_REMOVE(&fcr->csessions, cse, next); 1533 KASSERT(cse->refs == 1, 1534 ("%s: crypto session %p with %d refs", __func__, cse, 1535 cse->refs)); 1536 csefree(cse); 1537 } 1538 free(fcr, M_XDATA); 1539 fp->f_data = NULL; 1540 return 0; 1541 } 1542 1543 static int 1544 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp) 1545 { 1546 1547 kif->kf_type = KF_TYPE_CRYPTO; 1548 return (0); 1549 } 1550 1551 static struct csession * 1552 csefind(struct fcrypt *fcr, u_int ses) 1553 { 1554 struct csession *cse; 1555 1556 mtx_lock(&fcr->lock); 1557 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1558 if (cse->ses == ses) { 1559 refcount_acquire(&cse->refs); 1560 mtx_unlock(&fcr->lock); 1561 return (cse); 1562 } 1563 } 1564 mtx_unlock(&fcr->lock); 1565 return (NULL); 1566 } 1567 1568 static bool 1569 csedelete(struct fcrypt *fcr, u_int ses) 1570 { 1571 struct csession *cse; 1572 1573 mtx_lock(&fcr->lock); 1574 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1575 if (cse->ses == ses) { 1576 TAILQ_REMOVE(&fcr->csessions, cse, next); 1577 mtx_unlock(&fcr->lock); 1578 csefree(cse); 1579 return (true); 1580 } 1581 } 1582 mtx_unlock(&fcr->lock); 1583 return (false); 1584 } 1585 1586 struct csession * 1587 csecreate(struct fcrypt *fcr, crypto_session_t cses, 1588 struct crypto_session_params *csp, struct enc_xform *txform, 1589 void *key, struct auth_hash *thash, void *mackey) 1590 { 1591 struct csession *cse; 1592 1593 cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT | M_ZERO); 1594 if (cse == NULL) 1595 return NULL; 1596 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 1597 refcount_init(&cse->refs, 1); 1598 cse->key = key; 1599 cse->mackey = mackey; 1600 cse->mode = csp->csp_mode; 1601 cse->cses = cses; 1602 cse->txform = txform; 1603 if (thash != NULL) 1604 cse->hashsize = thash->hashsize; 1605 else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) 1606 cse->hashsize = AES_GMAC_HASH_LEN; 1607 else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) 1608 cse->hashsize = AES_CBC_MAC_HASH_LEN; 1609 cse->ivsize = csp->csp_ivlen; 1610 mtx_lock(&fcr->lock); 1611 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 1612 cse->ses = fcr->sesn++; 1613 mtx_unlock(&fcr->lock); 1614 return (cse); 1615 } 1616 1617 static void 1618 csefree(struct csession *cse) 1619 { 1620 1621 if (!refcount_release(&cse->refs)) 1622 return; 1623 crypto_freesession(cse->cses); 1624 mtx_destroy(&cse->lock); 1625 if (cse->key) 1626 free(cse->key, M_XDATA); 1627 if (cse->mackey) 1628 free(cse->mackey, M_XDATA); 1629 free(cse, M_XDATA); 1630 } 1631 1632 static int 1633 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td) 1634 { 1635 struct file *f; 1636 struct fcrypt *fcr; 1637 int fd, error; 1638 1639 switch (cmd) { 1640 case CRIOGET: 1641 error = falloc_noinstall(td, &f); 1642 if (error) 1643 break; 1644 1645 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1646 TAILQ_INIT(&fcr->csessions); 1647 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1648 1649 finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops); 1650 error = finstall(td, f, &fd, 0, NULL); 1651 if (error) { 1652 mtx_destroy(&fcr->lock); 1653 free(fcr, M_XDATA); 1654 } else 1655 *(uint32_t *)data = fd; 1656 fdrop(f, td); 1657 break; 1658 case CRIOFINDDEV: 1659 error = cryptodev_find((struct crypt_find_op *)data); 1660 break; 1661 case CRIOASYMFEAT: 1662 error = crypto_getfeat((int *)data); 1663 break; 1664 default: 1665 error = EINVAL; 1666 break; 1667 } 1668 return (error); 1669 } 1670 1671 static struct cdevsw crypto_cdevsw = { 1672 .d_version = D_VERSION, 1673 .d_ioctl = cryptoioctl, 1674 .d_name = "crypto", 1675 }; 1676 static struct cdev *crypto_dev; 1677 1678 /* 1679 * Initialization code, both for static and dynamic loading. 1680 */ 1681 static int 1682 cryptodev_modevent(module_t mod, int type, void *unused) 1683 { 1684 switch (type) { 1685 case MOD_LOAD: 1686 if (bootverbose) 1687 printf("crypto: <crypto device>\n"); 1688 crypto_dev = make_dev(&crypto_cdevsw, 0, 1689 UID_ROOT, GID_WHEEL, 0666, 1690 "crypto"); 1691 return 0; 1692 case MOD_UNLOAD: 1693 /*XXX disallow if active sessions */ 1694 destroy_dev(crypto_dev); 1695 return 0; 1696 } 1697 return EINVAL; 1698 } 1699 1700 static moduledata_t cryptodev_mod = { 1701 "cryptodev", 1702 cryptodev_modevent, 1703 0 1704 }; 1705 MODULE_VERSION(cryptodev, 1); 1706 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1707 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1708 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1709