1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 30 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 34 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Effort sponsored in part by the Defense Advanced Research Projects 37 * Agency (DARPA) and Air Force Research Laboratory, Air Force 38 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/sysctl.h> 51 #include <sys/file.h> 52 #include <sys/filedesc.h> 53 #include <sys/errno.h> 54 #include <sys/random.h> 55 #include <sys/conf.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/fcntl.h> 59 #include <sys/bus.h> 60 #include <sys/user.h> 61 #include <sys/sdt.h> 62 63 #include <opencrypto/cryptodev.h> 64 #include <opencrypto/xform.h> 65 66 SDT_PROVIDER_DECLARE(opencrypto); 67 68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 69 70 #ifdef COMPAT_FREEBSD32 71 #include <sys/mount.h> 72 #include <compat/freebsd32/freebsd32.h> 73 74 struct session_op32 { 75 u_int32_t cipher; 76 u_int32_t mac; 77 u_int32_t keylen; 78 u_int32_t key; 79 int mackeylen; 80 u_int32_t mackey; 81 u_int32_t ses; 82 }; 83 84 struct session2_op32 { 85 u_int32_t cipher; 86 u_int32_t mac; 87 u_int32_t keylen; 88 u_int32_t key; 89 int mackeylen; 90 u_int32_t mackey; 91 u_int32_t ses; 92 int crid; 93 int pad[4]; 94 }; 95 96 struct crypt_op32 { 97 u_int32_t ses; 98 u_int16_t op; 99 u_int16_t flags; 100 u_int len; 101 u_int32_t src, dst; 102 u_int32_t mac; 103 u_int32_t iv; 104 }; 105 106 struct crparam32 { 107 u_int32_t crp_p; 108 u_int crp_nbits; 109 }; 110 111 struct crypt_kop32 { 112 u_int crk_op; 113 u_int crk_status; 114 u_short crk_iparams; 115 u_short crk_oparams; 116 u_int crk_crid; 117 struct crparam32 crk_param[CRK_MAXPARAM]; 118 }; 119 120 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 121 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 122 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32) 123 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 124 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32) 125 126 static void 127 session_op_from_32(const struct session_op32 *from, struct session_op *to) 128 { 129 130 CP(*from, *to, cipher); 131 CP(*from, *to, mac); 132 CP(*from, *to, keylen); 133 PTRIN_CP(*from, *to, key); 134 CP(*from, *to, mackeylen); 135 PTRIN_CP(*from, *to, mackey); 136 CP(*from, *to, ses); 137 } 138 139 static void 140 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 141 { 142 143 session_op_from_32((const struct session_op32 *)from, 144 (struct session_op *)to); 145 CP(*from, *to, crid); 146 } 147 148 static void 149 session_op_to_32(const struct session_op *from, struct session_op32 *to) 150 { 151 152 CP(*from, *to, cipher); 153 CP(*from, *to, mac); 154 CP(*from, *to, keylen); 155 PTROUT_CP(*from, *to, key); 156 CP(*from, *to, mackeylen); 157 PTROUT_CP(*from, *to, mackey); 158 CP(*from, *to, ses); 159 } 160 161 static void 162 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 163 { 164 165 session_op_to_32((const struct session_op *)from, 166 (struct session_op32 *)to); 167 CP(*from, *to, crid); 168 } 169 170 static void 171 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 172 { 173 174 CP(*from, *to, ses); 175 CP(*from, *to, op); 176 CP(*from, *to, flags); 177 CP(*from, *to, len); 178 PTRIN_CP(*from, *to, src); 179 PTRIN_CP(*from, *to, dst); 180 PTRIN_CP(*from, *to, mac); 181 PTRIN_CP(*from, *to, iv); 182 } 183 184 static void 185 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 186 { 187 188 CP(*from, *to, ses); 189 CP(*from, *to, op); 190 CP(*from, *to, flags); 191 CP(*from, *to, len); 192 PTROUT_CP(*from, *to, src); 193 PTROUT_CP(*from, *to, dst); 194 PTROUT_CP(*from, *to, mac); 195 PTROUT_CP(*from, *to, iv); 196 } 197 198 static void 199 crparam_from_32(const struct crparam32 *from, struct crparam *to) 200 { 201 202 PTRIN_CP(*from, *to, crp_p); 203 CP(*from, *to, crp_nbits); 204 } 205 206 static void 207 crparam_to_32(const struct crparam *from, struct crparam32 *to) 208 { 209 210 PTROUT_CP(*from, *to, crp_p); 211 CP(*from, *to, crp_nbits); 212 } 213 214 static void 215 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to) 216 { 217 int i; 218 219 CP(*from, *to, crk_op); 220 CP(*from, *to, crk_status); 221 CP(*from, *to, crk_iparams); 222 CP(*from, *to, crk_oparams); 223 CP(*from, *to, crk_crid); 224 for (i = 0; i < CRK_MAXPARAM; i++) 225 crparam_from_32(&from->crk_param[i], &to->crk_param[i]); 226 } 227 228 static void 229 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to) 230 { 231 int i; 232 233 CP(*from, *to, crk_op); 234 CP(*from, *to, crk_status); 235 CP(*from, *to, crk_iparams); 236 CP(*from, *to, crk_oparams); 237 CP(*from, *to, crk_crid); 238 for (i = 0; i < CRK_MAXPARAM; i++) 239 crparam_to_32(&from->crk_param[i], &to->crk_param[i]); 240 } 241 #endif 242 243 struct csession { 244 TAILQ_ENTRY(csession) next; 245 crypto_session_t cses; 246 volatile u_int refs; 247 u_int32_t ses; 248 struct mtx lock; /* for op submission */ 249 250 struct enc_xform *txform; 251 int hashsize; 252 int ivsize; 253 int mode; 254 255 void *key; 256 void *mackey; 257 }; 258 259 struct cryptop_data { 260 struct csession *cse; 261 262 char *buf; 263 char *obuf; 264 char *aad; 265 bool done; 266 }; 267 268 struct fcrypt { 269 TAILQ_HEAD(csessionlist, csession) csessions; 270 int sesn; 271 struct mtx lock; 272 }; 273 274 static bool use_outputbuffers; 275 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 276 &use_outputbuffers, 0, 277 "Use separate output buffers for /dev/crypto requests."); 278 279 static bool use_separate_aad; 280 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 281 &use_separate_aad, 0, 282 "Use separate AAD buffer for /dev/crypto requests."); 283 284 static int cryptof_ioctl(struct file *, u_long, void *, 285 struct ucred *, struct thread *); 286 static int cryptof_stat(struct file *, struct stat *, 287 struct ucred *, struct thread *); 288 static int cryptof_close(struct file *, struct thread *); 289 static int cryptof_fill_kinfo(struct file *, struct kinfo_file *, 290 struct filedesc *); 291 292 static struct fileops cryptofops = { 293 .fo_read = invfo_rdwr, 294 .fo_write = invfo_rdwr, 295 .fo_truncate = invfo_truncate, 296 .fo_ioctl = cryptof_ioctl, 297 .fo_poll = invfo_poll, 298 .fo_kqfilter = invfo_kqfilter, 299 .fo_stat = cryptof_stat, 300 .fo_close = cryptof_close, 301 .fo_chmod = invfo_chmod, 302 .fo_chown = invfo_chown, 303 .fo_sendfile = invfo_sendfile, 304 .fo_fill_kinfo = cryptof_fill_kinfo, 305 }; 306 307 static struct csession *csefind(struct fcrypt *, u_int); 308 static bool csedelete(struct fcrypt *, u_int); 309 static struct csession *csecreate(struct fcrypt *, crypto_session_t, 310 struct crypto_session_params *, struct enc_xform *, void *, 311 struct auth_hash *, void *); 312 static void csefree(struct csession *); 313 314 static int cryptodev_op(struct csession *, struct crypt_op *, 315 struct ucred *, struct thread *td); 316 static int cryptodev_aead(struct csession *, struct crypt_aead *, 317 struct ucred *, struct thread *); 318 static int cryptodev_key(struct crypt_kop *); 319 static int cryptodev_find(struct crypt_find_op *); 320 321 /* 322 * Check a crypto identifier to see if it requested 323 * a software device/driver. This can be done either 324 * by device name/class or through search constraints. 325 */ 326 static int 327 checkforsoftware(int *cridp) 328 { 329 int crid; 330 331 crid = *cridp; 332 333 if (!crypto_devallowsoft) { 334 if (crid & CRYPTOCAP_F_SOFTWARE) { 335 if (crid & CRYPTOCAP_F_HARDWARE) { 336 *cridp = CRYPTOCAP_F_HARDWARE; 337 return 0; 338 } 339 return EINVAL; 340 } 341 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 342 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 343 return EINVAL; 344 } 345 return 0; 346 } 347 348 /* ARGSUSED */ 349 static int 350 cryptof_ioctl( 351 struct file *fp, 352 u_long cmd, 353 void *data, 354 struct ucred *active_cred, 355 struct thread *td) 356 { 357 #define SES2(p) ((struct session2_op *)p) 358 struct crypto_session_params csp; 359 struct fcrypt *fcr = fp->f_data; 360 struct csession *cse; 361 struct session_op *sop; 362 struct crypt_op *cop; 363 struct crypt_aead *caead; 364 struct enc_xform *txform = NULL; 365 struct auth_hash *thash = NULL; 366 void *key = NULL; 367 void *mackey = NULL; 368 struct crypt_kop *kop; 369 crypto_session_t cses; 370 u_int32_t ses; 371 int error = 0, crid; 372 #ifdef COMPAT_FREEBSD32 373 struct session2_op sopc; 374 struct crypt_op copc; 375 struct crypt_kop kopc; 376 #endif 377 378 switch (cmd) { 379 case CIOCGSESSION: 380 case CIOCGSESSION2: 381 #ifdef COMPAT_FREEBSD32 382 case CIOCGSESSION32: 383 case CIOCGSESSION232: 384 if (cmd == CIOCGSESSION32) { 385 session_op_from_32(data, (struct session_op *)&sopc); 386 sop = (struct session_op *)&sopc; 387 } else if (cmd == CIOCGSESSION232) { 388 session2_op_from_32(data, &sopc); 389 sop = (struct session_op *)&sopc; 390 } else 391 #endif 392 sop = (struct session_op *)data; 393 switch (sop->cipher) { 394 case 0: 395 break; 396 case CRYPTO_AES_CBC: 397 txform = &enc_xform_rijndael128; 398 break; 399 case CRYPTO_AES_XTS: 400 txform = &enc_xform_aes_xts; 401 break; 402 case CRYPTO_NULL_CBC: 403 txform = &enc_xform_null; 404 break; 405 case CRYPTO_CAMELLIA_CBC: 406 txform = &enc_xform_camellia; 407 break; 408 case CRYPTO_AES_ICM: 409 txform = &enc_xform_aes_icm; 410 break; 411 case CRYPTO_AES_NIST_GCM_16: 412 txform = &enc_xform_aes_nist_gcm; 413 break; 414 case CRYPTO_CHACHA20: 415 txform = &enc_xform_chacha20; 416 break; 417 case CRYPTO_AES_CCM_16: 418 txform = &enc_xform_ccm; 419 break; 420 421 default: 422 CRYPTDEB("invalid cipher"); 423 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 424 return (EINVAL); 425 } 426 427 switch (sop->mac) { 428 case 0: 429 break; 430 case CRYPTO_POLY1305: 431 thash = &auth_hash_poly1305; 432 break; 433 case CRYPTO_SHA1_HMAC: 434 thash = &auth_hash_hmac_sha1; 435 break; 436 case CRYPTO_SHA2_224_HMAC: 437 thash = &auth_hash_hmac_sha2_224; 438 break; 439 case CRYPTO_SHA2_256_HMAC: 440 thash = &auth_hash_hmac_sha2_256; 441 break; 442 case CRYPTO_SHA2_384_HMAC: 443 thash = &auth_hash_hmac_sha2_384; 444 break; 445 case CRYPTO_SHA2_512_HMAC: 446 thash = &auth_hash_hmac_sha2_512; 447 break; 448 case CRYPTO_RIPEMD160_HMAC: 449 thash = &auth_hash_hmac_ripemd_160; 450 break; 451 #ifdef COMPAT_FREEBSD12 452 case CRYPTO_AES_128_NIST_GMAC: 453 case CRYPTO_AES_192_NIST_GMAC: 454 case CRYPTO_AES_256_NIST_GMAC: 455 /* Should always be paired with GCM. */ 456 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 457 CRYPTDEB("GMAC without GCM"); 458 SDT_PROBE1(opencrypto, dev, ioctl, error, 459 __LINE__); 460 return (EINVAL); 461 } 462 break; 463 #endif 464 case CRYPTO_AES_NIST_GMAC: 465 switch (sop->mackeylen * 8) { 466 case 128: 467 thash = &auth_hash_nist_gmac_aes_128; 468 break; 469 case 192: 470 thash = &auth_hash_nist_gmac_aes_192; 471 break; 472 case 256: 473 thash = &auth_hash_nist_gmac_aes_256; 474 break; 475 default: 476 CRYPTDEB("invalid GMAC key length"); 477 SDT_PROBE1(opencrypto, dev, ioctl, error, 478 __LINE__); 479 return (EINVAL); 480 } 481 break; 482 case CRYPTO_AES_CCM_CBC_MAC: 483 switch (sop->mackeylen) { 484 case 16: 485 thash = &auth_hash_ccm_cbc_mac_128; 486 break; 487 case 24: 488 thash = &auth_hash_ccm_cbc_mac_192; 489 break; 490 case 32: 491 thash = &auth_hash_ccm_cbc_mac_256; 492 break; 493 default: 494 CRYPTDEB("Invalid CBC MAC key size %d", 495 sop->keylen); 496 SDT_PROBE1(opencrypto, dev, ioctl, 497 error, __LINE__); 498 return (EINVAL); 499 } 500 break; 501 case CRYPTO_SHA1: 502 thash = &auth_hash_sha1; 503 break; 504 case CRYPTO_SHA2_224: 505 thash = &auth_hash_sha2_224; 506 break; 507 case CRYPTO_SHA2_256: 508 thash = &auth_hash_sha2_256; 509 break; 510 case CRYPTO_SHA2_384: 511 thash = &auth_hash_sha2_384; 512 break; 513 case CRYPTO_SHA2_512: 514 thash = &auth_hash_sha2_512; 515 break; 516 517 case CRYPTO_NULL_HMAC: 518 thash = &auth_hash_null; 519 break; 520 521 case CRYPTO_BLAKE2B: 522 thash = &auth_hash_blake2b; 523 break; 524 case CRYPTO_BLAKE2S: 525 thash = &auth_hash_blake2s; 526 break; 527 528 default: 529 CRYPTDEB("invalid mac"); 530 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 531 return (EINVAL); 532 } 533 534 if (txform == NULL && thash == NULL) { 535 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 536 return (EINVAL); 537 } 538 539 memset(&csp, 0, sizeof(csp)); 540 if (use_outputbuffers) 541 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 542 543 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { 544 switch (sop->mac) { 545 #ifdef COMPAT_FREEBSD12 546 case CRYPTO_AES_128_NIST_GMAC: 547 case CRYPTO_AES_192_NIST_GMAC: 548 case CRYPTO_AES_256_NIST_GMAC: 549 if (sop->keylen != sop->mackeylen) { 550 SDT_PROBE1(opencrypto, dev, ioctl, 551 error, __LINE__); 552 return (EINVAL); 553 } 554 break; 555 #endif 556 case 0: 557 break; 558 default: 559 SDT_PROBE1(opencrypto, dev, ioctl, error, 560 __LINE__); 561 return (EINVAL); 562 } 563 csp.csp_mode = CSP_MODE_AEAD; 564 } else if (sop->cipher == CRYPTO_AES_CCM_16) { 565 switch (sop->mac) { 566 #ifdef COMPAT_FREEBSD12 567 case CRYPTO_AES_CCM_CBC_MAC: 568 if (sop->keylen != sop->mackeylen) { 569 SDT_PROBE1(opencrypto, dev, ioctl, 570 error, __LINE__); 571 return (EINVAL); 572 } 573 thash = NULL; 574 break; 575 #endif 576 case 0: 577 break; 578 default: 579 SDT_PROBE1(opencrypto, dev, ioctl, error, 580 __LINE__); 581 return (EINVAL); 582 } 583 csp.csp_mode = CSP_MODE_AEAD; 584 } else if (txform && thash) 585 csp.csp_mode = CSP_MODE_ETA; 586 else if (txform) 587 csp.csp_mode = CSP_MODE_CIPHER; 588 else 589 csp.csp_mode = CSP_MODE_DIGEST; 590 591 switch (csp.csp_mode) { 592 case CSP_MODE_AEAD: 593 case CSP_MODE_ETA: 594 if (use_separate_aad) 595 csp.csp_flags |= CSP_F_SEPARATE_AAD; 596 break; 597 } 598 599 if (txform) { 600 csp.csp_cipher_alg = txform->type; 601 csp.csp_cipher_klen = sop->keylen; 602 if (sop->keylen > txform->maxkey || 603 sop->keylen < txform->minkey) { 604 CRYPTDEB("invalid cipher parameters"); 605 error = EINVAL; 606 SDT_PROBE1(opencrypto, dev, ioctl, error, 607 __LINE__); 608 goto bail; 609 } 610 611 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 612 error = copyin(sop->key, key, csp.csp_cipher_klen); 613 if (error) { 614 CRYPTDEB("invalid key"); 615 SDT_PROBE1(opencrypto, dev, ioctl, error, 616 __LINE__); 617 goto bail; 618 } 619 csp.csp_cipher_key = key; 620 csp.csp_ivlen = txform->ivsize; 621 } 622 623 if (thash) { 624 csp.csp_auth_alg = thash->type; 625 csp.csp_auth_klen = sop->mackeylen; 626 if (sop->mackeylen > thash->keysize || 627 sop->mackeylen < 0) { 628 CRYPTDEB("invalid mac key length"); 629 error = EINVAL; 630 SDT_PROBE1(opencrypto, dev, ioctl, error, 631 __LINE__); 632 goto bail; 633 } 634 635 if (csp.csp_auth_klen) { 636 mackey = malloc(csp.csp_auth_klen, M_XDATA, 637 M_WAITOK); 638 error = copyin(sop->mackey, mackey, 639 csp.csp_auth_klen); 640 if (error) { 641 CRYPTDEB("invalid mac key"); 642 SDT_PROBE1(opencrypto, dev, ioctl, 643 error, __LINE__); 644 goto bail; 645 } 646 csp.csp_auth_key = mackey; 647 } 648 649 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 650 csp.csp_ivlen = AES_GCM_IV_LEN; 651 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 652 csp.csp_ivlen = AES_CCM_IV_LEN; 653 } 654 655 /* NB: CIOCGSESSION2 has the crid */ 656 if (cmd == CIOCGSESSION2 657 #ifdef COMPAT_FREEBSD32 658 || cmd == CIOCGSESSION232 659 #endif 660 ) { 661 crid = SES2(sop)->crid; 662 error = checkforsoftware(&crid); 663 if (error) { 664 CRYPTDEB("checkforsoftware"); 665 SDT_PROBE1(opencrypto, dev, ioctl, error, 666 __LINE__); 667 goto bail; 668 } 669 } else 670 crid = CRYPTOCAP_F_HARDWARE; 671 error = crypto_newsession(&cses, &csp, crid); 672 if (error) { 673 CRYPTDEB("crypto_newsession"); 674 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 675 goto bail; 676 } 677 678 cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey); 679 680 if (cse == NULL) { 681 crypto_freesession(cses); 682 error = EINVAL; 683 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 684 CRYPTDEB("csecreate"); 685 goto bail; 686 } 687 sop->ses = cse->ses; 688 if (cmd == CIOCGSESSION2 689 #ifdef COMPAT_FREEBSD32 690 || cmd == CIOCGSESSION232 691 #endif 692 ) { 693 /* return hardware/driver id */ 694 SES2(sop)->crid = crypto_ses2hid(cse->cses); 695 } 696 bail: 697 if (error) { 698 free(key, M_XDATA); 699 free(mackey, M_XDATA); 700 } 701 #ifdef COMPAT_FREEBSD32 702 else { 703 if (cmd == CIOCGSESSION32) 704 session_op_to_32(sop, data); 705 else if (cmd == CIOCGSESSION232) 706 session2_op_to_32((struct session2_op *)sop, 707 data); 708 } 709 #endif 710 break; 711 case CIOCFSESSION: 712 ses = *(u_int32_t *)data; 713 if (!csedelete(fcr, ses)) { 714 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 715 return (EINVAL); 716 } 717 break; 718 case CIOCCRYPT: 719 #ifdef COMPAT_FREEBSD32 720 case CIOCCRYPT32: 721 if (cmd == CIOCCRYPT32) { 722 cop = &copc; 723 crypt_op_from_32(data, cop); 724 } else 725 #endif 726 cop = (struct crypt_op *)data; 727 cse = csefind(fcr, cop->ses); 728 if (cse == NULL) { 729 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 730 return (EINVAL); 731 } 732 error = cryptodev_op(cse, cop, active_cred, td); 733 csefree(cse); 734 #ifdef COMPAT_FREEBSD32 735 if (error == 0 && cmd == CIOCCRYPT32) 736 crypt_op_to_32(cop, data); 737 #endif 738 break; 739 case CIOCKEY: 740 case CIOCKEY2: 741 #ifdef COMPAT_FREEBSD32 742 case CIOCKEY32: 743 case CIOCKEY232: 744 #endif 745 if (!crypto_userasymcrypto) { 746 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 747 return (EPERM); /* XXX compat? */ 748 } 749 #ifdef COMPAT_FREEBSD32 750 if (cmd == CIOCKEY32 || cmd == CIOCKEY232) { 751 kop = &kopc; 752 crypt_kop_from_32(data, kop); 753 } else 754 #endif 755 kop = (struct crypt_kop *)data; 756 if (cmd == CIOCKEY 757 #ifdef COMPAT_FREEBSD32 758 || cmd == CIOCKEY32 759 #endif 760 ) { 761 /* NB: crypto core enforces s/w driver use */ 762 kop->crk_crid = 763 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE; 764 } 765 mtx_lock(&Giant); 766 error = cryptodev_key(kop); 767 mtx_unlock(&Giant); 768 #ifdef COMPAT_FREEBSD32 769 if (cmd == CIOCKEY32 || cmd == CIOCKEY232) 770 crypt_kop_to_32(kop, data); 771 #endif 772 break; 773 case CIOCASYMFEAT: 774 if (!crypto_userasymcrypto) { 775 /* 776 * NB: if user asym crypto operations are 777 * not permitted return "no algorithms" 778 * so well-behaved applications will just 779 * fallback to doing them in software. 780 */ 781 *(int *)data = 0; 782 } else { 783 error = crypto_getfeat((int *)data); 784 if (error) 785 SDT_PROBE1(opencrypto, dev, ioctl, error, 786 __LINE__); 787 } 788 break; 789 case CIOCFINDDEV: 790 error = cryptodev_find((struct crypt_find_op *)data); 791 break; 792 case CIOCCRYPTAEAD: 793 caead = (struct crypt_aead *)data; 794 cse = csefind(fcr, caead->ses); 795 if (cse == NULL) { 796 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 797 return (EINVAL); 798 } 799 error = cryptodev_aead(cse, caead, active_cred, td); 800 csefree(cse); 801 break; 802 default: 803 error = EINVAL; 804 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 805 break; 806 } 807 return (error); 808 #undef SES2 809 } 810 811 static int cryptodev_cb(struct cryptop *); 812 813 static struct cryptop_data * 814 cod_alloc(struct csession *cse, size_t aad_len, size_t len, struct thread *td) 815 { 816 struct cryptop_data *cod; 817 818 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 819 820 cod->cse = cse; 821 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 822 if (aad_len != 0) 823 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 824 cod->buf = malloc(len, M_XDATA, M_WAITOK); 825 } else 826 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 827 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 828 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 829 return (cod); 830 } 831 832 static void 833 cod_free(struct cryptop_data *cod) 834 { 835 836 free(cod->aad, M_XDATA); 837 free(cod->obuf, M_XDATA); 838 free(cod->buf, M_XDATA); 839 free(cod, M_XDATA); 840 } 841 842 static int 843 cryptodev_op( 844 struct csession *cse, 845 struct crypt_op *cop, 846 struct ucred *active_cred, 847 struct thread *td) 848 { 849 struct cryptop_data *cod = NULL; 850 struct cryptop *crp = NULL; 851 int error; 852 853 if (cop->len > 256*1024-4) { 854 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 855 return (E2BIG); 856 } 857 858 if (cse->txform) { 859 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { 860 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 861 return (EINVAL); 862 } 863 } 864 865 if (cop->mac && cse->hashsize == 0) { 866 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 867 return (EINVAL); 868 } 869 870 /* 871 * The COP_F_CIPHER_FIRST flag predates explicit session 872 * modes, but the only way it was used was for EtA so allow it 873 * as long as it is consistent with EtA. 874 */ 875 if (cop->flags & COP_F_CIPHER_FIRST) { 876 if (cop->op != COP_ENCRYPT) { 877 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 878 return (EINVAL); 879 } 880 } 881 882 cod = cod_alloc(cse, 0, cop->len + cse->hashsize, td); 883 884 crp = crypto_getreq(cse->cses, M_WAITOK); 885 886 error = copyin(cop->src, cod->buf, cop->len); 887 if (error) { 888 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 889 goto bail; 890 } 891 crp->crp_payload_start = 0; 892 crp->crp_payload_length = cop->len; 893 if (cse->hashsize) 894 crp->crp_digest_start = cop->len; 895 896 switch (cse->mode) { 897 case CSP_MODE_COMPRESS: 898 switch (cop->op) { 899 case COP_ENCRYPT: 900 crp->crp_op = CRYPTO_OP_COMPRESS; 901 break; 902 case COP_DECRYPT: 903 crp->crp_op = CRYPTO_OP_DECOMPRESS; 904 break; 905 default: 906 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 907 error = EINVAL; 908 goto bail; 909 } 910 break; 911 case CSP_MODE_CIPHER: 912 switch (cop->op) { 913 case COP_ENCRYPT: 914 crp->crp_op = CRYPTO_OP_ENCRYPT; 915 break; 916 case COP_DECRYPT: 917 crp->crp_op = CRYPTO_OP_DECRYPT; 918 break; 919 default: 920 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 921 error = EINVAL; 922 goto bail; 923 } 924 break; 925 case CSP_MODE_DIGEST: 926 switch (cop->op) { 927 case 0: 928 case COP_ENCRYPT: 929 case COP_DECRYPT: 930 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 931 if (cod->obuf != NULL) 932 crp->crp_digest_start = 0; 933 break; 934 default: 935 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 936 error = EINVAL; 937 goto bail; 938 } 939 break; 940 case CSP_MODE_ETA: 941 switch (cop->op) { 942 case COP_ENCRYPT: 943 crp->crp_op = CRYPTO_OP_ENCRYPT | 944 CRYPTO_OP_COMPUTE_DIGEST; 945 break; 946 case COP_DECRYPT: 947 crp->crp_op = CRYPTO_OP_DECRYPT | 948 CRYPTO_OP_VERIFY_DIGEST; 949 break; 950 default: 951 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 952 error = EINVAL; 953 goto bail; 954 } 955 break; 956 default: 957 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 958 error = EINVAL; 959 goto bail; 960 } 961 962 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 963 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 964 if (cod->obuf) 965 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 966 crp->crp_callback = cryptodev_cb; 967 crp->crp_opaque = cod; 968 969 if (cop->iv) { 970 if (cse->ivsize == 0) { 971 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 972 error = EINVAL; 973 goto bail; 974 } 975 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 976 if (error) { 977 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 978 goto bail; 979 } 980 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 981 } else if (cse->ivsize != 0) { 982 crp->crp_iv_start = 0; 983 crp->crp_payload_start += cse->ivsize; 984 crp->crp_payload_length -= cse->ivsize; 985 cop->dst += cse->ivsize; 986 } 987 988 if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 989 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 990 cse->hashsize); 991 if (error) { 992 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 993 goto bail; 994 } 995 } 996 again: 997 /* 998 * Let the dispatch run unlocked, then, interlock against the 999 * callback before checking if the operation completed and going 1000 * to sleep. This insures drivers don't inherit our lock which 1001 * results in a lock order reversal between crypto_dispatch forced 1002 * entry and the crypto_done callback into us. 1003 */ 1004 error = crypto_dispatch(crp); 1005 if (error != 0) { 1006 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1007 goto bail; 1008 } 1009 1010 mtx_lock(&cse->lock); 1011 while (!cod->done) 1012 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1013 mtx_unlock(&cse->lock); 1014 1015 if (crp->crp_etype == EAGAIN) { 1016 crp->crp_etype = 0; 1017 crp->crp_flags &= ~CRYPTO_F_DONE; 1018 cod->done = false; 1019 goto again; 1020 } 1021 1022 if (crp->crp_etype != 0) { 1023 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1024 error = crp->crp_etype; 1025 goto bail; 1026 } 1027 1028 if (cop->dst != NULL) { 1029 error = copyout(cod->obuf != NULL ? cod->obuf : 1030 cod->buf + crp->crp_payload_start, cop->dst, 1031 crp->crp_payload_length); 1032 if (error) { 1033 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1034 goto bail; 1035 } 1036 } 1037 1038 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1039 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1040 crp->crp_digest_start, cop->mac, cse->hashsize); 1041 if (error) { 1042 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1043 goto bail; 1044 } 1045 } 1046 1047 bail: 1048 crypto_freereq(crp); 1049 cod_free(cod); 1050 1051 return (error); 1052 } 1053 1054 static int 1055 cryptodev_aead( 1056 struct csession *cse, 1057 struct crypt_aead *caead, 1058 struct ucred *active_cred, 1059 struct thread *td) 1060 { 1061 struct cryptop_data *cod = NULL; 1062 struct cryptop *crp = NULL; 1063 int error; 1064 1065 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 1066 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1067 return (E2BIG); 1068 } 1069 1070 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || 1071 (caead->len % cse->txform->blocksize) != 0) { 1072 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1073 return (EINVAL); 1074 } 1075 1076 /* 1077 * The COP_F_CIPHER_FIRST flag predates explicit session 1078 * modes, but the only way it was used was for EtA so allow it 1079 * as long as it is consistent with EtA. 1080 */ 1081 if (caead->flags & COP_F_CIPHER_FIRST) { 1082 if (caead->op != COP_ENCRYPT) { 1083 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1084 return (EINVAL); 1085 } 1086 } 1087 1088 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize, td); 1089 1090 crp = crypto_getreq(cse->cses, M_WAITOK); 1091 1092 if (cod->aad != NULL) 1093 error = copyin(caead->aad, cod->aad, caead->aadlen); 1094 else 1095 error = copyin(caead->aad, cod->buf, caead->aadlen); 1096 if (error) { 1097 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1098 goto bail; 1099 } 1100 crp->crp_aad = cod->aad; 1101 crp->crp_aad_start = 0; 1102 crp->crp_aad_length = caead->aadlen; 1103 1104 if (cod->aad != NULL) 1105 crp->crp_payload_start = 0; 1106 else 1107 crp->crp_payload_start = caead->aadlen; 1108 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 1109 caead->len); 1110 if (error) { 1111 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1112 goto bail; 1113 } 1114 crp->crp_payload_length = caead->len; 1115 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 1116 crp->crp_digest_start = crp->crp_payload_output_start + 1117 caead->len; 1118 else 1119 crp->crp_digest_start = crp->crp_payload_start + caead->len; 1120 1121 switch (cse->mode) { 1122 case CSP_MODE_AEAD: 1123 case CSP_MODE_ETA: 1124 switch (caead->op) { 1125 case COP_ENCRYPT: 1126 crp->crp_op = CRYPTO_OP_ENCRYPT | 1127 CRYPTO_OP_COMPUTE_DIGEST; 1128 break; 1129 case COP_DECRYPT: 1130 crp->crp_op = CRYPTO_OP_DECRYPT | 1131 CRYPTO_OP_VERIFY_DIGEST; 1132 break; 1133 default: 1134 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1135 error = EINVAL; 1136 goto bail; 1137 } 1138 break; 1139 default: 1140 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1141 error = EINVAL; 1142 goto bail; 1143 } 1144 1145 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 1146 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 1147 cse->hashsize); 1148 if (cod->obuf != NULL) 1149 crypto_use_output_buf(crp, cod->obuf, caead->len + 1150 cse->hashsize); 1151 crp->crp_callback = cryptodev_cb; 1152 crp->crp_opaque = cod; 1153 1154 if (caead->iv) { 1155 /* 1156 * Permit a 16-byte IV for AES-XTS, but only use the 1157 * first 8 bytes as a block number. 1158 */ 1159 if (cse->mode == CSP_MODE_ETA && 1160 caead->ivlen == AES_BLOCK_LEN && 1161 cse->ivsize == AES_XTS_IV_LEN) 1162 caead->ivlen = AES_XTS_IV_LEN; 1163 1164 if (caead->ivlen != cse->ivsize) { 1165 error = EINVAL; 1166 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1167 goto bail; 1168 } 1169 1170 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 1171 if (error) { 1172 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1173 goto bail; 1174 } 1175 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1176 } else { 1177 crp->crp_iv_start = crp->crp_payload_start; 1178 crp->crp_payload_start += cse->ivsize; 1179 crp->crp_payload_length -= cse->ivsize; 1180 caead->dst += cse->ivsize; 1181 } 1182 1183 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1184 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1185 cse->hashsize); 1186 if (error) { 1187 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1188 goto bail; 1189 } 1190 } 1191 again: 1192 /* 1193 * Let the dispatch run unlocked, then, interlock against the 1194 * callback before checking if the operation completed and going 1195 * to sleep. This insures drivers don't inherit our lock which 1196 * results in a lock order reversal between crypto_dispatch forced 1197 * entry and the crypto_done callback into us. 1198 */ 1199 error = crypto_dispatch(crp); 1200 if (error != 0) { 1201 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1202 goto bail; 1203 } 1204 1205 mtx_lock(&cse->lock); 1206 while (!cod->done) 1207 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1208 mtx_unlock(&cse->lock); 1209 1210 if (crp->crp_etype == EAGAIN) { 1211 crp->crp_etype = 0; 1212 crp->crp_flags &= ~CRYPTO_F_DONE; 1213 cod->done = false; 1214 goto again; 1215 } 1216 1217 if (crp->crp_etype != 0) { 1218 error = crp->crp_etype; 1219 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1220 goto bail; 1221 } 1222 1223 if (caead->dst != NULL) { 1224 error = copyout(cod->obuf != NULL ? cod->obuf : 1225 cod->buf + crp->crp_payload_start, caead->dst, 1226 crp->crp_payload_length); 1227 if (error) { 1228 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1229 goto bail; 1230 } 1231 } 1232 1233 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1234 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1235 crp->crp_digest_start, caead->tag, cse->hashsize); 1236 if (error) { 1237 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1238 goto bail; 1239 } 1240 } 1241 1242 bail: 1243 crypto_freereq(crp); 1244 cod_free(cod); 1245 1246 return (error); 1247 } 1248 1249 static int 1250 cryptodev_cb(struct cryptop *crp) 1251 { 1252 struct cryptop_data *cod = crp->crp_opaque; 1253 1254 /* 1255 * Lock to ensure the wakeup() is not missed by the loops 1256 * waiting on cod->done in cryptodev_op() and 1257 * cryptodev_aead(). 1258 */ 1259 mtx_lock(&cod->cse->lock); 1260 cod->done = true; 1261 mtx_unlock(&cod->cse->lock); 1262 wakeup(cod); 1263 return (0); 1264 } 1265 1266 static void 1267 cryptodevkey_cb(struct cryptkop *krp) 1268 { 1269 1270 wakeup_one(krp); 1271 } 1272 1273 static int 1274 cryptodev_key(struct crypt_kop *kop) 1275 { 1276 struct cryptkop *krp = NULL; 1277 int error = EINVAL; 1278 int in, out, size, i; 1279 1280 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) { 1281 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1282 return (EFBIG); 1283 } 1284 1285 in = kop->crk_iparams; 1286 out = kop->crk_oparams; 1287 switch (kop->crk_op) { 1288 case CRK_MOD_EXP: 1289 if (in == 3 && out == 1) 1290 break; 1291 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1292 return (EINVAL); 1293 case CRK_MOD_EXP_CRT: 1294 if (in == 6 && out == 1) 1295 break; 1296 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1297 return (EINVAL); 1298 case CRK_DSA_SIGN: 1299 if (in == 5 && out == 2) 1300 break; 1301 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1302 return (EINVAL); 1303 case CRK_DSA_VERIFY: 1304 if (in == 7 && out == 0) 1305 break; 1306 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1307 return (EINVAL); 1308 case CRK_DH_COMPUTE_KEY: 1309 if (in == 3 && out == 1) 1310 break; 1311 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1312 return (EINVAL); 1313 default: 1314 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1315 return (EINVAL); 1316 } 1317 1318 krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO); 1319 krp->krp_op = kop->crk_op; 1320 krp->krp_status = kop->crk_status; 1321 krp->krp_iparams = kop->crk_iparams; 1322 krp->krp_oparams = kop->crk_oparams; 1323 krp->krp_crid = kop->crk_crid; 1324 krp->krp_status = 0; 1325 krp->krp_callback = cryptodevkey_cb; 1326 1327 for (i = 0; i < CRK_MAXPARAM; i++) { 1328 if (kop->crk_param[i].crp_nbits > 65536) { 1329 /* Limit is the same as in OpenBSD */ 1330 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1331 goto fail; 1332 } 1333 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 1334 } 1335 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1336 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1337 if (size == 0) 1338 continue; 1339 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK); 1340 if (i >= krp->krp_iparams) 1341 continue; 1342 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size); 1343 if (error) { 1344 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1345 goto fail; 1346 } 1347 } 1348 1349 error = crypto_kdispatch(krp); 1350 if (error) { 1351 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1352 goto fail; 1353 } 1354 error = tsleep(krp, PSOCK, "crydev", 0); 1355 if (error) { 1356 /* XXX can this happen? if so, how do we recover? */ 1357 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1358 goto fail; 1359 } 1360 1361 kop->crk_crid = krp->krp_hid; /* device that did the work */ 1362 if (krp->krp_status != 0) { 1363 error = krp->krp_status; 1364 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1365 goto fail; 1366 } 1367 1368 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) { 1369 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1370 if (size == 0) 1371 continue; 1372 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size); 1373 if (error) { 1374 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1375 goto fail; 1376 } 1377 } 1378 1379 fail: 1380 if (krp) { 1381 kop->crk_status = krp->krp_status; 1382 for (i = 0; i < CRK_MAXPARAM; i++) { 1383 if (krp->krp_param[i].crp_p) 1384 free(krp->krp_param[i].crp_p, M_XDATA); 1385 } 1386 free(krp, M_XDATA); 1387 } 1388 return (error); 1389 } 1390 1391 static int 1392 cryptodev_find(struct crypt_find_op *find) 1393 { 1394 device_t dev; 1395 size_t fnlen = sizeof find->name; 1396 1397 if (find->crid != -1) { 1398 dev = crypto_find_device_byhid(find->crid); 1399 if (dev == NULL) 1400 return (ENOENT); 1401 strncpy(find->name, device_get_nameunit(dev), fnlen); 1402 find->name[fnlen - 1] = '\x0'; 1403 } else { 1404 find->name[fnlen - 1] = '\x0'; 1405 find->crid = crypto_find_driver(find->name); 1406 if (find->crid == -1) 1407 return (ENOENT); 1408 } 1409 return (0); 1410 } 1411 1412 /* ARGSUSED */ 1413 static int 1414 cryptof_stat( 1415 struct file *fp, 1416 struct stat *sb, 1417 struct ucred *active_cred, 1418 struct thread *td) 1419 { 1420 1421 return (EOPNOTSUPP); 1422 } 1423 1424 /* ARGSUSED */ 1425 static int 1426 cryptof_close(struct file *fp, struct thread *td) 1427 { 1428 struct fcrypt *fcr = fp->f_data; 1429 struct csession *cse; 1430 1431 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1432 TAILQ_REMOVE(&fcr->csessions, cse, next); 1433 KASSERT(cse->refs == 1, 1434 ("%s: crypto session %p with %d refs", __func__, cse, 1435 cse->refs)); 1436 csefree(cse); 1437 } 1438 free(fcr, M_XDATA); 1439 fp->f_data = NULL; 1440 return 0; 1441 } 1442 1443 static int 1444 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp) 1445 { 1446 1447 kif->kf_type = KF_TYPE_CRYPTO; 1448 return (0); 1449 } 1450 1451 static struct csession * 1452 csefind(struct fcrypt *fcr, u_int ses) 1453 { 1454 struct csession *cse; 1455 1456 mtx_lock(&fcr->lock); 1457 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1458 if (cse->ses == ses) { 1459 refcount_acquire(&cse->refs); 1460 mtx_unlock(&fcr->lock); 1461 return (cse); 1462 } 1463 } 1464 mtx_unlock(&fcr->lock); 1465 return (NULL); 1466 } 1467 1468 static bool 1469 csedelete(struct fcrypt *fcr, u_int ses) 1470 { 1471 struct csession *cse; 1472 1473 mtx_lock(&fcr->lock); 1474 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1475 if (cse->ses == ses) { 1476 TAILQ_REMOVE(&fcr->csessions, cse, next); 1477 mtx_unlock(&fcr->lock); 1478 csefree(cse); 1479 return (true); 1480 } 1481 } 1482 mtx_unlock(&fcr->lock); 1483 return (false); 1484 } 1485 1486 struct csession * 1487 csecreate(struct fcrypt *fcr, crypto_session_t cses, 1488 struct crypto_session_params *csp, struct enc_xform *txform, 1489 void *key, struct auth_hash *thash, void *mackey) 1490 { 1491 struct csession *cse; 1492 1493 cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT | M_ZERO); 1494 if (cse == NULL) 1495 return NULL; 1496 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 1497 refcount_init(&cse->refs, 1); 1498 cse->key = key; 1499 cse->mackey = mackey; 1500 cse->mode = csp->csp_mode; 1501 cse->cses = cses; 1502 cse->txform = txform; 1503 if (thash != NULL) 1504 cse->hashsize = thash->hashsize; 1505 else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) 1506 cse->hashsize = AES_GMAC_HASH_LEN; 1507 else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) 1508 cse->hashsize = AES_CBC_MAC_HASH_LEN; 1509 cse->ivsize = csp->csp_ivlen; 1510 mtx_lock(&fcr->lock); 1511 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 1512 cse->ses = fcr->sesn++; 1513 mtx_unlock(&fcr->lock); 1514 return (cse); 1515 } 1516 1517 static void 1518 csefree(struct csession *cse) 1519 { 1520 1521 if (!refcount_release(&cse->refs)) 1522 return; 1523 crypto_freesession(cse->cses); 1524 mtx_destroy(&cse->lock); 1525 if (cse->key) 1526 free(cse->key, M_XDATA); 1527 if (cse->mackey) 1528 free(cse->mackey, M_XDATA); 1529 free(cse, M_XDATA); 1530 } 1531 1532 static int 1533 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td) 1534 { 1535 struct file *f; 1536 struct fcrypt *fcr; 1537 int fd, error; 1538 1539 switch (cmd) { 1540 case CRIOGET: 1541 error = falloc_noinstall(td, &f); 1542 if (error) 1543 break; 1544 1545 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1546 TAILQ_INIT(&fcr->csessions); 1547 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1548 1549 finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops); 1550 error = finstall(td, f, &fd, 0, NULL); 1551 if (error) { 1552 mtx_destroy(&fcr->lock); 1553 free(fcr, M_XDATA); 1554 } else 1555 *(uint32_t *)data = fd; 1556 fdrop(f, td); 1557 break; 1558 case CRIOFINDDEV: 1559 error = cryptodev_find((struct crypt_find_op *)data); 1560 break; 1561 case CRIOASYMFEAT: 1562 error = crypto_getfeat((int *)data); 1563 break; 1564 default: 1565 error = EINVAL; 1566 break; 1567 } 1568 return (error); 1569 } 1570 1571 static struct cdevsw crypto_cdevsw = { 1572 .d_version = D_VERSION, 1573 .d_ioctl = cryptoioctl, 1574 .d_name = "crypto", 1575 }; 1576 static struct cdev *crypto_dev; 1577 1578 /* 1579 * Initialization code, both for static and dynamic loading. 1580 */ 1581 static int 1582 cryptodev_modevent(module_t mod, int type, void *unused) 1583 { 1584 switch (type) { 1585 case MOD_LOAD: 1586 if (bootverbose) 1587 printf("crypto: <crypto device>\n"); 1588 crypto_dev = make_dev(&crypto_cdevsw, 0, 1589 UID_ROOT, GID_WHEEL, 0666, 1590 "crypto"); 1591 return 0; 1592 case MOD_UNLOAD: 1593 /*XXX disallow if active sessions */ 1594 destroy_dev(crypto_dev); 1595 return 0; 1596 } 1597 return EINVAL; 1598 } 1599 1600 static moduledata_t cryptodev_mod = { 1601 "cryptodev", 1602 cryptodev_modevent, 1603 0 1604 }; 1605 MODULE_VERSION(cryptodev, 1); 1606 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1607 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1608 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1609