1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 30 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 34 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Effort sponsored in part by the Defense Advanced Research Projects 37 * Agency (DARPA) and Air Force Research Laboratory, Air Force 38 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/sysctl.h> 51 #include <sys/file.h> 52 #include <sys/filedesc.h> 53 #include <sys/errno.h> 54 #include <sys/random.h> 55 #include <sys/conf.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/fcntl.h> 59 #include <sys/bus.h> 60 #include <sys/user.h> 61 #include <sys/sdt.h> 62 63 #include <opencrypto/cryptodev.h> 64 #include <opencrypto/xform.h> 65 66 SDT_PROVIDER_DECLARE(opencrypto); 67 68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 69 70 #ifdef COMPAT_FREEBSD32 71 #include <sys/mount.h> 72 #include <compat/freebsd32/freebsd32.h> 73 74 struct session_op32 { 75 u_int32_t cipher; 76 u_int32_t mac; 77 u_int32_t keylen; 78 u_int32_t key; 79 int mackeylen; 80 u_int32_t mackey; 81 u_int32_t ses; 82 }; 83 84 struct session2_op32 { 85 u_int32_t cipher; 86 u_int32_t mac; 87 u_int32_t keylen; 88 u_int32_t key; 89 int mackeylen; 90 u_int32_t mackey; 91 u_int32_t ses; 92 int crid; 93 int pad[4]; 94 }; 95 96 struct crypt_op32 { 97 u_int32_t ses; 98 u_int16_t op; 99 u_int16_t flags; 100 u_int len; 101 u_int32_t src, dst; 102 u_int32_t mac; 103 u_int32_t iv; 104 }; 105 106 struct crparam32 { 107 u_int32_t crp_p; 108 u_int crp_nbits; 109 }; 110 111 struct crypt_kop32 { 112 u_int crk_op; 113 u_int crk_status; 114 u_short crk_iparams; 115 u_short crk_oparams; 116 u_int crk_crid; 117 struct crparam32 crk_param[CRK_MAXPARAM]; 118 }; 119 120 struct cryptotstat32 { 121 struct timespec32 acc; 122 struct timespec32 min; 123 struct timespec32 max; 124 u_int32_t count; 125 }; 126 127 struct cryptostats32 { 128 u_int32_t cs_ops; 129 u_int32_t cs_errs; 130 u_int32_t cs_kops; 131 u_int32_t cs_kerrs; 132 u_int32_t cs_intrs; 133 u_int32_t cs_rets; 134 u_int32_t cs_blocks; 135 u_int32_t cs_kblocks; 136 struct cryptotstat32 cs_invoke; 137 struct cryptotstat32 cs_done; 138 struct cryptotstat32 cs_cb; 139 struct cryptotstat32 cs_finis; 140 }; 141 142 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 143 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 144 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32) 145 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 146 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32) 147 148 static void 149 session_op_from_32(const struct session_op32 *from, struct session_op *to) 150 { 151 152 CP(*from, *to, cipher); 153 CP(*from, *to, mac); 154 CP(*from, *to, keylen); 155 PTRIN_CP(*from, *to, key); 156 CP(*from, *to, mackeylen); 157 PTRIN_CP(*from, *to, mackey); 158 CP(*from, *to, ses); 159 } 160 161 static void 162 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 163 { 164 165 session_op_from_32((const struct session_op32 *)from, 166 (struct session_op *)to); 167 CP(*from, *to, crid); 168 } 169 170 static void 171 session_op_to_32(const struct session_op *from, struct session_op32 *to) 172 { 173 174 CP(*from, *to, cipher); 175 CP(*from, *to, mac); 176 CP(*from, *to, keylen); 177 PTROUT_CP(*from, *to, key); 178 CP(*from, *to, mackeylen); 179 PTROUT_CP(*from, *to, mackey); 180 CP(*from, *to, ses); 181 } 182 183 static void 184 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 185 { 186 187 session_op_to_32((const struct session_op *)from, 188 (struct session_op32 *)to); 189 CP(*from, *to, crid); 190 } 191 192 static void 193 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 194 { 195 196 CP(*from, *to, ses); 197 CP(*from, *to, op); 198 CP(*from, *to, flags); 199 CP(*from, *to, len); 200 PTRIN_CP(*from, *to, src); 201 PTRIN_CP(*from, *to, dst); 202 PTRIN_CP(*from, *to, mac); 203 PTRIN_CP(*from, *to, iv); 204 } 205 206 static void 207 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 208 { 209 210 CP(*from, *to, ses); 211 CP(*from, *to, op); 212 CP(*from, *to, flags); 213 CP(*from, *to, len); 214 PTROUT_CP(*from, *to, src); 215 PTROUT_CP(*from, *to, dst); 216 PTROUT_CP(*from, *to, mac); 217 PTROUT_CP(*from, *to, iv); 218 } 219 220 static void 221 crparam_from_32(const struct crparam32 *from, struct crparam *to) 222 { 223 224 PTRIN_CP(*from, *to, crp_p); 225 CP(*from, *to, crp_nbits); 226 } 227 228 static void 229 crparam_to_32(const struct crparam *from, struct crparam32 *to) 230 { 231 232 PTROUT_CP(*from, *to, crp_p); 233 CP(*from, *to, crp_nbits); 234 } 235 236 static void 237 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to) 238 { 239 int i; 240 241 CP(*from, *to, crk_op); 242 CP(*from, *to, crk_status); 243 CP(*from, *to, crk_iparams); 244 CP(*from, *to, crk_oparams); 245 CP(*from, *to, crk_crid); 246 for (i = 0; i < CRK_MAXPARAM; i++) 247 crparam_from_32(&from->crk_param[i], &to->crk_param[i]); 248 } 249 250 static void 251 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to) 252 { 253 int i; 254 255 CP(*from, *to, crk_op); 256 CP(*from, *to, crk_status); 257 CP(*from, *to, crk_iparams); 258 CP(*from, *to, crk_oparams); 259 CP(*from, *to, crk_crid); 260 for (i = 0; i < CRK_MAXPARAM; i++) 261 crparam_to_32(&from->crk_param[i], &to->crk_param[i]); 262 } 263 #endif 264 265 struct csession { 266 TAILQ_ENTRY(csession) next; 267 crypto_session_t cses; 268 volatile u_int refs; 269 u_int32_t ses; 270 struct mtx lock; /* for op submission */ 271 272 struct enc_xform *txform; 273 int hashsize; 274 int ivsize; 275 int mode; 276 277 void *key; 278 void *mackey; 279 }; 280 281 struct cryptop_data { 282 struct csession *cse; 283 284 char *buf; 285 char *obuf; 286 char *aad; 287 bool done; 288 }; 289 290 struct fcrypt { 291 TAILQ_HEAD(csessionlist, csession) csessions; 292 int sesn; 293 struct mtx lock; 294 }; 295 296 static bool use_outputbuffers; 297 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 298 &use_outputbuffers, 0, 299 "Use separate output buffers for /dev/crypto requests."); 300 301 static bool use_separate_aad; 302 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 303 &use_separate_aad, 0, 304 "Use separate AAD buffer for /dev/crypto requests."); 305 306 static int cryptof_ioctl(struct file *, u_long, void *, 307 struct ucred *, struct thread *); 308 static int cryptof_stat(struct file *, struct stat *, 309 struct ucred *, struct thread *); 310 static int cryptof_close(struct file *, struct thread *); 311 static int cryptof_fill_kinfo(struct file *, struct kinfo_file *, 312 struct filedesc *); 313 314 static struct fileops cryptofops = { 315 .fo_read = invfo_rdwr, 316 .fo_write = invfo_rdwr, 317 .fo_truncate = invfo_truncate, 318 .fo_ioctl = cryptof_ioctl, 319 .fo_poll = invfo_poll, 320 .fo_kqfilter = invfo_kqfilter, 321 .fo_stat = cryptof_stat, 322 .fo_close = cryptof_close, 323 .fo_chmod = invfo_chmod, 324 .fo_chown = invfo_chown, 325 .fo_sendfile = invfo_sendfile, 326 .fo_fill_kinfo = cryptof_fill_kinfo, 327 }; 328 329 static struct csession *csefind(struct fcrypt *, u_int); 330 static bool csedelete(struct fcrypt *, u_int); 331 static struct csession *csecreate(struct fcrypt *, crypto_session_t, 332 struct crypto_session_params *, struct enc_xform *, void *, 333 struct auth_hash *, void *); 334 static void csefree(struct csession *); 335 336 static int cryptodev_op(struct csession *, struct crypt_op *, 337 struct ucred *, struct thread *td); 338 static int cryptodev_aead(struct csession *, struct crypt_aead *, 339 struct ucred *, struct thread *); 340 static int cryptodev_key(struct crypt_kop *); 341 static int cryptodev_find(struct crypt_find_op *); 342 343 /* 344 * Check a crypto identifier to see if it requested 345 * a software device/driver. This can be done either 346 * by device name/class or through search constraints. 347 */ 348 static int 349 checkforsoftware(int *cridp) 350 { 351 int crid; 352 353 crid = *cridp; 354 355 if (!crypto_devallowsoft) { 356 if (crid & CRYPTOCAP_F_SOFTWARE) { 357 if (crid & CRYPTOCAP_F_HARDWARE) { 358 *cridp = CRYPTOCAP_F_HARDWARE; 359 return 0; 360 } 361 return EINVAL; 362 } 363 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 364 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 365 return EINVAL; 366 } 367 return 0; 368 } 369 370 /* ARGSUSED */ 371 static int 372 cryptof_ioctl( 373 struct file *fp, 374 u_long cmd, 375 void *data, 376 struct ucred *active_cred, 377 struct thread *td) 378 { 379 #define SES2(p) ((struct session2_op *)p) 380 struct crypto_session_params csp; 381 struct fcrypt *fcr = fp->f_data; 382 struct csession *cse; 383 struct session_op *sop; 384 struct crypt_op *cop; 385 struct crypt_aead *caead; 386 struct enc_xform *txform = NULL; 387 struct auth_hash *thash = NULL; 388 void *key = NULL; 389 void *mackey = NULL; 390 struct crypt_kop *kop; 391 crypto_session_t cses; 392 u_int32_t ses; 393 int error = 0, crid; 394 #ifdef COMPAT_FREEBSD32 395 struct session2_op sopc; 396 struct crypt_op copc; 397 struct crypt_kop kopc; 398 #endif 399 400 switch (cmd) { 401 case CIOCGSESSION: 402 case CIOCGSESSION2: 403 #ifdef COMPAT_FREEBSD32 404 case CIOCGSESSION32: 405 case CIOCGSESSION232: 406 if (cmd == CIOCGSESSION32) { 407 session_op_from_32(data, (struct session_op *)&sopc); 408 sop = (struct session_op *)&sopc; 409 } else if (cmd == CIOCGSESSION232) { 410 session2_op_from_32(data, &sopc); 411 sop = (struct session_op *)&sopc; 412 } else 413 #endif 414 sop = (struct session_op *)data; 415 switch (sop->cipher) { 416 case 0: 417 break; 418 case CRYPTO_AES_CBC: 419 txform = &enc_xform_rijndael128; 420 break; 421 case CRYPTO_AES_XTS: 422 txform = &enc_xform_aes_xts; 423 break; 424 case CRYPTO_NULL_CBC: 425 txform = &enc_xform_null; 426 break; 427 case CRYPTO_CAMELLIA_CBC: 428 txform = &enc_xform_camellia; 429 break; 430 case CRYPTO_AES_ICM: 431 txform = &enc_xform_aes_icm; 432 break; 433 case CRYPTO_AES_NIST_GCM_16: 434 txform = &enc_xform_aes_nist_gcm; 435 break; 436 case CRYPTO_CHACHA20: 437 txform = &enc_xform_chacha20; 438 break; 439 case CRYPTO_AES_CCM_16: 440 txform = &enc_xform_ccm; 441 break; 442 443 default: 444 CRYPTDEB("invalid cipher"); 445 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 446 return (EINVAL); 447 } 448 449 switch (sop->mac) { 450 case 0: 451 break; 452 case CRYPTO_POLY1305: 453 thash = &auth_hash_poly1305; 454 break; 455 case CRYPTO_SHA1_HMAC: 456 thash = &auth_hash_hmac_sha1; 457 break; 458 case CRYPTO_SHA2_224_HMAC: 459 thash = &auth_hash_hmac_sha2_224; 460 break; 461 case CRYPTO_SHA2_256_HMAC: 462 thash = &auth_hash_hmac_sha2_256; 463 break; 464 case CRYPTO_SHA2_384_HMAC: 465 thash = &auth_hash_hmac_sha2_384; 466 break; 467 case CRYPTO_SHA2_512_HMAC: 468 thash = &auth_hash_hmac_sha2_512; 469 break; 470 case CRYPTO_RIPEMD160_HMAC: 471 thash = &auth_hash_hmac_ripemd_160; 472 break; 473 #ifdef COMPAT_FREEBSD12 474 case CRYPTO_AES_128_NIST_GMAC: 475 case CRYPTO_AES_192_NIST_GMAC: 476 case CRYPTO_AES_256_NIST_GMAC: 477 /* Should always be paired with GCM. */ 478 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 479 CRYPTDEB("GMAC without GCM"); 480 SDT_PROBE1(opencrypto, dev, ioctl, error, 481 __LINE__); 482 return (EINVAL); 483 } 484 break; 485 #endif 486 case CRYPTO_AES_NIST_GMAC: 487 switch (sop->mackeylen * 8) { 488 case 128: 489 thash = &auth_hash_nist_gmac_aes_128; 490 break; 491 case 192: 492 thash = &auth_hash_nist_gmac_aes_192; 493 break; 494 case 256: 495 thash = &auth_hash_nist_gmac_aes_256; 496 break; 497 default: 498 CRYPTDEB("invalid GMAC key length"); 499 SDT_PROBE1(opencrypto, dev, ioctl, error, 500 __LINE__); 501 return (EINVAL); 502 } 503 break; 504 case CRYPTO_AES_CCM_CBC_MAC: 505 switch (sop->mackeylen) { 506 case 16: 507 thash = &auth_hash_ccm_cbc_mac_128; 508 break; 509 case 24: 510 thash = &auth_hash_ccm_cbc_mac_192; 511 break; 512 case 32: 513 thash = &auth_hash_ccm_cbc_mac_256; 514 break; 515 default: 516 CRYPTDEB("Invalid CBC MAC key size %d", 517 sop->keylen); 518 SDT_PROBE1(opencrypto, dev, ioctl, 519 error, __LINE__); 520 return (EINVAL); 521 } 522 break; 523 case CRYPTO_SHA1: 524 thash = &auth_hash_sha1; 525 break; 526 case CRYPTO_SHA2_224: 527 thash = &auth_hash_sha2_224; 528 break; 529 case CRYPTO_SHA2_256: 530 thash = &auth_hash_sha2_256; 531 break; 532 case CRYPTO_SHA2_384: 533 thash = &auth_hash_sha2_384; 534 break; 535 case CRYPTO_SHA2_512: 536 thash = &auth_hash_sha2_512; 537 break; 538 539 case CRYPTO_NULL_HMAC: 540 thash = &auth_hash_null; 541 break; 542 543 case CRYPTO_BLAKE2B: 544 thash = &auth_hash_blake2b; 545 break; 546 case CRYPTO_BLAKE2S: 547 thash = &auth_hash_blake2s; 548 break; 549 550 default: 551 CRYPTDEB("invalid mac"); 552 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 553 return (EINVAL); 554 } 555 556 if (txform == NULL && thash == NULL) { 557 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 558 return (EINVAL); 559 } 560 561 memset(&csp, 0, sizeof(csp)); 562 if (use_outputbuffers) 563 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 564 565 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { 566 switch (sop->mac) { 567 #ifdef COMPAT_FREEBSD12 568 case CRYPTO_AES_128_NIST_GMAC: 569 case CRYPTO_AES_192_NIST_GMAC: 570 case CRYPTO_AES_256_NIST_GMAC: 571 if (sop->keylen != sop->mackeylen) { 572 SDT_PROBE1(opencrypto, dev, ioctl, 573 error, __LINE__); 574 return (EINVAL); 575 } 576 break; 577 #endif 578 case 0: 579 break; 580 default: 581 SDT_PROBE1(opencrypto, dev, ioctl, error, 582 __LINE__); 583 return (EINVAL); 584 } 585 csp.csp_mode = CSP_MODE_AEAD; 586 } else if (sop->cipher == CRYPTO_AES_CCM_16) { 587 switch (sop->mac) { 588 #ifdef COMPAT_FREEBSD12 589 case CRYPTO_AES_CCM_CBC_MAC: 590 if (sop->keylen != sop->mackeylen) { 591 SDT_PROBE1(opencrypto, dev, ioctl, 592 error, __LINE__); 593 return (EINVAL); 594 } 595 thash = NULL; 596 break; 597 #endif 598 case 0: 599 break; 600 default: 601 SDT_PROBE1(opencrypto, dev, ioctl, error, 602 __LINE__); 603 return (EINVAL); 604 } 605 csp.csp_mode = CSP_MODE_AEAD; 606 } else if (txform && thash) 607 csp.csp_mode = CSP_MODE_ETA; 608 else if (txform) 609 csp.csp_mode = CSP_MODE_CIPHER; 610 else 611 csp.csp_mode = CSP_MODE_DIGEST; 612 613 switch (csp.csp_mode) { 614 case CSP_MODE_AEAD: 615 case CSP_MODE_ETA: 616 if (use_separate_aad) 617 csp.csp_flags |= CSP_F_SEPARATE_AAD; 618 break; 619 } 620 621 if (txform) { 622 csp.csp_cipher_alg = txform->type; 623 csp.csp_cipher_klen = sop->keylen; 624 if (sop->keylen > txform->maxkey || 625 sop->keylen < txform->minkey) { 626 CRYPTDEB("invalid cipher parameters"); 627 error = EINVAL; 628 SDT_PROBE1(opencrypto, dev, ioctl, error, 629 __LINE__); 630 goto bail; 631 } 632 633 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 634 error = copyin(sop->key, key, csp.csp_cipher_klen); 635 if (error) { 636 CRYPTDEB("invalid key"); 637 SDT_PROBE1(opencrypto, dev, ioctl, error, 638 __LINE__); 639 goto bail; 640 } 641 csp.csp_cipher_key = key; 642 csp.csp_ivlen = txform->ivsize; 643 } 644 645 if (thash) { 646 csp.csp_auth_alg = thash->type; 647 csp.csp_auth_klen = sop->mackeylen; 648 if (sop->mackeylen > thash->keysize || 649 sop->mackeylen < 0) { 650 CRYPTDEB("invalid mac key length"); 651 error = EINVAL; 652 SDT_PROBE1(opencrypto, dev, ioctl, error, 653 __LINE__); 654 goto bail; 655 } 656 657 if (csp.csp_auth_klen) { 658 mackey = malloc(csp.csp_auth_klen, M_XDATA, 659 M_WAITOK); 660 error = copyin(sop->mackey, mackey, 661 csp.csp_auth_klen); 662 if (error) { 663 CRYPTDEB("invalid mac key"); 664 SDT_PROBE1(opencrypto, dev, ioctl, 665 error, __LINE__); 666 goto bail; 667 } 668 csp.csp_auth_key = mackey; 669 } 670 671 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 672 csp.csp_ivlen = AES_GCM_IV_LEN; 673 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 674 csp.csp_ivlen = AES_CCM_IV_LEN; 675 } 676 677 /* NB: CIOCGSESSION2 has the crid */ 678 if (cmd == CIOCGSESSION2 679 #ifdef COMPAT_FREEBSD32 680 || cmd == CIOCGSESSION232 681 #endif 682 ) { 683 crid = SES2(sop)->crid; 684 error = checkforsoftware(&crid); 685 if (error) { 686 CRYPTDEB("checkforsoftware"); 687 SDT_PROBE1(opencrypto, dev, ioctl, error, 688 __LINE__); 689 goto bail; 690 } 691 } else 692 crid = CRYPTOCAP_F_HARDWARE; 693 error = crypto_newsession(&cses, &csp, crid); 694 if (error) { 695 CRYPTDEB("crypto_newsession"); 696 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 697 goto bail; 698 } 699 700 cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey); 701 702 if (cse == NULL) { 703 crypto_freesession(cses); 704 error = EINVAL; 705 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 706 CRYPTDEB("csecreate"); 707 goto bail; 708 } 709 sop->ses = cse->ses; 710 if (cmd == CIOCGSESSION2 711 #ifdef COMPAT_FREEBSD32 712 || cmd == CIOCGSESSION232 713 #endif 714 ) { 715 /* return hardware/driver id */ 716 SES2(sop)->crid = crypto_ses2hid(cse->cses); 717 } 718 bail: 719 if (error) { 720 free(key, M_XDATA); 721 free(mackey, M_XDATA); 722 } 723 #ifdef COMPAT_FREEBSD32 724 else { 725 if (cmd == CIOCGSESSION32) 726 session_op_to_32(sop, data); 727 else if (cmd == CIOCGSESSION232) 728 session2_op_to_32((struct session2_op *)sop, 729 data); 730 } 731 #endif 732 break; 733 case CIOCFSESSION: 734 ses = *(u_int32_t *)data; 735 if (!csedelete(fcr, ses)) { 736 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 737 return (EINVAL); 738 } 739 break; 740 case CIOCCRYPT: 741 #ifdef COMPAT_FREEBSD32 742 case CIOCCRYPT32: 743 if (cmd == CIOCCRYPT32) { 744 cop = &copc; 745 crypt_op_from_32(data, cop); 746 } else 747 #endif 748 cop = (struct crypt_op *)data; 749 cse = csefind(fcr, cop->ses); 750 if (cse == NULL) { 751 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 752 return (EINVAL); 753 } 754 error = cryptodev_op(cse, cop, active_cred, td); 755 csefree(cse); 756 #ifdef COMPAT_FREEBSD32 757 if (error == 0 && cmd == CIOCCRYPT32) 758 crypt_op_to_32(cop, data); 759 #endif 760 break; 761 case CIOCKEY: 762 case CIOCKEY2: 763 #ifdef COMPAT_FREEBSD32 764 case CIOCKEY32: 765 case CIOCKEY232: 766 #endif 767 if (!crypto_userasymcrypto) { 768 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 769 return (EPERM); /* XXX compat? */ 770 } 771 #ifdef COMPAT_FREEBSD32 772 if (cmd == CIOCKEY32 || cmd == CIOCKEY232) { 773 kop = &kopc; 774 crypt_kop_from_32(data, kop); 775 } else 776 #endif 777 kop = (struct crypt_kop *)data; 778 if (cmd == CIOCKEY 779 #ifdef COMPAT_FREEBSD32 780 || cmd == CIOCKEY32 781 #endif 782 ) { 783 /* NB: crypto core enforces s/w driver use */ 784 kop->crk_crid = 785 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE; 786 } 787 mtx_lock(&Giant); 788 error = cryptodev_key(kop); 789 mtx_unlock(&Giant); 790 #ifdef COMPAT_FREEBSD32 791 if (cmd == CIOCKEY32 || cmd == CIOCKEY232) 792 crypt_kop_to_32(kop, data); 793 #endif 794 break; 795 case CIOCASYMFEAT: 796 if (!crypto_userasymcrypto) { 797 /* 798 * NB: if user asym crypto operations are 799 * not permitted return "no algorithms" 800 * so well-behaved applications will just 801 * fallback to doing them in software. 802 */ 803 *(int *)data = 0; 804 } else { 805 error = crypto_getfeat((int *)data); 806 if (error) 807 SDT_PROBE1(opencrypto, dev, ioctl, error, 808 __LINE__); 809 } 810 break; 811 case CIOCFINDDEV: 812 error = cryptodev_find((struct crypt_find_op *)data); 813 break; 814 case CIOCCRYPTAEAD: 815 caead = (struct crypt_aead *)data; 816 cse = csefind(fcr, caead->ses); 817 if (cse == NULL) { 818 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 819 return (EINVAL); 820 } 821 error = cryptodev_aead(cse, caead, active_cred, td); 822 csefree(cse); 823 break; 824 default: 825 error = EINVAL; 826 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 827 break; 828 } 829 return (error); 830 #undef SES2 831 } 832 833 static int cryptodev_cb(struct cryptop *); 834 835 static struct cryptop_data * 836 cod_alloc(struct csession *cse, size_t aad_len, size_t len, struct thread *td) 837 { 838 struct cryptop_data *cod; 839 840 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 841 842 cod->cse = cse; 843 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 844 if (aad_len != 0) 845 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 846 cod->buf = malloc(len, M_XDATA, M_WAITOK); 847 } else 848 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 849 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 850 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 851 return (cod); 852 } 853 854 static void 855 cod_free(struct cryptop_data *cod) 856 { 857 858 free(cod->aad, M_XDATA); 859 free(cod->obuf, M_XDATA); 860 free(cod->buf, M_XDATA); 861 free(cod, M_XDATA); 862 } 863 864 static int 865 cryptodev_op( 866 struct csession *cse, 867 struct crypt_op *cop, 868 struct ucred *active_cred, 869 struct thread *td) 870 { 871 struct cryptop_data *cod = NULL; 872 struct cryptop *crp = NULL; 873 int error; 874 875 if (cop->len > 256*1024-4) { 876 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 877 return (E2BIG); 878 } 879 880 if (cse->txform) { 881 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { 882 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 883 return (EINVAL); 884 } 885 } 886 887 if (cop->mac && cse->hashsize == 0) { 888 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 889 return (EINVAL); 890 } 891 892 /* 893 * The COP_F_CIPHER_FIRST flag predates explicit session 894 * modes, but the only way it was used was for EtA so allow it 895 * as long as it is consistent with EtA. 896 */ 897 if (cop->flags & COP_F_CIPHER_FIRST) { 898 if (cop->op != COP_ENCRYPT) { 899 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 900 return (EINVAL); 901 } 902 } 903 904 cod = cod_alloc(cse, 0, cop->len + cse->hashsize, td); 905 906 crp = crypto_getreq(cse->cses, M_WAITOK); 907 908 error = copyin(cop->src, cod->buf, cop->len); 909 if (error) { 910 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 911 goto bail; 912 } 913 crp->crp_payload_start = 0; 914 crp->crp_payload_length = cop->len; 915 if (cse->hashsize) 916 crp->crp_digest_start = cop->len; 917 918 switch (cse->mode) { 919 case CSP_MODE_COMPRESS: 920 switch (cop->op) { 921 case COP_ENCRYPT: 922 crp->crp_op = CRYPTO_OP_COMPRESS; 923 break; 924 case COP_DECRYPT: 925 crp->crp_op = CRYPTO_OP_DECOMPRESS; 926 break; 927 default: 928 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 929 error = EINVAL; 930 goto bail; 931 } 932 break; 933 case CSP_MODE_CIPHER: 934 switch (cop->op) { 935 case COP_ENCRYPT: 936 crp->crp_op = CRYPTO_OP_ENCRYPT; 937 break; 938 case COP_DECRYPT: 939 crp->crp_op = CRYPTO_OP_DECRYPT; 940 break; 941 default: 942 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 943 error = EINVAL; 944 goto bail; 945 } 946 break; 947 case CSP_MODE_DIGEST: 948 switch (cop->op) { 949 case 0: 950 case COP_ENCRYPT: 951 case COP_DECRYPT: 952 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 953 if (cod->obuf != NULL) 954 crp->crp_digest_start = 0; 955 break; 956 default: 957 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 958 error = EINVAL; 959 goto bail; 960 } 961 break; 962 case CSP_MODE_ETA: 963 switch (cop->op) { 964 case COP_ENCRYPT: 965 crp->crp_op = CRYPTO_OP_ENCRYPT | 966 CRYPTO_OP_COMPUTE_DIGEST; 967 break; 968 case COP_DECRYPT: 969 crp->crp_op = CRYPTO_OP_DECRYPT | 970 CRYPTO_OP_VERIFY_DIGEST; 971 break; 972 default: 973 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 974 error = EINVAL; 975 goto bail; 976 } 977 break; 978 default: 979 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 980 error = EINVAL; 981 goto bail; 982 } 983 984 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 985 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 986 if (cod->obuf) 987 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 988 crp->crp_callback = cryptodev_cb; 989 crp->crp_opaque = cod; 990 991 if (cop->iv) { 992 if (cse->ivsize == 0) { 993 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 994 error = EINVAL; 995 goto bail; 996 } 997 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 998 if (error) { 999 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1000 goto bail; 1001 } 1002 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1003 } else if (cse->ivsize != 0) { 1004 crp->crp_iv_start = 0; 1005 crp->crp_payload_start += cse->ivsize; 1006 crp->crp_payload_length -= cse->ivsize; 1007 cop->dst += cse->ivsize; 1008 } 1009 1010 if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1011 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 1012 cse->hashsize); 1013 if (error) { 1014 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1015 goto bail; 1016 } 1017 } 1018 again: 1019 /* 1020 * Let the dispatch run unlocked, then, interlock against the 1021 * callback before checking if the operation completed and going 1022 * to sleep. This insures drivers don't inherit our lock which 1023 * results in a lock order reversal between crypto_dispatch forced 1024 * entry and the crypto_done callback into us. 1025 */ 1026 error = crypto_dispatch(crp); 1027 if (error != 0) { 1028 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1029 goto bail; 1030 } 1031 1032 mtx_lock(&cse->lock); 1033 while (!cod->done) 1034 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1035 mtx_unlock(&cse->lock); 1036 1037 if (crp->crp_etype == EAGAIN) { 1038 crp->crp_etype = 0; 1039 crp->crp_flags &= ~CRYPTO_F_DONE; 1040 cod->done = false; 1041 goto again; 1042 } 1043 1044 if (crp->crp_etype != 0) { 1045 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1046 error = crp->crp_etype; 1047 goto bail; 1048 } 1049 1050 if (cop->dst != NULL) { 1051 error = copyout(cod->obuf != NULL ? cod->obuf : 1052 cod->buf + crp->crp_payload_start, cop->dst, 1053 crp->crp_payload_length); 1054 if (error) { 1055 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1056 goto bail; 1057 } 1058 } 1059 1060 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1061 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1062 crp->crp_digest_start, cop->mac, cse->hashsize); 1063 if (error) { 1064 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1065 goto bail; 1066 } 1067 } 1068 1069 bail: 1070 crypto_freereq(crp); 1071 cod_free(cod); 1072 1073 return (error); 1074 } 1075 1076 static int 1077 cryptodev_aead( 1078 struct csession *cse, 1079 struct crypt_aead *caead, 1080 struct ucred *active_cred, 1081 struct thread *td) 1082 { 1083 struct cryptop_data *cod = NULL; 1084 struct cryptop *crp = NULL; 1085 int error; 1086 1087 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 1088 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1089 return (E2BIG); 1090 } 1091 1092 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || 1093 (caead->len % cse->txform->blocksize) != 0) { 1094 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1095 return (EINVAL); 1096 } 1097 1098 /* 1099 * The COP_F_CIPHER_FIRST flag predates explicit session 1100 * modes, but the only way it was used was for EtA so allow it 1101 * as long as it is consistent with EtA. 1102 */ 1103 if (caead->flags & COP_F_CIPHER_FIRST) { 1104 if (caead->op != COP_ENCRYPT) { 1105 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1106 return (EINVAL); 1107 } 1108 } 1109 1110 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize, td); 1111 1112 crp = crypto_getreq(cse->cses, M_WAITOK); 1113 1114 if (cod->aad != NULL) 1115 error = copyin(caead->aad, cod->aad, caead->aadlen); 1116 else 1117 error = copyin(caead->aad, cod->buf, caead->aadlen); 1118 if (error) { 1119 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1120 goto bail; 1121 } 1122 crp->crp_aad = cod->aad; 1123 crp->crp_aad_start = 0; 1124 crp->crp_aad_length = caead->aadlen; 1125 1126 if (cod->aad != NULL) 1127 crp->crp_payload_start = 0; 1128 else 1129 crp->crp_payload_start = caead->aadlen; 1130 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 1131 caead->len); 1132 if (error) { 1133 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1134 goto bail; 1135 } 1136 crp->crp_payload_length = caead->len; 1137 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 1138 crp->crp_digest_start = crp->crp_payload_output_start + 1139 caead->len; 1140 else 1141 crp->crp_digest_start = crp->crp_payload_start + caead->len; 1142 1143 switch (cse->mode) { 1144 case CSP_MODE_AEAD: 1145 case CSP_MODE_ETA: 1146 switch (caead->op) { 1147 case COP_ENCRYPT: 1148 crp->crp_op = CRYPTO_OP_ENCRYPT | 1149 CRYPTO_OP_COMPUTE_DIGEST; 1150 break; 1151 case COP_DECRYPT: 1152 crp->crp_op = CRYPTO_OP_DECRYPT | 1153 CRYPTO_OP_VERIFY_DIGEST; 1154 break; 1155 default: 1156 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1157 error = EINVAL; 1158 goto bail; 1159 } 1160 break; 1161 default: 1162 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1163 error = EINVAL; 1164 goto bail; 1165 } 1166 1167 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 1168 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 1169 cse->hashsize); 1170 if (cod->obuf != NULL) 1171 crypto_use_output_buf(crp, cod->obuf, caead->len + 1172 cse->hashsize); 1173 crp->crp_callback = cryptodev_cb; 1174 crp->crp_opaque = cod; 1175 1176 if (caead->iv) { 1177 /* 1178 * Permit a 16-byte IV for AES-XTS, but only use the 1179 * first 8 bytes as a block number. 1180 */ 1181 if (cse->mode == CSP_MODE_ETA && 1182 caead->ivlen == AES_BLOCK_LEN && 1183 cse->ivsize == AES_XTS_IV_LEN) 1184 caead->ivlen = AES_XTS_IV_LEN; 1185 1186 if (caead->ivlen != cse->ivsize) { 1187 error = EINVAL; 1188 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1189 goto bail; 1190 } 1191 1192 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 1193 if (error) { 1194 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1195 goto bail; 1196 } 1197 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1198 } else { 1199 crp->crp_iv_start = crp->crp_payload_start; 1200 crp->crp_payload_start += cse->ivsize; 1201 crp->crp_payload_length -= cse->ivsize; 1202 caead->dst += cse->ivsize; 1203 } 1204 1205 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1206 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1207 cse->hashsize); 1208 if (error) { 1209 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1210 goto bail; 1211 } 1212 } 1213 again: 1214 /* 1215 * Let the dispatch run unlocked, then, interlock against the 1216 * callback before checking if the operation completed and going 1217 * to sleep. This insures drivers don't inherit our lock which 1218 * results in a lock order reversal between crypto_dispatch forced 1219 * entry and the crypto_done callback into us. 1220 */ 1221 error = crypto_dispatch(crp); 1222 if (error != 0) { 1223 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1224 goto bail; 1225 } 1226 1227 mtx_lock(&cse->lock); 1228 while (!cod->done) 1229 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1230 mtx_unlock(&cse->lock); 1231 1232 if (crp->crp_etype == EAGAIN) { 1233 crp->crp_etype = 0; 1234 crp->crp_flags &= ~CRYPTO_F_DONE; 1235 cod->done = false; 1236 goto again; 1237 } 1238 1239 if (crp->crp_etype != 0) { 1240 error = crp->crp_etype; 1241 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1242 goto bail; 1243 } 1244 1245 if (caead->dst != NULL) { 1246 error = copyout(cod->obuf != NULL ? cod->obuf : 1247 cod->buf + crp->crp_payload_start, caead->dst, 1248 crp->crp_payload_length); 1249 if (error) { 1250 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1251 goto bail; 1252 } 1253 } 1254 1255 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1256 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1257 crp->crp_digest_start, caead->tag, cse->hashsize); 1258 if (error) { 1259 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1260 goto bail; 1261 } 1262 } 1263 1264 bail: 1265 crypto_freereq(crp); 1266 cod_free(cod); 1267 1268 return (error); 1269 } 1270 1271 static int 1272 cryptodev_cb(struct cryptop *crp) 1273 { 1274 struct cryptop_data *cod = crp->crp_opaque; 1275 1276 /* 1277 * Lock to ensure the wakeup() is not missed by the loops 1278 * waiting on cod->done in cryptodev_op() and 1279 * cryptodev_aead(). 1280 */ 1281 mtx_lock(&cod->cse->lock); 1282 cod->done = true; 1283 mtx_unlock(&cod->cse->lock); 1284 wakeup(cod); 1285 return (0); 1286 } 1287 1288 static void 1289 cryptodevkey_cb(struct cryptkop *krp) 1290 { 1291 1292 wakeup_one(krp); 1293 } 1294 1295 static int 1296 cryptodev_key(struct crypt_kop *kop) 1297 { 1298 struct cryptkop *krp = NULL; 1299 int error = EINVAL; 1300 int in, out, size, i; 1301 1302 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) { 1303 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1304 return (EFBIG); 1305 } 1306 1307 in = kop->crk_iparams; 1308 out = kop->crk_oparams; 1309 switch (kop->crk_op) { 1310 case CRK_MOD_EXP: 1311 if (in == 3 && out == 1) 1312 break; 1313 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1314 return (EINVAL); 1315 case CRK_MOD_EXP_CRT: 1316 if (in == 6 && out == 1) 1317 break; 1318 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1319 return (EINVAL); 1320 case CRK_DSA_SIGN: 1321 if (in == 5 && out == 2) 1322 break; 1323 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1324 return (EINVAL); 1325 case CRK_DSA_VERIFY: 1326 if (in == 7 && out == 0) 1327 break; 1328 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1329 return (EINVAL); 1330 case CRK_DH_COMPUTE_KEY: 1331 if (in == 3 && out == 1) 1332 break; 1333 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1334 return (EINVAL); 1335 default: 1336 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1337 return (EINVAL); 1338 } 1339 1340 krp = (struct cryptkop *)malloc(sizeof *krp, M_XDATA, M_WAITOK|M_ZERO); 1341 if (!krp) { 1342 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1343 return (ENOMEM); 1344 } 1345 krp->krp_op = kop->crk_op; 1346 krp->krp_status = kop->crk_status; 1347 krp->krp_iparams = kop->crk_iparams; 1348 krp->krp_oparams = kop->crk_oparams; 1349 krp->krp_crid = kop->crk_crid; 1350 krp->krp_status = 0; 1351 krp->krp_callback = cryptodevkey_cb; 1352 1353 for (i = 0; i < CRK_MAXPARAM; i++) { 1354 if (kop->crk_param[i].crp_nbits > 65536) { 1355 /* Limit is the same as in OpenBSD */ 1356 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1357 goto fail; 1358 } 1359 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 1360 } 1361 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1362 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1363 if (size == 0) 1364 continue; 1365 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK); 1366 if (i >= krp->krp_iparams) 1367 continue; 1368 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size); 1369 if (error) { 1370 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1371 goto fail; 1372 } 1373 } 1374 1375 error = crypto_kdispatch(krp); 1376 if (error) { 1377 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1378 goto fail; 1379 } 1380 error = tsleep(krp, PSOCK, "crydev", 0); 1381 if (error) { 1382 /* XXX can this happen? if so, how do we recover? */ 1383 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1384 goto fail; 1385 } 1386 1387 kop->crk_crid = krp->krp_hid; /* device that did the work */ 1388 if (krp->krp_status != 0) { 1389 error = krp->krp_status; 1390 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1391 goto fail; 1392 } 1393 1394 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) { 1395 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1396 if (size == 0) 1397 continue; 1398 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size); 1399 if (error) { 1400 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1401 goto fail; 1402 } 1403 } 1404 1405 fail: 1406 if (krp) { 1407 kop->crk_status = krp->krp_status; 1408 for (i = 0; i < CRK_MAXPARAM; i++) { 1409 if (krp->krp_param[i].crp_p) 1410 free(krp->krp_param[i].crp_p, M_XDATA); 1411 } 1412 free(krp, M_XDATA); 1413 } 1414 return (error); 1415 } 1416 1417 static int 1418 cryptodev_find(struct crypt_find_op *find) 1419 { 1420 device_t dev; 1421 size_t fnlen = sizeof find->name; 1422 1423 if (find->crid != -1) { 1424 dev = crypto_find_device_byhid(find->crid); 1425 if (dev == NULL) 1426 return (ENOENT); 1427 strncpy(find->name, device_get_nameunit(dev), fnlen); 1428 find->name[fnlen - 1] = '\x0'; 1429 } else { 1430 find->name[fnlen - 1] = '\x0'; 1431 find->crid = crypto_find_driver(find->name); 1432 if (find->crid == -1) 1433 return (ENOENT); 1434 } 1435 return (0); 1436 } 1437 1438 /* ARGSUSED */ 1439 static int 1440 cryptof_stat( 1441 struct file *fp, 1442 struct stat *sb, 1443 struct ucred *active_cred, 1444 struct thread *td) 1445 { 1446 1447 return (EOPNOTSUPP); 1448 } 1449 1450 /* ARGSUSED */ 1451 static int 1452 cryptof_close(struct file *fp, struct thread *td) 1453 { 1454 struct fcrypt *fcr = fp->f_data; 1455 struct csession *cse; 1456 1457 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1458 TAILQ_REMOVE(&fcr->csessions, cse, next); 1459 KASSERT(cse->refs == 1, 1460 ("%s: crypto session %p with %d refs", __func__, cse, 1461 cse->refs)); 1462 csefree(cse); 1463 } 1464 free(fcr, M_XDATA); 1465 fp->f_data = NULL; 1466 return 0; 1467 } 1468 1469 static int 1470 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp) 1471 { 1472 1473 kif->kf_type = KF_TYPE_CRYPTO; 1474 return (0); 1475 } 1476 1477 static struct csession * 1478 csefind(struct fcrypt *fcr, u_int ses) 1479 { 1480 struct csession *cse; 1481 1482 mtx_lock(&fcr->lock); 1483 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1484 if (cse->ses == ses) { 1485 refcount_acquire(&cse->refs); 1486 mtx_unlock(&fcr->lock); 1487 return (cse); 1488 } 1489 } 1490 mtx_unlock(&fcr->lock); 1491 return (NULL); 1492 } 1493 1494 static bool 1495 csedelete(struct fcrypt *fcr, u_int ses) 1496 { 1497 struct csession *cse; 1498 1499 mtx_lock(&fcr->lock); 1500 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1501 if (cse->ses == ses) { 1502 TAILQ_REMOVE(&fcr->csessions, cse, next); 1503 mtx_unlock(&fcr->lock); 1504 csefree(cse); 1505 return (true); 1506 } 1507 } 1508 mtx_unlock(&fcr->lock); 1509 return (false); 1510 } 1511 1512 struct csession * 1513 csecreate(struct fcrypt *fcr, crypto_session_t cses, 1514 struct crypto_session_params *csp, struct enc_xform *txform, 1515 void *key, struct auth_hash *thash, void *mackey) 1516 { 1517 struct csession *cse; 1518 1519 cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT | M_ZERO); 1520 if (cse == NULL) 1521 return NULL; 1522 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 1523 refcount_init(&cse->refs, 1); 1524 cse->key = key; 1525 cse->mackey = mackey; 1526 cse->mode = csp->csp_mode; 1527 cse->cses = cses; 1528 cse->txform = txform; 1529 if (thash != NULL) 1530 cse->hashsize = thash->hashsize; 1531 else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) 1532 cse->hashsize = AES_GMAC_HASH_LEN; 1533 else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) 1534 cse->hashsize = AES_CBC_MAC_HASH_LEN; 1535 cse->ivsize = csp->csp_ivlen; 1536 mtx_lock(&fcr->lock); 1537 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 1538 cse->ses = fcr->sesn++; 1539 mtx_unlock(&fcr->lock); 1540 return (cse); 1541 } 1542 1543 static void 1544 csefree(struct csession *cse) 1545 { 1546 1547 if (!refcount_release(&cse->refs)) 1548 return; 1549 crypto_freesession(cse->cses); 1550 mtx_destroy(&cse->lock); 1551 if (cse->key) 1552 free(cse->key, M_XDATA); 1553 if (cse->mackey) 1554 free(cse->mackey, M_XDATA); 1555 free(cse, M_XDATA); 1556 } 1557 1558 static int 1559 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td) 1560 { 1561 struct file *f; 1562 struct fcrypt *fcr; 1563 int fd, error; 1564 1565 switch (cmd) { 1566 case CRIOGET: 1567 error = falloc_noinstall(td, &f); 1568 if (error) 1569 break; 1570 1571 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1572 TAILQ_INIT(&fcr->csessions); 1573 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1574 1575 finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops); 1576 error = finstall(td, f, &fd, 0, NULL); 1577 if (error) { 1578 mtx_destroy(&fcr->lock); 1579 free(fcr, M_XDATA); 1580 } else 1581 *(uint32_t *)data = fd; 1582 fdrop(f, td); 1583 break; 1584 case CRIOFINDDEV: 1585 error = cryptodev_find((struct crypt_find_op *)data); 1586 break; 1587 case CRIOASYMFEAT: 1588 error = crypto_getfeat((int *)data); 1589 break; 1590 default: 1591 error = EINVAL; 1592 break; 1593 } 1594 return (error); 1595 } 1596 1597 static struct cdevsw crypto_cdevsw = { 1598 .d_version = D_VERSION, 1599 .d_ioctl = cryptoioctl, 1600 .d_name = "crypto", 1601 }; 1602 static struct cdev *crypto_dev; 1603 1604 /* 1605 * Initialization code, both for static and dynamic loading. 1606 */ 1607 static int 1608 cryptodev_modevent(module_t mod, int type, void *unused) 1609 { 1610 switch (type) { 1611 case MOD_LOAD: 1612 if (bootverbose) 1613 printf("crypto: <crypto device>\n"); 1614 crypto_dev = make_dev(&crypto_cdevsw, 0, 1615 UID_ROOT, GID_WHEEL, 0666, 1616 "crypto"); 1617 return 0; 1618 case MOD_UNLOAD: 1619 /*XXX disallow if active sessions */ 1620 destroy_dev(crypto_dev); 1621 return 0; 1622 } 1623 return EINVAL; 1624 } 1625 1626 static moduledata_t cryptodev_mod = { 1627 "cryptodev", 1628 cryptodev_modevent, 1629 0 1630 }; 1631 MODULE_VERSION(cryptodev, 1); 1632 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1633 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1634 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1635