1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014-2021 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Portions of this software were developed by Ararat River 14 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. The name of the author may not be used to endorse or promote products 26 * derived from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * Effort sponsored in part by the Defense Advanced Research Projects 40 * Agency (DARPA) and Air Force Research Laboratory, Air Force 41 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 42 */ 43 44 #include <sys/cdefs.h> 45 __FBSDID("$FreeBSD$"); 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/sysctl.h> 55 #include <sys/errno.h> 56 #include <sys/random.h> 57 #include <sys/conf.h> 58 #include <sys/kernel.h> 59 #include <sys/module.h> 60 #include <sys/fcntl.h> 61 #include <sys/bus.h> 62 #include <sys/sdt.h> 63 #include <sys/syscallsubr.h> 64 65 #include <opencrypto/cryptodev.h> 66 #include <opencrypto/xform.h> 67 68 SDT_PROVIDER_DECLARE(opencrypto); 69 70 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 71 72 #ifdef COMPAT_FREEBSD12 73 /* 74 * Previously, most ioctls were performed against a cloned descriptor 75 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed 76 * against /dev/crypto directly. 77 */ 78 #define CRIOGET _IOWR('c', 100, uint32_t) 79 #endif 80 81 /* the following are done against the cloned descriptor */ 82 83 #ifdef COMPAT_FREEBSD32 84 #include <sys/mount.h> 85 #include <compat/freebsd32/freebsd32.h> 86 87 struct session_op32 { 88 uint32_t cipher; 89 uint32_t mac; 90 uint32_t keylen; 91 uint32_t key; 92 int mackeylen; 93 uint32_t mackey; 94 uint32_t ses; 95 }; 96 97 struct session2_op32 { 98 uint32_t cipher; 99 uint32_t mac; 100 uint32_t keylen; 101 uint32_t key; 102 int mackeylen; 103 uint32_t mackey; 104 uint32_t ses; 105 int crid; 106 int ivlen; 107 int maclen; 108 int pad[2]; 109 }; 110 111 struct crypt_op32 { 112 uint32_t ses; 113 uint16_t op; 114 uint16_t flags; 115 u_int len; 116 uint32_t src, dst; 117 uint32_t mac; 118 uint32_t iv; 119 }; 120 121 struct crypt_aead32 { 122 uint32_t ses; 123 uint16_t op; 124 uint16_t flags; 125 u_int len; 126 u_int aadlen; 127 u_int ivlen; 128 uint32_t src; 129 uint32_t dst; 130 uint32_t aad; 131 uint32_t tag; 132 uint32_t iv; 133 }; 134 135 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 136 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 137 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 138 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 139 140 static void 141 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 142 { 143 144 memset(to, 0, sizeof(*to)); 145 CP(*from, *to, cipher); 146 CP(*from, *to, mac); 147 CP(*from, *to, keylen); 148 PTRIN_CP(*from, *to, key); 149 CP(*from, *to, mackeylen); 150 PTRIN_CP(*from, *to, mackey); 151 CP(*from, *to, ses); 152 to->crid = CRYPTOCAP_F_HARDWARE; 153 } 154 155 static void 156 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 157 { 158 159 session_op_from_32((const struct session_op32 *)from, to); 160 CP(*from, *to, crid); 161 CP(*from, *to, ivlen); 162 CP(*from, *to, maclen); 163 } 164 165 static void 166 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 167 { 168 169 CP(*from, *to, cipher); 170 CP(*from, *to, mac); 171 CP(*from, *to, keylen); 172 PTROUT_CP(*from, *to, key); 173 CP(*from, *to, mackeylen); 174 PTROUT_CP(*from, *to, mackey); 175 CP(*from, *to, ses); 176 } 177 178 static void 179 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 180 { 181 182 session_op_to_32(from, (struct session_op32 *)to); 183 CP(*from, *to, crid); 184 } 185 186 static void 187 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 188 { 189 190 CP(*from, *to, ses); 191 CP(*from, *to, op); 192 CP(*from, *to, flags); 193 CP(*from, *to, len); 194 PTRIN_CP(*from, *to, src); 195 PTRIN_CP(*from, *to, dst); 196 PTRIN_CP(*from, *to, mac); 197 PTRIN_CP(*from, *to, iv); 198 } 199 200 static void 201 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 202 { 203 204 CP(*from, *to, ses); 205 CP(*from, *to, op); 206 CP(*from, *to, flags); 207 CP(*from, *to, len); 208 PTROUT_CP(*from, *to, src); 209 PTROUT_CP(*from, *to, dst); 210 PTROUT_CP(*from, *to, mac); 211 PTROUT_CP(*from, *to, iv); 212 } 213 214 static void 215 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 216 { 217 218 CP(*from, *to, ses); 219 CP(*from, *to, op); 220 CP(*from, *to, flags); 221 CP(*from, *to, len); 222 CP(*from, *to, aadlen); 223 CP(*from, *to, ivlen); 224 PTRIN_CP(*from, *to, src); 225 PTRIN_CP(*from, *to, dst); 226 PTRIN_CP(*from, *to, aad); 227 PTRIN_CP(*from, *to, tag); 228 PTRIN_CP(*from, *to, iv); 229 } 230 231 static void 232 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 233 { 234 235 CP(*from, *to, ses); 236 CP(*from, *to, op); 237 CP(*from, *to, flags); 238 CP(*from, *to, len); 239 CP(*from, *to, aadlen); 240 CP(*from, *to, ivlen); 241 PTROUT_CP(*from, *to, src); 242 PTROUT_CP(*from, *to, dst); 243 PTROUT_CP(*from, *to, aad); 244 PTROUT_CP(*from, *to, tag); 245 PTROUT_CP(*from, *to, iv); 246 } 247 #endif 248 249 static void 250 session2_op_from_op(const struct session_op *from, struct session2_op *to) 251 { 252 253 memset(to, 0, sizeof(*to)); 254 memcpy(to, from, sizeof(*from)); 255 to->crid = CRYPTOCAP_F_HARDWARE; 256 } 257 258 static void 259 session2_op_to_op(const struct session2_op *from, struct session_op *to) 260 { 261 262 memcpy(to, from, sizeof(*to)); 263 } 264 265 struct csession { 266 TAILQ_ENTRY(csession) next; 267 crypto_session_t cses; 268 volatile u_int refs; 269 uint32_t ses; 270 struct mtx lock; /* for op submission */ 271 272 u_int blocksize; 273 int hashsize; 274 int ivsize; 275 276 void *key; 277 void *mackey; 278 }; 279 280 struct cryptop_data { 281 struct csession *cse; 282 283 char *buf; 284 char *obuf; 285 char *aad; 286 bool done; 287 }; 288 289 struct fcrypt { 290 TAILQ_HEAD(csessionlist, csession) csessions; 291 int sesn; 292 struct mtx lock; 293 }; 294 295 static bool use_outputbuffers; 296 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 297 &use_outputbuffers, 0, 298 "Use separate output buffers for /dev/crypto requests."); 299 300 static bool use_separate_aad; 301 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 302 &use_separate_aad, 0, 303 "Use separate AAD buffer for /dev/crypto requests."); 304 305 static MALLOC_DEFINE(M_CRYPTODEV, "cryptodev", "/dev/crypto data buffers"); 306 307 /* 308 * Check a crypto identifier to see if it requested 309 * a software device/driver. This can be done either 310 * by device name/class or through search constraints. 311 */ 312 static int 313 checkforsoftware(int *cridp) 314 { 315 int crid; 316 317 crid = *cridp; 318 319 if (!crypto_devallowsoft) { 320 if (crid & CRYPTOCAP_F_SOFTWARE) { 321 if (crid & CRYPTOCAP_F_HARDWARE) { 322 *cridp = CRYPTOCAP_F_HARDWARE; 323 return 0; 324 } 325 return EINVAL; 326 } 327 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 328 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 329 return EINVAL; 330 } 331 return 0; 332 } 333 334 static int 335 cse_create(struct fcrypt *fcr, struct session2_op *sop) 336 { 337 struct crypto_session_params csp; 338 struct csession *cse; 339 const struct enc_xform *txform; 340 const struct auth_hash *thash; 341 void *key = NULL; 342 void *mackey = NULL; 343 crypto_session_t cses; 344 int crid, error, mac; 345 346 mac = sop->mac; 347 #ifdef COMPAT_FREEBSD12 348 switch (sop->mac) { 349 case CRYPTO_AES_128_NIST_GMAC: 350 case CRYPTO_AES_192_NIST_GMAC: 351 case CRYPTO_AES_256_NIST_GMAC: 352 /* Should always be paired with GCM. */ 353 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 354 CRYPTDEB("GMAC without GCM"); 355 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 356 return (EINVAL); 357 } 358 if (sop->keylen != sop->mackeylen) { 359 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 360 return (EINVAL); 361 } 362 mac = 0; 363 break; 364 case CRYPTO_AES_CCM_CBC_MAC: 365 /* Should always be paired with CCM. */ 366 if (sop->cipher != CRYPTO_AES_CCM_16) { 367 CRYPTDEB("CBC-MAC without CCM"); 368 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 369 return (EINVAL); 370 } 371 if (sop->keylen != sop->mackeylen) { 372 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 373 return (EINVAL); 374 } 375 mac = 0; 376 break; 377 } 378 #endif 379 380 memset(&csp, 0, sizeof(csp)); 381 if (use_outputbuffers) 382 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 383 if (mac != 0) { 384 csp.csp_auth_alg = mac; 385 csp.csp_auth_klen = sop->mackeylen; 386 } 387 if (sop->cipher != 0) { 388 csp.csp_cipher_alg = sop->cipher; 389 csp.csp_cipher_klen = sop->keylen; 390 } 391 thash = crypto_auth_hash(&csp); 392 txform = crypto_cipher(&csp); 393 394 if (txform != NULL && txform->macsize != 0) { 395 if (mac != 0) { 396 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 397 return (EINVAL); 398 } 399 csp.csp_mode = CSP_MODE_AEAD; 400 } else if (txform != NULL && thash != NULL) { 401 csp.csp_mode = CSP_MODE_ETA; 402 } else if (txform != NULL) { 403 csp.csp_mode = CSP_MODE_CIPHER; 404 } else if (thash != NULL) { 405 csp.csp_mode = CSP_MODE_DIGEST; 406 } else { 407 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 408 return (EINVAL); 409 } 410 411 switch (csp.csp_mode) { 412 case CSP_MODE_AEAD: 413 case CSP_MODE_ETA: 414 if (use_separate_aad) 415 csp.csp_flags |= CSP_F_SEPARATE_AAD; 416 break; 417 } 418 419 if (txform != NULL) { 420 if (sop->keylen > txform->maxkey || 421 sop->keylen < txform->minkey) { 422 CRYPTDEB("invalid cipher parameters"); 423 error = EINVAL; 424 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 425 goto bail; 426 } 427 428 key = malloc(csp.csp_cipher_klen, M_CRYPTODEV, M_WAITOK); 429 error = copyin(sop->key, key, csp.csp_cipher_klen); 430 if (error) { 431 CRYPTDEB("invalid key"); 432 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 433 goto bail; 434 } 435 csp.csp_cipher_key = key; 436 csp.csp_ivlen = txform->ivsize; 437 } 438 439 if (thash != NULL) { 440 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { 441 CRYPTDEB("invalid mac key length"); 442 error = EINVAL; 443 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 444 goto bail; 445 } 446 447 if (csp.csp_auth_klen != 0) { 448 mackey = malloc(csp.csp_auth_klen, M_CRYPTODEV, 449 M_WAITOK); 450 error = copyin(sop->mackey, mackey, csp.csp_auth_klen); 451 if (error) { 452 CRYPTDEB("invalid mac key"); 453 SDT_PROBE1(opencrypto, dev, ioctl, error, 454 __LINE__); 455 goto bail; 456 } 457 csp.csp_auth_key = mackey; 458 } 459 460 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 461 csp.csp_ivlen = AES_GCM_IV_LEN; 462 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 463 csp.csp_ivlen = AES_CCM_IV_LEN; 464 } 465 466 if (sop->ivlen != 0) { 467 if (csp.csp_ivlen == 0) { 468 CRYPTDEB("does not support an IV"); 469 error = EINVAL; 470 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 471 goto bail; 472 } 473 csp.csp_ivlen = sop->ivlen; 474 } 475 if (sop->maclen != 0) { 476 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) { 477 CRYPTDEB("does not support a MAC"); 478 error = EINVAL; 479 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 480 goto bail; 481 } 482 csp.csp_auth_mlen = sop->maclen; 483 } 484 485 crid = sop->crid; 486 error = checkforsoftware(&crid); 487 if (error) { 488 CRYPTDEB("checkforsoftware"); 489 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 490 goto bail; 491 } 492 error = crypto_newsession(&cses, &csp, crid); 493 if (error) { 494 CRYPTDEB("crypto_newsession"); 495 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 496 goto bail; 497 } 498 499 cse = malloc(sizeof(struct csession), M_CRYPTODEV, M_WAITOK | M_ZERO); 500 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 501 refcount_init(&cse->refs, 1); 502 cse->key = key; 503 cse->mackey = mackey; 504 cse->cses = cses; 505 if (sop->maclen != 0) 506 cse->hashsize = sop->maclen; 507 else if (thash != NULL) 508 cse->hashsize = thash->hashsize; 509 else if (csp.csp_mode == CSP_MODE_AEAD) 510 cse->hashsize = txform->macsize; 511 cse->ivsize = csp.csp_ivlen; 512 513 /* 514 * NB: This isn't necessarily the block size of the underlying 515 * MAC or cipher but is instead a restriction on valid input 516 * sizes. 517 */ 518 if (txform != NULL) 519 cse->blocksize = txform->blocksize; 520 else 521 cse->blocksize = 1; 522 523 mtx_lock(&fcr->lock); 524 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 525 cse->ses = fcr->sesn++; 526 mtx_unlock(&fcr->lock); 527 528 sop->ses = cse->ses; 529 530 /* return hardware/driver id */ 531 sop->crid = crypto_ses2hid(cse->cses); 532 bail: 533 if (error) { 534 free(key, M_CRYPTODEV); 535 free(mackey, M_CRYPTODEV); 536 } 537 return (error); 538 } 539 540 static struct csession * 541 cse_find(struct fcrypt *fcr, u_int ses) 542 { 543 struct csession *cse; 544 545 mtx_lock(&fcr->lock); 546 TAILQ_FOREACH(cse, &fcr->csessions, next) { 547 if (cse->ses == ses) { 548 refcount_acquire(&cse->refs); 549 mtx_unlock(&fcr->lock); 550 return (cse); 551 } 552 } 553 mtx_unlock(&fcr->lock); 554 return (NULL); 555 } 556 557 static void 558 cse_free(struct csession *cse) 559 { 560 561 if (!refcount_release(&cse->refs)) 562 return; 563 crypto_freesession(cse->cses); 564 mtx_destroy(&cse->lock); 565 if (cse->key) 566 free(cse->key, M_CRYPTODEV); 567 if (cse->mackey) 568 free(cse->mackey, M_CRYPTODEV); 569 free(cse, M_CRYPTODEV); 570 } 571 572 static bool 573 cse_delete(struct fcrypt *fcr, u_int ses) 574 { 575 struct csession *cse; 576 577 mtx_lock(&fcr->lock); 578 TAILQ_FOREACH(cse, &fcr->csessions, next) { 579 if (cse->ses == ses) { 580 TAILQ_REMOVE(&fcr->csessions, cse, next); 581 mtx_unlock(&fcr->lock); 582 cse_free(cse); 583 return (true); 584 } 585 } 586 mtx_unlock(&fcr->lock); 587 return (false); 588 } 589 590 static struct cryptop_data * 591 cod_alloc(struct csession *cse, size_t aad_len, size_t len) 592 { 593 struct cryptop_data *cod; 594 595 cod = malloc(sizeof(struct cryptop_data), M_CRYPTODEV, M_WAITOK | 596 M_ZERO); 597 598 cod->cse = cse; 599 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 600 if (aad_len != 0) 601 cod->aad = malloc(aad_len, M_CRYPTODEV, M_WAITOK); 602 cod->buf = malloc(len, M_CRYPTODEV, M_WAITOK); 603 } else 604 cod->buf = malloc(aad_len + len, M_CRYPTODEV, M_WAITOK); 605 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 606 cod->obuf = malloc(len, M_CRYPTODEV, M_WAITOK); 607 return (cod); 608 } 609 610 static void 611 cod_free(struct cryptop_data *cod) 612 { 613 614 free(cod->aad, M_CRYPTODEV); 615 free(cod->obuf, M_CRYPTODEV); 616 free(cod->buf, M_CRYPTODEV); 617 free(cod, M_CRYPTODEV); 618 } 619 620 static int 621 cryptodev_cb(struct cryptop *crp) 622 { 623 struct cryptop_data *cod = crp->crp_opaque; 624 625 /* 626 * Lock to ensure the wakeup() is not missed by the loops 627 * waiting on cod->done in cryptodev_op() and 628 * cryptodev_aead(). 629 */ 630 mtx_lock(&cod->cse->lock); 631 cod->done = true; 632 mtx_unlock(&cod->cse->lock); 633 wakeup(cod); 634 return (0); 635 } 636 637 static int 638 cryptodev_op(struct csession *cse, const struct crypt_op *cop) 639 { 640 const struct crypto_session_params *csp; 641 struct cryptop_data *cod = NULL; 642 struct cryptop *crp = NULL; 643 char *dst; 644 int error; 645 646 if (cop->len > 256*1024-4) { 647 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 648 return (E2BIG); 649 } 650 651 if ((cop->len % cse->blocksize) != 0) { 652 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 653 return (EINVAL); 654 } 655 656 if (cop->mac && cse->hashsize == 0) { 657 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 658 return (EINVAL); 659 } 660 661 /* 662 * The COP_F_CIPHER_FIRST flag predates explicit session 663 * modes, but the only way it was used was for EtA so allow it 664 * as long as it is consistent with EtA. 665 */ 666 if (cop->flags & COP_F_CIPHER_FIRST) { 667 if (cop->op != COP_ENCRYPT) { 668 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 669 return (EINVAL); 670 } 671 } 672 673 cod = cod_alloc(cse, 0, cop->len + cse->hashsize); 674 dst = cop->dst; 675 676 crp = crypto_getreq(cse->cses, M_WAITOK); 677 678 error = copyin(cop->src, cod->buf, cop->len); 679 if (error) { 680 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 681 goto bail; 682 } 683 crp->crp_payload_start = 0; 684 crp->crp_payload_length = cop->len; 685 if (cse->hashsize) 686 crp->crp_digest_start = cop->len; 687 688 csp = crypto_get_params(cse->cses); 689 switch (csp->csp_mode) { 690 case CSP_MODE_COMPRESS: 691 switch (cop->op) { 692 case COP_ENCRYPT: 693 crp->crp_op = CRYPTO_OP_COMPRESS; 694 break; 695 case COP_DECRYPT: 696 crp->crp_op = CRYPTO_OP_DECOMPRESS; 697 break; 698 default: 699 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 700 error = EINVAL; 701 goto bail; 702 } 703 break; 704 case CSP_MODE_CIPHER: 705 if (cop->len == 0 || 706 (cop->iv == NULL && cop->len == cse->ivsize)) { 707 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 708 error = EINVAL; 709 goto bail; 710 } 711 switch (cop->op) { 712 case COP_ENCRYPT: 713 crp->crp_op = CRYPTO_OP_ENCRYPT; 714 break; 715 case COP_DECRYPT: 716 crp->crp_op = CRYPTO_OP_DECRYPT; 717 break; 718 default: 719 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 720 error = EINVAL; 721 goto bail; 722 } 723 break; 724 case CSP_MODE_DIGEST: 725 switch (cop->op) { 726 case 0: 727 case COP_ENCRYPT: 728 case COP_DECRYPT: 729 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 730 if (cod->obuf != NULL) 731 crp->crp_digest_start = 0; 732 break; 733 default: 734 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 735 error = EINVAL; 736 goto bail; 737 } 738 break; 739 case CSP_MODE_AEAD: 740 if (cse->ivsize != 0 && cop->iv == NULL) { 741 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 742 error = EINVAL; 743 goto bail; 744 } 745 /* FALLTHROUGH */ 746 case CSP_MODE_ETA: 747 switch (cop->op) { 748 case COP_ENCRYPT: 749 crp->crp_op = CRYPTO_OP_ENCRYPT | 750 CRYPTO_OP_COMPUTE_DIGEST; 751 break; 752 case COP_DECRYPT: 753 crp->crp_op = CRYPTO_OP_DECRYPT | 754 CRYPTO_OP_VERIFY_DIGEST; 755 break; 756 default: 757 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 758 error = EINVAL; 759 goto bail; 760 } 761 break; 762 default: 763 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 764 error = EINVAL; 765 goto bail; 766 } 767 768 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 769 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 770 if (cod->obuf) 771 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 772 crp->crp_callback = cryptodev_cb; 773 crp->crp_opaque = cod; 774 775 if (cop->iv) { 776 if (cse->ivsize == 0) { 777 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 778 error = EINVAL; 779 goto bail; 780 } 781 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 782 if (error) { 783 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 784 goto bail; 785 } 786 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 787 } else if (cse->ivsize != 0) { 788 if (crp->crp_payload_length < cse->ivsize) { 789 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 790 error = EINVAL; 791 goto bail; 792 } 793 crp->crp_iv_start = 0; 794 crp->crp_payload_length -= cse->ivsize; 795 if (crp->crp_payload_length != 0) 796 crp->crp_payload_start = cse->ivsize; 797 dst += cse->ivsize; 798 } 799 800 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 801 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 802 cse->hashsize); 803 if (error) { 804 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 805 goto bail; 806 } 807 } 808 again: 809 /* 810 * Let the dispatch run unlocked, then, interlock against the 811 * callback before checking if the operation completed and going 812 * to sleep. This insures drivers don't inherit our lock which 813 * results in a lock order reversal between crypto_dispatch forced 814 * entry and the crypto_done callback into us. 815 */ 816 error = crypto_dispatch(crp); 817 if (error != 0) { 818 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 819 goto bail; 820 } 821 822 mtx_lock(&cse->lock); 823 while (!cod->done) 824 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 825 mtx_unlock(&cse->lock); 826 827 if (crp->crp_etype == EAGAIN) { 828 crp->crp_etype = 0; 829 crp->crp_flags &= ~CRYPTO_F_DONE; 830 cod->done = false; 831 goto again; 832 } 833 834 if (crp->crp_etype != 0) { 835 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 836 error = crp->crp_etype; 837 goto bail; 838 } 839 840 if (cop->dst != NULL) { 841 error = copyout(cod->obuf != NULL ? cod->obuf : 842 cod->buf + crp->crp_payload_start, dst, 843 crp->crp_payload_length); 844 if (error) { 845 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 846 goto bail; 847 } 848 } 849 850 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 851 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 852 crp->crp_digest_start, cop->mac, cse->hashsize); 853 if (error) { 854 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 855 goto bail; 856 } 857 } 858 859 bail: 860 crypto_freereq(crp); 861 cod_free(cod); 862 863 return (error); 864 } 865 866 static int 867 cryptodev_aead(struct csession *cse, struct crypt_aead *caead) 868 { 869 const struct crypto_session_params *csp; 870 struct cryptop_data *cod = NULL; 871 struct cryptop *crp = NULL; 872 char *dst; 873 int error; 874 875 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 876 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 877 return (E2BIG); 878 } 879 880 if ((caead->len % cse->blocksize) != 0) { 881 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 882 return (EINVAL); 883 } 884 885 if (cse->hashsize == 0 || caead->tag == NULL) { 886 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 887 return (EINVAL); 888 } 889 890 /* 891 * The COP_F_CIPHER_FIRST flag predates explicit session 892 * modes, but the only way it was used was for EtA so allow it 893 * as long as it is consistent with EtA. 894 */ 895 if (caead->flags & COP_F_CIPHER_FIRST) { 896 if (caead->op != COP_ENCRYPT) { 897 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 898 return (EINVAL); 899 } 900 } 901 902 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize); 903 dst = caead->dst; 904 905 crp = crypto_getreq(cse->cses, M_WAITOK); 906 907 if (cod->aad != NULL) 908 error = copyin(caead->aad, cod->aad, caead->aadlen); 909 else 910 error = copyin(caead->aad, cod->buf, caead->aadlen); 911 if (error) { 912 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 913 goto bail; 914 } 915 crp->crp_aad = cod->aad; 916 crp->crp_aad_start = 0; 917 crp->crp_aad_length = caead->aadlen; 918 919 if (cod->aad != NULL) 920 crp->crp_payload_start = 0; 921 else 922 crp->crp_payload_start = caead->aadlen; 923 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 924 caead->len); 925 if (error) { 926 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 927 goto bail; 928 } 929 crp->crp_payload_length = caead->len; 930 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 931 crp->crp_digest_start = crp->crp_payload_output_start + 932 caead->len; 933 else 934 crp->crp_digest_start = crp->crp_payload_start + caead->len; 935 936 csp = crypto_get_params(cse->cses); 937 switch (csp->csp_mode) { 938 case CSP_MODE_AEAD: 939 case CSP_MODE_ETA: 940 switch (caead->op) { 941 case COP_ENCRYPT: 942 crp->crp_op = CRYPTO_OP_ENCRYPT | 943 CRYPTO_OP_COMPUTE_DIGEST; 944 break; 945 case COP_DECRYPT: 946 crp->crp_op = CRYPTO_OP_DECRYPT | 947 CRYPTO_OP_VERIFY_DIGEST; 948 break; 949 default: 950 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 951 error = EINVAL; 952 goto bail; 953 } 954 break; 955 default: 956 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 957 error = EINVAL; 958 goto bail; 959 } 960 961 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 962 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 963 cse->hashsize); 964 if (cod->obuf != NULL) 965 crypto_use_output_buf(crp, cod->obuf, caead->len + 966 cse->hashsize); 967 crp->crp_callback = cryptodev_cb; 968 crp->crp_opaque = cod; 969 970 if (caead->iv) { 971 /* 972 * Permit a 16-byte IV for AES-XTS, but only use the 973 * first 8 bytes as a block number. 974 */ 975 if (csp->csp_mode == CSP_MODE_ETA && 976 csp->csp_cipher_alg == CRYPTO_AES_XTS && 977 caead->ivlen == AES_BLOCK_LEN) 978 caead->ivlen = AES_XTS_IV_LEN; 979 980 if (cse->ivsize == 0) { 981 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 982 error = EINVAL; 983 goto bail; 984 } 985 if (caead->ivlen != cse->ivsize) { 986 error = EINVAL; 987 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 988 goto bail; 989 } 990 991 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 992 if (error) { 993 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 994 goto bail; 995 } 996 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 997 } else { 998 error = EINVAL; 999 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1000 goto bail; 1001 } 1002 1003 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1004 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1005 cse->hashsize); 1006 if (error) { 1007 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1008 goto bail; 1009 } 1010 } 1011 again: 1012 /* 1013 * Let the dispatch run unlocked, then, interlock against the 1014 * callback before checking if the operation completed and going 1015 * to sleep. This insures drivers don't inherit our lock which 1016 * results in a lock order reversal between crypto_dispatch forced 1017 * entry and the crypto_done callback into us. 1018 */ 1019 error = crypto_dispatch(crp); 1020 if (error != 0) { 1021 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1022 goto bail; 1023 } 1024 1025 mtx_lock(&cse->lock); 1026 while (!cod->done) 1027 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1028 mtx_unlock(&cse->lock); 1029 1030 if (crp->crp_etype == EAGAIN) { 1031 crp->crp_etype = 0; 1032 crp->crp_flags &= ~CRYPTO_F_DONE; 1033 cod->done = false; 1034 goto again; 1035 } 1036 1037 if (crp->crp_etype != 0) { 1038 error = crp->crp_etype; 1039 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1040 goto bail; 1041 } 1042 1043 if (caead->dst != NULL) { 1044 error = copyout(cod->obuf != NULL ? cod->obuf : 1045 cod->buf + crp->crp_payload_start, dst, 1046 crp->crp_payload_length); 1047 if (error) { 1048 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1049 goto bail; 1050 } 1051 } 1052 1053 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1054 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1055 crp->crp_digest_start, caead->tag, cse->hashsize); 1056 if (error) { 1057 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1058 goto bail; 1059 } 1060 } 1061 1062 bail: 1063 crypto_freereq(crp); 1064 cod_free(cod); 1065 1066 return (error); 1067 } 1068 1069 static int 1070 cryptodev_find(struct crypt_find_op *find) 1071 { 1072 device_t dev; 1073 size_t fnlen = sizeof find->name; 1074 1075 if (find->crid != -1) { 1076 dev = crypto_find_device_byhid(find->crid); 1077 if (dev == NULL) 1078 return (ENOENT); 1079 strncpy(find->name, device_get_nameunit(dev), fnlen); 1080 find->name[fnlen - 1] = '\x0'; 1081 } else { 1082 find->name[fnlen - 1] = '\x0'; 1083 find->crid = crypto_find_driver(find->name); 1084 if (find->crid == -1) 1085 return (ENOENT); 1086 } 1087 return (0); 1088 } 1089 1090 static void 1091 fcrypt_dtor(void *data) 1092 { 1093 struct fcrypt *fcr = data; 1094 struct csession *cse; 1095 1096 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1097 TAILQ_REMOVE(&fcr->csessions, cse, next); 1098 KASSERT(refcount_load(&cse->refs) == 1, 1099 ("%s: crypto session %p with %d refs", __func__, cse, 1100 refcount_load(&cse->refs))); 1101 cse_free(cse); 1102 } 1103 mtx_destroy(&fcr->lock); 1104 free(fcr, M_CRYPTODEV); 1105 } 1106 1107 static int 1108 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1109 { 1110 struct fcrypt *fcr; 1111 int error; 1112 1113 fcr = malloc(sizeof(struct fcrypt), M_CRYPTODEV, M_WAITOK | M_ZERO); 1114 TAILQ_INIT(&fcr->csessions); 1115 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1116 error = devfs_set_cdevpriv(fcr, fcrypt_dtor); 1117 if (error) 1118 fcrypt_dtor(fcr); 1119 return (error); 1120 } 1121 1122 static int 1123 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, 1124 struct thread *td) 1125 { 1126 struct fcrypt *fcr; 1127 struct csession *cse; 1128 struct session2_op *sop; 1129 struct crypt_op *cop; 1130 struct crypt_aead *caead; 1131 uint32_t ses; 1132 int error = 0; 1133 union { 1134 struct session2_op sopc; 1135 #ifdef COMPAT_FREEBSD32 1136 struct crypt_op copc; 1137 struct crypt_aead aeadc; 1138 #endif 1139 } thunk; 1140 #ifdef COMPAT_FREEBSD32 1141 u_long cmd32; 1142 void *data32; 1143 1144 cmd32 = 0; 1145 data32 = NULL; 1146 switch (cmd) { 1147 case CIOCGSESSION32: 1148 cmd32 = cmd; 1149 data32 = data; 1150 cmd = CIOCGSESSION; 1151 data = (void *)&thunk.sopc; 1152 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 1153 break; 1154 case CIOCGSESSION232: 1155 cmd32 = cmd; 1156 data32 = data; 1157 cmd = CIOCGSESSION2; 1158 data = (void *)&thunk.sopc; 1159 session2_op_from_32((struct session2_op32 *)data32, 1160 &thunk.sopc); 1161 break; 1162 case CIOCCRYPT32: 1163 cmd32 = cmd; 1164 data32 = data; 1165 cmd = CIOCCRYPT; 1166 data = (void *)&thunk.copc; 1167 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 1168 break; 1169 case CIOCCRYPTAEAD32: 1170 cmd32 = cmd; 1171 data32 = data; 1172 cmd = CIOCCRYPTAEAD; 1173 data = (void *)&thunk.aeadc; 1174 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 1175 break; 1176 } 1177 #endif 1178 1179 devfs_get_cdevpriv((void **)&fcr); 1180 1181 switch (cmd) { 1182 #ifdef COMPAT_FREEBSD12 1183 case CRIOGET: 1184 /* 1185 * NB: This may fail in cases that the old 1186 * implementation did not if the current process has 1187 * restricted filesystem access (e.g. running in a 1188 * jail that does not expose /dev/crypto or in 1189 * capability mode). 1190 */ 1191 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE, 1192 O_RDWR, 0); 1193 if (error == 0) 1194 *(uint32_t *)data = td->td_retval[0]; 1195 break; 1196 #endif 1197 case CIOCGSESSION: 1198 case CIOCGSESSION2: 1199 if (cmd == CIOCGSESSION) { 1200 session2_op_from_op((void *)data, &thunk.sopc); 1201 sop = &thunk.sopc; 1202 } else 1203 sop = (struct session2_op *)data; 1204 1205 error = cse_create(fcr, sop); 1206 if (cmd == CIOCGSESSION && error == 0) 1207 session2_op_to_op(sop, (void *)data); 1208 break; 1209 case CIOCFSESSION: 1210 ses = *(uint32_t *)data; 1211 if (!cse_delete(fcr, ses)) { 1212 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1213 return (EINVAL); 1214 } 1215 break; 1216 case CIOCCRYPT: 1217 cop = (struct crypt_op *)data; 1218 cse = cse_find(fcr, cop->ses); 1219 if (cse == NULL) { 1220 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1221 return (EINVAL); 1222 } 1223 error = cryptodev_op(cse, cop); 1224 cse_free(cse); 1225 break; 1226 case CIOCFINDDEV: 1227 error = cryptodev_find((struct crypt_find_op *)data); 1228 break; 1229 case CIOCCRYPTAEAD: 1230 caead = (struct crypt_aead *)data; 1231 cse = cse_find(fcr, caead->ses); 1232 if (cse == NULL) { 1233 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1234 return (EINVAL); 1235 } 1236 error = cryptodev_aead(cse, caead); 1237 cse_free(cse); 1238 break; 1239 default: 1240 error = EINVAL; 1241 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1242 break; 1243 } 1244 1245 #ifdef COMPAT_FREEBSD32 1246 switch (cmd32) { 1247 case CIOCGSESSION32: 1248 if (error == 0) 1249 session_op_to_32((void *)data, data32); 1250 break; 1251 case CIOCGSESSION232: 1252 if (error == 0) 1253 session2_op_to_32((void *)data, data32); 1254 break; 1255 case CIOCCRYPT32: 1256 if (error == 0) 1257 crypt_op_to_32((void *)data, data32); 1258 break; 1259 case CIOCCRYPTAEAD32: 1260 if (error == 0) 1261 crypt_aead_to_32((void *)data, data32); 1262 break; 1263 } 1264 #endif 1265 return (error); 1266 } 1267 1268 static struct cdevsw crypto_cdevsw = { 1269 .d_version = D_VERSION, 1270 .d_open = crypto_open, 1271 .d_ioctl = crypto_ioctl, 1272 .d_name = "crypto", 1273 }; 1274 static struct cdev *crypto_dev; 1275 1276 /* 1277 * Initialization code, both for static and dynamic loading. 1278 */ 1279 static int 1280 cryptodev_modevent(module_t mod, int type, void *unused) 1281 { 1282 switch (type) { 1283 case MOD_LOAD: 1284 if (bootverbose) 1285 printf("crypto: <crypto device>\n"); 1286 crypto_dev = make_dev(&crypto_cdevsw, 0, 1287 UID_ROOT, GID_WHEEL, 0666, 1288 "crypto"); 1289 return 0; 1290 case MOD_UNLOAD: 1291 /*XXX disallow if active sessions */ 1292 destroy_dev(crypto_dev); 1293 return 0; 1294 } 1295 return EINVAL; 1296 } 1297 1298 static moduledata_t cryptodev_mod = { 1299 "cryptodev", 1300 cryptodev_modevent, 1301 0 1302 }; 1303 MODULE_VERSION(cryptodev, 1); 1304 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1305 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1306 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1307