1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014-2021 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Portions of this software were developed by Ararat River 14 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. The name of the author may not be used to endorse or promote products 26 * derived from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * Effort sponsored in part by the Defense Advanced Research Projects 40 * Agency (DARPA) and Air Force Research Laboratory, Air Force 41 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 42 */ 43 44 #include <sys/cdefs.h> 45 __FBSDID("$FreeBSD$"); 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/sysctl.h> 55 #include <sys/errno.h> 56 #include <sys/random.h> 57 #include <sys/conf.h> 58 #include <sys/kernel.h> 59 #include <sys/module.h> 60 #include <sys/fcntl.h> 61 #include <sys/bus.h> 62 #include <sys/sdt.h> 63 #include <sys/syscallsubr.h> 64 65 #include <opencrypto/cryptodev.h> 66 #include <opencrypto/xform.h> 67 68 SDT_PROVIDER_DECLARE(opencrypto); 69 70 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 71 72 #ifdef COMPAT_FREEBSD12 73 /* 74 * Previously, most ioctls were performed against a cloned descriptor 75 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed 76 * against /dev/crypto directly. 77 */ 78 #define CRIOGET _IOWR('c', 100, uint32_t) 79 #endif 80 81 /* the following are done against the cloned descriptor */ 82 83 #ifdef COMPAT_FREEBSD32 84 #include <sys/mount.h> 85 #include <compat/freebsd32/freebsd32.h> 86 87 struct session_op32 { 88 uint32_t cipher; 89 uint32_t mac; 90 uint32_t keylen; 91 uint32_t key; 92 int mackeylen; 93 uint32_t mackey; 94 uint32_t ses; 95 }; 96 97 struct session2_op32 { 98 uint32_t cipher; 99 uint32_t mac; 100 uint32_t keylen; 101 uint32_t key; 102 int mackeylen; 103 uint32_t mackey; 104 uint32_t ses; 105 int crid; 106 int ivlen; 107 int maclen; 108 int pad[2]; 109 }; 110 111 struct crypt_op32 { 112 uint32_t ses; 113 uint16_t op; 114 uint16_t flags; 115 u_int len; 116 uint32_t src, dst; 117 uint32_t mac; 118 uint32_t iv; 119 }; 120 121 struct crypt_aead32 { 122 uint32_t ses; 123 uint16_t op; 124 uint16_t flags; 125 u_int len; 126 u_int aadlen; 127 u_int ivlen; 128 uint32_t src; 129 uint32_t dst; 130 uint32_t aad; 131 uint32_t tag; 132 uint32_t iv; 133 }; 134 135 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 136 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 137 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 138 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 139 140 static void 141 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 142 { 143 144 memset(to, 0, sizeof(*to)); 145 CP(*from, *to, cipher); 146 CP(*from, *to, mac); 147 CP(*from, *to, keylen); 148 PTRIN_CP(*from, *to, key); 149 CP(*from, *to, mackeylen); 150 PTRIN_CP(*from, *to, mackey); 151 CP(*from, *to, ses); 152 to->crid = CRYPTOCAP_F_HARDWARE; 153 } 154 155 static void 156 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 157 { 158 159 session_op_from_32((const struct session_op32 *)from, to); 160 CP(*from, *to, crid); 161 CP(*from, *to, ivlen); 162 CP(*from, *to, maclen); 163 } 164 165 static void 166 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 167 { 168 169 CP(*from, *to, cipher); 170 CP(*from, *to, mac); 171 CP(*from, *to, keylen); 172 PTROUT_CP(*from, *to, key); 173 CP(*from, *to, mackeylen); 174 PTROUT_CP(*from, *to, mackey); 175 CP(*from, *to, ses); 176 } 177 178 static void 179 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 180 { 181 182 session_op_to_32(from, (struct session_op32 *)to); 183 CP(*from, *to, crid); 184 } 185 186 static void 187 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 188 { 189 190 CP(*from, *to, ses); 191 CP(*from, *to, op); 192 CP(*from, *to, flags); 193 CP(*from, *to, len); 194 PTRIN_CP(*from, *to, src); 195 PTRIN_CP(*from, *to, dst); 196 PTRIN_CP(*from, *to, mac); 197 PTRIN_CP(*from, *to, iv); 198 } 199 200 static void 201 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 202 { 203 204 CP(*from, *to, ses); 205 CP(*from, *to, op); 206 CP(*from, *to, flags); 207 CP(*from, *to, len); 208 PTROUT_CP(*from, *to, src); 209 PTROUT_CP(*from, *to, dst); 210 PTROUT_CP(*from, *to, mac); 211 PTROUT_CP(*from, *to, iv); 212 } 213 214 static void 215 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 216 { 217 218 CP(*from, *to, ses); 219 CP(*from, *to, op); 220 CP(*from, *to, flags); 221 CP(*from, *to, len); 222 CP(*from, *to, aadlen); 223 CP(*from, *to, ivlen); 224 PTRIN_CP(*from, *to, src); 225 PTRIN_CP(*from, *to, dst); 226 PTRIN_CP(*from, *to, aad); 227 PTRIN_CP(*from, *to, tag); 228 PTRIN_CP(*from, *to, iv); 229 } 230 231 static void 232 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 233 { 234 235 CP(*from, *to, ses); 236 CP(*from, *to, op); 237 CP(*from, *to, flags); 238 CP(*from, *to, len); 239 CP(*from, *to, aadlen); 240 CP(*from, *to, ivlen); 241 PTROUT_CP(*from, *to, src); 242 PTROUT_CP(*from, *to, dst); 243 PTROUT_CP(*from, *to, aad); 244 PTROUT_CP(*from, *to, tag); 245 PTROUT_CP(*from, *to, iv); 246 } 247 #endif 248 249 static void 250 session2_op_from_op(const struct session_op *from, struct session2_op *to) 251 { 252 253 memset(to, 0, sizeof(*to)); 254 memcpy(to, from, sizeof(*from)); 255 to->crid = CRYPTOCAP_F_HARDWARE; 256 } 257 258 static void 259 session2_op_to_op(const struct session2_op *from, struct session_op *to) 260 { 261 262 memcpy(to, from, sizeof(*to)); 263 } 264 265 struct csession { 266 TAILQ_ENTRY(csession) next; 267 crypto_session_t cses; 268 volatile u_int refs; 269 uint32_t ses; 270 struct mtx lock; /* for op submission */ 271 272 u_int blocksize; 273 int hashsize; 274 int ivsize; 275 276 void *key; 277 void *mackey; 278 }; 279 280 struct cryptop_data { 281 struct csession *cse; 282 283 char *buf; 284 char *obuf; 285 char *aad; 286 bool done; 287 }; 288 289 struct fcrypt { 290 TAILQ_HEAD(csessionlist, csession) csessions; 291 int sesn; 292 struct mtx lock; 293 }; 294 295 static bool use_outputbuffers; 296 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 297 &use_outputbuffers, 0, 298 "Use separate output buffers for /dev/crypto requests."); 299 300 static bool use_separate_aad; 301 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 302 &use_separate_aad, 0, 303 "Use separate AAD buffer for /dev/crypto requests."); 304 305 /* 306 * Check a crypto identifier to see if it requested 307 * a software device/driver. This can be done either 308 * by device name/class or through search constraints. 309 */ 310 static int 311 checkforsoftware(int *cridp) 312 { 313 int crid; 314 315 crid = *cridp; 316 317 if (!crypto_devallowsoft) { 318 if (crid & CRYPTOCAP_F_SOFTWARE) { 319 if (crid & CRYPTOCAP_F_HARDWARE) { 320 *cridp = CRYPTOCAP_F_HARDWARE; 321 return 0; 322 } 323 return EINVAL; 324 } 325 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 326 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 327 return EINVAL; 328 } 329 return 0; 330 } 331 332 static int 333 cse_create(struct fcrypt *fcr, struct session2_op *sop) 334 { 335 struct crypto_session_params csp; 336 struct csession *cse; 337 const struct enc_xform *txform; 338 const struct auth_hash *thash; 339 void *key = NULL; 340 void *mackey = NULL; 341 crypto_session_t cses; 342 int crid, error, mac; 343 344 mac = sop->mac; 345 #ifdef COMPAT_FREEBSD12 346 switch (sop->mac) { 347 case CRYPTO_AES_128_NIST_GMAC: 348 case CRYPTO_AES_192_NIST_GMAC: 349 case CRYPTO_AES_256_NIST_GMAC: 350 /* Should always be paired with GCM. */ 351 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 352 CRYPTDEB("GMAC without GCM"); 353 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 354 return (EINVAL); 355 } 356 if (sop->keylen != sop->mackeylen) { 357 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 358 return (EINVAL); 359 } 360 mac = 0; 361 break; 362 case CRYPTO_AES_CCM_CBC_MAC: 363 /* Should always be paired with CCM. */ 364 if (sop->cipher != CRYPTO_AES_CCM_16) { 365 CRYPTDEB("CBC-MAC without CCM"); 366 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 367 return (EINVAL); 368 } 369 if (sop->keylen != sop->mackeylen) { 370 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 371 return (EINVAL); 372 } 373 mac = 0; 374 break; 375 } 376 #endif 377 378 memset(&csp, 0, sizeof(csp)); 379 if (use_outputbuffers) 380 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 381 if (mac != 0) { 382 csp.csp_auth_alg = mac; 383 csp.csp_auth_klen = sop->mackeylen; 384 } 385 if (sop->cipher != 0) { 386 csp.csp_cipher_alg = sop->cipher; 387 csp.csp_cipher_klen = sop->keylen; 388 } 389 thash = crypto_auth_hash(&csp); 390 txform = crypto_cipher(&csp); 391 392 if (txform != NULL && txform->macsize != 0) { 393 if (mac != 0) { 394 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 395 return (EINVAL); 396 } 397 csp.csp_mode = CSP_MODE_AEAD; 398 } else if (txform != NULL && thash != NULL) { 399 csp.csp_mode = CSP_MODE_ETA; 400 } else if (txform != NULL) { 401 csp.csp_mode = CSP_MODE_CIPHER; 402 } else if (thash != NULL) { 403 csp.csp_mode = CSP_MODE_DIGEST; 404 } else { 405 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 406 return (EINVAL); 407 } 408 409 switch (csp.csp_mode) { 410 case CSP_MODE_AEAD: 411 case CSP_MODE_ETA: 412 if (use_separate_aad) 413 csp.csp_flags |= CSP_F_SEPARATE_AAD; 414 break; 415 } 416 417 if (txform != NULL) { 418 if (sop->keylen > txform->maxkey || 419 sop->keylen < txform->minkey) { 420 CRYPTDEB("invalid cipher parameters"); 421 error = EINVAL; 422 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 423 goto bail; 424 } 425 426 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 427 error = copyin(sop->key, key, csp.csp_cipher_klen); 428 if (error) { 429 CRYPTDEB("invalid key"); 430 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 431 goto bail; 432 } 433 csp.csp_cipher_key = key; 434 csp.csp_ivlen = txform->ivsize; 435 } 436 437 if (thash != NULL) { 438 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { 439 CRYPTDEB("invalid mac key length"); 440 error = EINVAL; 441 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 442 goto bail; 443 } 444 445 if (csp.csp_auth_klen != 0) { 446 mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK); 447 error = copyin(sop->mackey, mackey, csp.csp_auth_klen); 448 if (error) { 449 CRYPTDEB("invalid mac key"); 450 SDT_PROBE1(opencrypto, dev, ioctl, error, 451 __LINE__); 452 goto bail; 453 } 454 csp.csp_auth_key = mackey; 455 } 456 457 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 458 csp.csp_ivlen = AES_GCM_IV_LEN; 459 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 460 csp.csp_ivlen = AES_CCM_IV_LEN; 461 } 462 463 if (sop->ivlen != 0) { 464 if (csp.csp_ivlen == 0) { 465 CRYPTDEB("does not support an IV"); 466 error = EINVAL; 467 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 468 goto bail; 469 } 470 csp.csp_ivlen = sop->ivlen; 471 } 472 if (sop->maclen != 0) { 473 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) { 474 CRYPTDEB("does not support a MAC"); 475 error = EINVAL; 476 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 477 goto bail; 478 } 479 csp.csp_auth_mlen = sop->maclen; 480 } 481 482 crid = sop->crid; 483 error = checkforsoftware(&crid); 484 if (error) { 485 CRYPTDEB("checkforsoftware"); 486 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 487 goto bail; 488 } 489 error = crypto_newsession(&cses, &csp, crid); 490 if (error) { 491 CRYPTDEB("crypto_newsession"); 492 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 493 goto bail; 494 } 495 496 cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO); 497 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 498 refcount_init(&cse->refs, 1); 499 cse->key = key; 500 cse->mackey = mackey; 501 cse->cses = cses; 502 if (sop->maclen != 0) 503 cse->hashsize = sop->maclen; 504 else if (thash != NULL) 505 cse->hashsize = thash->hashsize; 506 else if (csp.csp_mode == CSP_MODE_AEAD) 507 cse->hashsize = txform->macsize; 508 cse->ivsize = csp.csp_ivlen; 509 510 /* 511 * NB: This isn't necessarily the block size of the underlying 512 * MAC or cipher but is instead a restriction on valid input 513 * sizes. 514 */ 515 if (txform != NULL) 516 cse->blocksize = txform->blocksize; 517 else 518 cse->blocksize = 1; 519 520 mtx_lock(&fcr->lock); 521 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 522 cse->ses = fcr->sesn++; 523 mtx_unlock(&fcr->lock); 524 525 sop->ses = cse->ses; 526 527 /* return hardware/driver id */ 528 sop->crid = crypto_ses2hid(cse->cses); 529 bail: 530 if (error) { 531 free(key, M_XDATA); 532 free(mackey, M_XDATA); 533 } 534 return (error); 535 } 536 537 static struct csession * 538 cse_find(struct fcrypt *fcr, u_int ses) 539 { 540 struct csession *cse; 541 542 mtx_lock(&fcr->lock); 543 TAILQ_FOREACH(cse, &fcr->csessions, next) { 544 if (cse->ses == ses) { 545 refcount_acquire(&cse->refs); 546 mtx_unlock(&fcr->lock); 547 return (cse); 548 } 549 } 550 mtx_unlock(&fcr->lock); 551 return (NULL); 552 } 553 554 static void 555 cse_free(struct csession *cse) 556 { 557 558 if (!refcount_release(&cse->refs)) 559 return; 560 crypto_freesession(cse->cses); 561 mtx_destroy(&cse->lock); 562 if (cse->key) 563 free(cse->key, M_XDATA); 564 if (cse->mackey) 565 free(cse->mackey, M_XDATA); 566 free(cse, M_XDATA); 567 } 568 569 static bool 570 cse_delete(struct fcrypt *fcr, u_int ses) 571 { 572 struct csession *cse; 573 574 mtx_lock(&fcr->lock); 575 TAILQ_FOREACH(cse, &fcr->csessions, next) { 576 if (cse->ses == ses) { 577 TAILQ_REMOVE(&fcr->csessions, cse, next); 578 mtx_unlock(&fcr->lock); 579 cse_free(cse); 580 return (true); 581 } 582 } 583 mtx_unlock(&fcr->lock); 584 return (false); 585 } 586 587 static struct cryptop_data * 588 cod_alloc(struct csession *cse, size_t aad_len, size_t len) 589 { 590 struct cryptop_data *cod; 591 592 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 593 594 cod->cse = cse; 595 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 596 if (aad_len != 0) 597 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 598 cod->buf = malloc(len, M_XDATA, M_WAITOK); 599 } else 600 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 601 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 602 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 603 return (cod); 604 } 605 606 static void 607 cod_free(struct cryptop_data *cod) 608 { 609 610 free(cod->aad, M_XDATA); 611 free(cod->obuf, M_XDATA); 612 free(cod->buf, M_XDATA); 613 free(cod, M_XDATA); 614 } 615 616 static int 617 cryptodev_cb(struct cryptop *crp) 618 { 619 struct cryptop_data *cod = crp->crp_opaque; 620 621 /* 622 * Lock to ensure the wakeup() is not missed by the loops 623 * waiting on cod->done in cryptodev_op() and 624 * cryptodev_aead(). 625 */ 626 mtx_lock(&cod->cse->lock); 627 cod->done = true; 628 mtx_unlock(&cod->cse->lock); 629 wakeup(cod); 630 return (0); 631 } 632 633 static int 634 cryptodev_op(struct csession *cse, const struct crypt_op *cop) 635 { 636 const struct crypto_session_params *csp; 637 struct cryptop_data *cod = NULL; 638 struct cryptop *crp = NULL; 639 char *dst; 640 int error; 641 642 if (cop->len > 256*1024-4) { 643 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 644 return (E2BIG); 645 } 646 647 if ((cop->len % cse->blocksize) != 0) { 648 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 649 return (EINVAL); 650 } 651 652 if (cop->mac && cse->hashsize == 0) { 653 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 654 return (EINVAL); 655 } 656 657 /* 658 * The COP_F_CIPHER_FIRST flag predates explicit session 659 * modes, but the only way it was used was for EtA so allow it 660 * as long as it is consistent with EtA. 661 */ 662 if (cop->flags & COP_F_CIPHER_FIRST) { 663 if (cop->op != COP_ENCRYPT) { 664 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 665 return (EINVAL); 666 } 667 } 668 669 cod = cod_alloc(cse, 0, cop->len + cse->hashsize); 670 dst = cop->dst; 671 672 crp = crypto_getreq(cse->cses, M_WAITOK); 673 674 error = copyin(cop->src, cod->buf, cop->len); 675 if (error) { 676 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 677 goto bail; 678 } 679 crp->crp_payload_start = 0; 680 crp->crp_payload_length = cop->len; 681 if (cse->hashsize) 682 crp->crp_digest_start = cop->len; 683 684 csp = crypto_get_params(cse->cses); 685 switch (csp->csp_mode) { 686 case CSP_MODE_COMPRESS: 687 switch (cop->op) { 688 case COP_ENCRYPT: 689 crp->crp_op = CRYPTO_OP_COMPRESS; 690 break; 691 case COP_DECRYPT: 692 crp->crp_op = CRYPTO_OP_DECOMPRESS; 693 break; 694 default: 695 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 696 error = EINVAL; 697 goto bail; 698 } 699 break; 700 case CSP_MODE_CIPHER: 701 if (cop->len == 0 || 702 (cop->iv == NULL && cop->len == cse->ivsize)) { 703 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 704 error = EINVAL; 705 goto bail; 706 } 707 switch (cop->op) { 708 case COP_ENCRYPT: 709 crp->crp_op = CRYPTO_OP_ENCRYPT; 710 break; 711 case COP_DECRYPT: 712 crp->crp_op = CRYPTO_OP_DECRYPT; 713 break; 714 default: 715 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 716 error = EINVAL; 717 goto bail; 718 } 719 break; 720 case CSP_MODE_DIGEST: 721 switch (cop->op) { 722 case 0: 723 case COP_ENCRYPT: 724 case COP_DECRYPT: 725 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 726 if (cod->obuf != NULL) 727 crp->crp_digest_start = 0; 728 break; 729 default: 730 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 731 error = EINVAL; 732 goto bail; 733 } 734 break; 735 case CSP_MODE_AEAD: 736 if (cse->ivsize != 0 && cop->iv == NULL) { 737 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 738 error = EINVAL; 739 goto bail; 740 } 741 /* FALLTHROUGH */ 742 case CSP_MODE_ETA: 743 switch (cop->op) { 744 case COP_ENCRYPT: 745 crp->crp_op = CRYPTO_OP_ENCRYPT | 746 CRYPTO_OP_COMPUTE_DIGEST; 747 break; 748 case COP_DECRYPT: 749 crp->crp_op = CRYPTO_OP_DECRYPT | 750 CRYPTO_OP_VERIFY_DIGEST; 751 break; 752 default: 753 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 754 error = EINVAL; 755 goto bail; 756 } 757 break; 758 default: 759 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 760 error = EINVAL; 761 goto bail; 762 } 763 764 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 765 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 766 if (cod->obuf) 767 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 768 crp->crp_callback = cryptodev_cb; 769 crp->crp_opaque = cod; 770 771 if (cop->iv) { 772 if (cse->ivsize == 0) { 773 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 774 error = EINVAL; 775 goto bail; 776 } 777 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 778 if (error) { 779 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 780 goto bail; 781 } 782 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 783 } else if (cse->ivsize != 0) { 784 if (crp->crp_payload_length < cse->ivsize) { 785 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 786 error = EINVAL; 787 goto bail; 788 } 789 crp->crp_iv_start = 0; 790 crp->crp_payload_length -= cse->ivsize; 791 if (crp->crp_payload_length != 0) 792 crp->crp_payload_start = cse->ivsize; 793 dst += cse->ivsize; 794 } 795 796 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 797 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 798 cse->hashsize); 799 if (error) { 800 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 801 goto bail; 802 } 803 } 804 again: 805 /* 806 * Let the dispatch run unlocked, then, interlock against the 807 * callback before checking if the operation completed and going 808 * to sleep. This insures drivers don't inherit our lock which 809 * results in a lock order reversal between crypto_dispatch forced 810 * entry and the crypto_done callback into us. 811 */ 812 error = crypto_dispatch(crp); 813 if (error != 0) { 814 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 815 goto bail; 816 } 817 818 mtx_lock(&cse->lock); 819 while (!cod->done) 820 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 821 mtx_unlock(&cse->lock); 822 823 if (crp->crp_etype == EAGAIN) { 824 crp->crp_etype = 0; 825 crp->crp_flags &= ~CRYPTO_F_DONE; 826 cod->done = false; 827 goto again; 828 } 829 830 if (crp->crp_etype != 0) { 831 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 832 error = crp->crp_etype; 833 goto bail; 834 } 835 836 if (cop->dst != NULL) { 837 error = copyout(cod->obuf != NULL ? cod->obuf : 838 cod->buf + crp->crp_payload_start, dst, 839 crp->crp_payload_length); 840 if (error) { 841 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 842 goto bail; 843 } 844 } 845 846 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 847 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 848 crp->crp_digest_start, cop->mac, cse->hashsize); 849 if (error) { 850 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 851 goto bail; 852 } 853 } 854 855 bail: 856 crypto_freereq(crp); 857 cod_free(cod); 858 859 return (error); 860 } 861 862 static int 863 cryptodev_aead(struct csession *cse, struct crypt_aead *caead) 864 { 865 const struct crypto_session_params *csp; 866 struct cryptop_data *cod = NULL; 867 struct cryptop *crp = NULL; 868 char *dst; 869 int error; 870 871 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 872 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 873 return (E2BIG); 874 } 875 876 if ((caead->len % cse->blocksize) != 0) { 877 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 878 return (EINVAL); 879 } 880 881 if (cse->hashsize == 0 || caead->tag == NULL) { 882 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 883 return (EINVAL); 884 } 885 886 /* 887 * The COP_F_CIPHER_FIRST flag predates explicit session 888 * modes, but the only way it was used was for EtA so allow it 889 * as long as it is consistent with EtA. 890 */ 891 if (caead->flags & COP_F_CIPHER_FIRST) { 892 if (caead->op != COP_ENCRYPT) { 893 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 894 return (EINVAL); 895 } 896 } 897 898 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize); 899 dst = caead->dst; 900 901 crp = crypto_getreq(cse->cses, M_WAITOK); 902 903 if (cod->aad != NULL) 904 error = copyin(caead->aad, cod->aad, caead->aadlen); 905 else 906 error = copyin(caead->aad, cod->buf, caead->aadlen); 907 if (error) { 908 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 909 goto bail; 910 } 911 crp->crp_aad = cod->aad; 912 crp->crp_aad_start = 0; 913 crp->crp_aad_length = caead->aadlen; 914 915 if (cod->aad != NULL) 916 crp->crp_payload_start = 0; 917 else 918 crp->crp_payload_start = caead->aadlen; 919 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 920 caead->len); 921 if (error) { 922 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 923 goto bail; 924 } 925 crp->crp_payload_length = caead->len; 926 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 927 crp->crp_digest_start = crp->crp_payload_output_start + 928 caead->len; 929 else 930 crp->crp_digest_start = crp->crp_payload_start + caead->len; 931 932 csp = crypto_get_params(cse->cses); 933 switch (csp->csp_mode) { 934 case CSP_MODE_AEAD: 935 case CSP_MODE_ETA: 936 switch (caead->op) { 937 case COP_ENCRYPT: 938 crp->crp_op = CRYPTO_OP_ENCRYPT | 939 CRYPTO_OP_COMPUTE_DIGEST; 940 break; 941 case COP_DECRYPT: 942 crp->crp_op = CRYPTO_OP_DECRYPT | 943 CRYPTO_OP_VERIFY_DIGEST; 944 break; 945 default: 946 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 947 error = EINVAL; 948 goto bail; 949 } 950 break; 951 default: 952 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 953 error = EINVAL; 954 goto bail; 955 } 956 957 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 958 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 959 cse->hashsize); 960 if (cod->obuf != NULL) 961 crypto_use_output_buf(crp, cod->obuf, caead->len + 962 cse->hashsize); 963 crp->crp_callback = cryptodev_cb; 964 crp->crp_opaque = cod; 965 966 if (caead->iv) { 967 /* 968 * Permit a 16-byte IV for AES-XTS, but only use the 969 * first 8 bytes as a block number. 970 */ 971 if (csp->csp_mode == CSP_MODE_ETA && 972 csp->csp_cipher_alg == CRYPTO_AES_XTS && 973 caead->ivlen == AES_BLOCK_LEN) 974 caead->ivlen = AES_XTS_IV_LEN; 975 976 if (cse->ivsize == 0) { 977 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 978 error = EINVAL; 979 goto bail; 980 } 981 if (caead->ivlen != cse->ivsize) { 982 error = EINVAL; 983 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 984 goto bail; 985 } 986 987 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 988 if (error) { 989 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 990 goto bail; 991 } 992 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 993 } else { 994 error = EINVAL; 995 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 996 goto bail; 997 } 998 999 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1000 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1001 cse->hashsize); 1002 if (error) { 1003 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1004 goto bail; 1005 } 1006 } 1007 again: 1008 /* 1009 * Let the dispatch run unlocked, then, interlock against the 1010 * callback before checking if the operation completed and going 1011 * to sleep. This insures drivers don't inherit our lock which 1012 * results in a lock order reversal between crypto_dispatch forced 1013 * entry and the crypto_done callback into us. 1014 */ 1015 error = crypto_dispatch(crp); 1016 if (error != 0) { 1017 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1018 goto bail; 1019 } 1020 1021 mtx_lock(&cse->lock); 1022 while (!cod->done) 1023 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1024 mtx_unlock(&cse->lock); 1025 1026 if (crp->crp_etype == EAGAIN) { 1027 crp->crp_etype = 0; 1028 crp->crp_flags &= ~CRYPTO_F_DONE; 1029 cod->done = false; 1030 goto again; 1031 } 1032 1033 if (crp->crp_etype != 0) { 1034 error = crp->crp_etype; 1035 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1036 goto bail; 1037 } 1038 1039 if (caead->dst != NULL) { 1040 error = copyout(cod->obuf != NULL ? cod->obuf : 1041 cod->buf + crp->crp_payload_start, dst, 1042 crp->crp_payload_length); 1043 if (error) { 1044 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1045 goto bail; 1046 } 1047 } 1048 1049 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1050 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1051 crp->crp_digest_start, caead->tag, cse->hashsize); 1052 if (error) { 1053 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1054 goto bail; 1055 } 1056 } 1057 1058 bail: 1059 crypto_freereq(crp); 1060 cod_free(cod); 1061 1062 return (error); 1063 } 1064 1065 static int 1066 cryptodev_find(struct crypt_find_op *find) 1067 { 1068 device_t dev; 1069 size_t fnlen = sizeof find->name; 1070 1071 if (find->crid != -1) { 1072 dev = crypto_find_device_byhid(find->crid); 1073 if (dev == NULL) 1074 return (ENOENT); 1075 strncpy(find->name, device_get_nameunit(dev), fnlen); 1076 find->name[fnlen - 1] = '\x0'; 1077 } else { 1078 find->name[fnlen - 1] = '\x0'; 1079 find->crid = crypto_find_driver(find->name); 1080 if (find->crid == -1) 1081 return (ENOENT); 1082 } 1083 return (0); 1084 } 1085 1086 static void 1087 fcrypt_dtor(void *data) 1088 { 1089 struct fcrypt *fcr = data; 1090 struct csession *cse; 1091 1092 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1093 TAILQ_REMOVE(&fcr->csessions, cse, next); 1094 KASSERT(refcount_load(&cse->refs) == 1, 1095 ("%s: crypto session %p with %d refs", __func__, cse, 1096 refcount_load(&cse->refs))); 1097 cse_free(cse); 1098 } 1099 mtx_destroy(&fcr->lock); 1100 free(fcr, M_XDATA); 1101 } 1102 1103 static int 1104 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1105 { 1106 struct fcrypt *fcr; 1107 int error; 1108 1109 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1110 TAILQ_INIT(&fcr->csessions); 1111 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1112 error = devfs_set_cdevpriv(fcr, fcrypt_dtor); 1113 if (error) 1114 fcrypt_dtor(fcr); 1115 return (error); 1116 } 1117 1118 static int 1119 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, 1120 struct thread *td) 1121 { 1122 struct fcrypt *fcr; 1123 struct csession *cse; 1124 struct session2_op *sop; 1125 struct crypt_op *cop; 1126 struct crypt_aead *caead; 1127 uint32_t ses; 1128 int error = 0; 1129 union { 1130 struct session2_op sopc; 1131 #ifdef COMPAT_FREEBSD32 1132 struct crypt_op copc; 1133 struct crypt_aead aeadc; 1134 #endif 1135 } thunk; 1136 #ifdef COMPAT_FREEBSD32 1137 u_long cmd32; 1138 void *data32; 1139 1140 cmd32 = 0; 1141 data32 = NULL; 1142 switch (cmd) { 1143 case CIOCGSESSION32: 1144 cmd32 = cmd; 1145 data32 = data; 1146 cmd = CIOCGSESSION; 1147 data = (void *)&thunk.sopc; 1148 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 1149 break; 1150 case CIOCGSESSION232: 1151 cmd32 = cmd; 1152 data32 = data; 1153 cmd = CIOCGSESSION2; 1154 data = (void *)&thunk.sopc; 1155 session2_op_from_32((struct session2_op32 *)data32, 1156 &thunk.sopc); 1157 break; 1158 case CIOCCRYPT32: 1159 cmd32 = cmd; 1160 data32 = data; 1161 cmd = CIOCCRYPT; 1162 data = (void *)&thunk.copc; 1163 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 1164 break; 1165 case CIOCCRYPTAEAD32: 1166 cmd32 = cmd; 1167 data32 = data; 1168 cmd = CIOCCRYPTAEAD; 1169 data = (void *)&thunk.aeadc; 1170 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 1171 break; 1172 } 1173 #endif 1174 1175 devfs_get_cdevpriv((void **)&fcr); 1176 1177 switch (cmd) { 1178 #ifdef COMPAT_FREEBSD12 1179 case CRIOGET: 1180 /* 1181 * NB: This may fail in cases that the old 1182 * implementation did not if the current process has 1183 * restricted filesystem access (e.g. running in a 1184 * jail that does not expose /dev/crypto or in 1185 * capability mode). 1186 */ 1187 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE, 1188 O_RDWR, 0); 1189 if (error == 0) 1190 *(uint32_t *)data = td->td_retval[0]; 1191 break; 1192 #endif 1193 case CIOCGSESSION: 1194 case CIOCGSESSION2: 1195 if (cmd == CIOCGSESSION) { 1196 session2_op_from_op((void *)data, &thunk.sopc); 1197 sop = &thunk.sopc; 1198 } else 1199 sop = (struct session2_op *)data; 1200 1201 error = cse_create(fcr, sop); 1202 if (cmd == CIOCGSESSION && error == 0) 1203 session2_op_to_op(sop, (void *)data); 1204 break; 1205 case CIOCFSESSION: 1206 ses = *(uint32_t *)data; 1207 if (!cse_delete(fcr, ses)) { 1208 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1209 return (EINVAL); 1210 } 1211 break; 1212 case CIOCCRYPT: 1213 cop = (struct crypt_op *)data; 1214 cse = cse_find(fcr, cop->ses); 1215 if (cse == NULL) { 1216 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1217 return (EINVAL); 1218 } 1219 error = cryptodev_op(cse, cop); 1220 cse_free(cse); 1221 break; 1222 case CIOCFINDDEV: 1223 error = cryptodev_find((struct crypt_find_op *)data); 1224 break; 1225 case CIOCCRYPTAEAD: 1226 caead = (struct crypt_aead *)data; 1227 cse = cse_find(fcr, caead->ses); 1228 if (cse == NULL) { 1229 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1230 return (EINVAL); 1231 } 1232 error = cryptodev_aead(cse, caead); 1233 cse_free(cse); 1234 break; 1235 default: 1236 error = EINVAL; 1237 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1238 break; 1239 } 1240 1241 #ifdef COMPAT_FREEBSD32 1242 switch (cmd32) { 1243 case CIOCGSESSION32: 1244 if (error == 0) 1245 session_op_to_32((void *)data, data32); 1246 break; 1247 case CIOCGSESSION232: 1248 if (error == 0) 1249 session2_op_to_32((void *)data, data32); 1250 break; 1251 case CIOCCRYPT32: 1252 if (error == 0) 1253 crypt_op_to_32((void *)data, data32); 1254 break; 1255 case CIOCCRYPTAEAD32: 1256 if (error == 0) 1257 crypt_aead_to_32((void *)data, data32); 1258 break; 1259 } 1260 #endif 1261 return (error); 1262 } 1263 1264 static struct cdevsw crypto_cdevsw = { 1265 .d_version = D_VERSION, 1266 .d_open = crypto_open, 1267 .d_ioctl = crypto_ioctl, 1268 .d_name = "crypto", 1269 }; 1270 static struct cdev *crypto_dev; 1271 1272 /* 1273 * Initialization code, both for static and dynamic loading. 1274 */ 1275 static int 1276 cryptodev_modevent(module_t mod, int type, void *unused) 1277 { 1278 switch (type) { 1279 case MOD_LOAD: 1280 if (bootverbose) 1281 printf("crypto: <crypto device>\n"); 1282 crypto_dev = make_dev(&crypto_cdevsw, 0, 1283 UID_ROOT, GID_WHEEL, 0666, 1284 "crypto"); 1285 return 0; 1286 case MOD_UNLOAD: 1287 /*XXX disallow if active sessions */ 1288 destroy_dev(crypto_dev); 1289 return 0; 1290 } 1291 return EINVAL; 1292 } 1293 1294 static moduledata_t cryptodev_mod = { 1295 "cryptodev", 1296 cryptodev_modevent, 1297 0 1298 }; 1299 MODULE_VERSION(cryptodev, 1); 1300 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1301 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1302 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1303