1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014-2021 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Portions of this software were developed by Ararat River 14 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. The name of the author may not be used to endorse or promote products 26 * derived from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * Effort sponsored in part by the Defense Advanced Research Projects 40 * Agency (DARPA) and Air Force Research Laboratory, Air Force 41 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 42 */ 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/proc.h> 51 #include <sys/sysctl.h> 52 #include <sys/errno.h> 53 #include <sys/random.h> 54 #include <sys/conf.h> 55 #include <sys/kernel.h> 56 #include <sys/module.h> 57 #include <sys/fcntl.h> 58 #include <sys/bus.h> 59 #include <sys/sdt.h> 60 #include <sys/syscallsubr.h> 61 62 #include <opencrypto/cryptodev.h> 63 #include <opencrypto/xform.h> 64 65 SDT_PROVIDER_DECLARE(opencrypto); 66 67 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 68 69 #ifdef COMPAT_FREEBSD12 70 /* 71 * Previously, most ioctls were performed against a cloned descriptor 72 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed 73 * against /dev/crypto directly. 74 */ 75 #define CRIOGET _IOWR('c', 100, uint32_t) 76 #endif 77 78 /* the following are done against the cloned descriptor */ 79 80 #ifdef COMPAT_FREEBSD32 81 #include <sys/mount.h> 82 #include <compat/freebsd32/freebsd32.h> 83 84 struct session_op32 { 85 uint32_t cipher; 86 uint32_t mac; 87 uint32_t keylen; 88 uint32_t key; 89 int mackeylen; 90 uint32_t mackey; 91 uint32_t ses; 92 }; 93 94 struct session2_op32 { 95 uint32_t cipher; 96 uint32_t mac; 97 uint32_t keylen; 98 uint32_t key; 99 int mackeylen; 100 uint32_t mackey; 101 uint32_t ses; 102 int crid; 103 int ivlen; 104 int maclen; 105 int pad[2]; 106 }; 107 108 struct crypt_op32 { 109 uint32_t ses; 110 uint16_t op; 111 uint16_t flags; 112 u_int len; 113 uint32_t src, dst; 114 uint32_t mac; 115 uint32_t iv; 116 }; 117 118 struct crypt_aead32 { 119 uint32_t ses; 120 uint16_t op; 121 uint16_t flags; 122 u_int len; 123 u_int aadlen; 124 u_int ivlen; 125 uint32_t src; 126 uint32_t dst; 127 uint32_t aad; 128 uint32_t tag; 129 uint32_t iv; 130 }; 131 132 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 133 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 134 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 135 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 136 137 static void 138 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 139 { 140 141 memset(to, 0, sizeof(*to)); 142 CP(*from, *to, cipher); 143 CP(*from, *to, mac); 144 CP(*from, *to, keylen); 145 PTRIN_CP(*from, *to, key); 146 CP(*from, *to, mackeylen); 147 PTRIN_CP(*from, *to, mackey); 148 CP(*from, *to, ses); 149 to->crid = CRYPTOCAP_F_HARDWARE; 150 } 151 152 static void 153 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 154 { 155 156 session_op_from_32((const struct session_op32 *)from, to); 157 CP(*from, *to, crid); 158 CP(*from, *to, ivlen); 159 CP(*from, *to, maclen); 160 } 161 162 static void 163 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 164 { 165 166 CP(*from, *to, cipher); 167 CP(*from, *to, mac); 168 CP(*from, *to, keylen); 169 PTROUT_CP(*from, *to, key); 170 CP(*from, *to, mackeylen); 171 PTROUT_CP(*from, *to, mackey); 172 CP(*from, *to, ses); 173 } 174 175 static void 176 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 177 { 178 179 session_op_to_32(from, (struct session_op32 *)to); 180 CP(*from, *to, crid); 181 } 182 183 static void 184 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 185 { 186 187 CP(*from, *to, ses); 188 CP(*from, *to, op); 189 CP(*from, *to, flags); 190 CP(*from, *to, len); 191 PTRIN_CP(*from, *to, src); 192 PTRIN_CP(*from, *to, dst); 193 PTRIN_CP(*from, *to, mac); 194 PTRIN_CP(*from, *to, iv); 195 } 196 197 static void 198 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 199 { 200 201 CP(*from, *to, ses); 202 CP(*from, *to, op); 203 CP(*from, *to, flags); 204 CP(*from, *to, len); 205 PTROUT_CP(*from, *to, src); 206 PTROUT_CP(*from, *to, dst); 207 PTROUT_CP(*from, *to, mac); 208 PTROUT_CP(*from, *to, iv); 209 } 210 211 static void 212 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 213 { 214 215 CP(*from, *to, ses); 216 CP(*from, *to, op); 217 CP(*from, *to, flags); 218 CP(*from, *to, len); 219 CP(*from, *to, aadlen); 220 CP(*from, *to, ivlen); 221 PTRIN_CP(*from, *to, src); 222 PTRIN_CP(*from, *to, dst); 223 PTRIN_CP(*from, *to, aad); 224 PTRIN_CP(*from, *to, tag); 225 PTRIN_CP(*from, *to, iv); 226 } 227 228 static void 229 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 230 { 231 232 CP(*from, *to, ses); 233 CP(*from, *to, op); 234 CP(*from, *to, flags); 235 CP(*from, *to, len); 236 CP(*from, *to, aadlen); 237 CP(*from, *to, ivlen); 238 PTROUT_CP(*from, *to, src); 239 PTROUT_CP(*from, *to, dst); 240 PTROUT_CP(*from, *to, aad); 241 PTROUT_CP(*from, *to, tag); 242 PTROUT_CP(*from, *to, iv); 243 } 244 #endif 245 246 static void 247 session2_op_from_op(const struct session_op *from, struct session2_op *to) 248 { 249 250 memset(to, 0, sizeof(*to)); 251 memcpy(to, from, sizeof(*from)); 252 to->crid = CRYPTOCAP_F_HARDWARE; 253 } 254 255 static void 256 session2_op_to_op(const struct session2_op *from, struct session_op *to) 257 { 258 259 memcpy(to, from, sizeof(*to)); 260 } 261 262 struct csession { 263 TAILQ_ENTRY(csession) next; 264 crypto_session_t cses; 265 volatile u_int refs; 266 uint32_t ses; 267 struct mtx lock; /* for op submission */ 268 269 u_int blocksize; 270 int hashsize; 271 int ivsize; 272 273 void *key; 274 void *mackey; 275 }; 276 277 struct cryptop_data { 278 struct csession *cse; 279 280 char *buf; 281 char *obuf; 282 char *aad; 283 bool done; 284 }; 285 286 struct fcrypt { 287 TAILQ_HEAD(csessionlist, csession) csessions; 288 int sesn; 289 struct mtx lock; 290 }; 291 292 static bool use_outputbuffers; 293 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 294 &use_outputbuffers, 0, 295 "Use separate output buffers for /dev/crypto requests."); 296 297 static bool use_separate_aad; 298 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 299 &use_separate_aad, 0, 300 "Use separate AAD buffer for /dev/crypto requests."); 301 302 static MALLOC_DEFINE(M_CRYPTODEV, "cryptodev", "/dev/crypto data buffers"); 303 304 /* 305 * Check a crypto identifier to see if it requested 306 * a software device/driver. This can be done either 307 * by device name/class or through search constraints. 308 */ 309 static int 310 checkforsoftware(int *cridp) 311 { 312 int crid; 313 314 crid = *cridp; 315 316 if (!crypto_devallowsoft) { 317 if (crid & CRYPTOCAP_F_SOFTWARE) { 318 if (crid & CRYPTOCAP_F_HARDWARE) { 319 *cridp = CRYPTOCAP_F_HARDWARE; 320 return 0; 321 } 322 return EINVAL; 323 } 324 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 325 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 326 return EINVAL; 327 } 328 return 0; 329 } 330 331 static int 332 cse_create(struct fcrypt *fcr, struct session2_op *sop) 333 { 334 struct crypto_session_params csp; 335 struct csession *cse; 336 const struct enc_xform *txform; 337 const struct auth_hash *thash; 338 void *key = NULL; 339 void *mackey = NULL; 340 crypto_session_t cses; 341 int crid, error, mac; 342 343 mac = sop->mac; 344 #ifdef COMPAT_FREEBSD12 345 switch (sop->mac) { 346 case CRYPTO_AES_128_NIST_GMAC: 347 case CRYPTO_AES_192_NIST_GMAC: 348 case CRYPTO_AES_256_NIST_GMAC: 349 /* Should always be paired with GCM. */ 350 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 351 CRYPTDEB("GMAC without GCM"); 352 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 353 return (EINVAL); 354 } 355 if (sop->keylen != sop->mackeylen) { 356 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 357 return (EINVAL); 358 } 359 mac = 0; 360 break; 361 case CRYPTO_AES_CCM_CBC_MAC: 362 /* Should always be paired with CCM. */ 363 if (sop->cipher != CRYPTO_AES_CCM_16) { 364 CRYPTDEB("CBC-MAC without CCM"); 365 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 366 return (EINVAL); 367 } 368 if (sop->keylen != sop->mackeylen) { 369 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 370 return (EINVAL); 371 } 372 mac = 0; 373 break; 374 } 375 #endif 376 377 memset(&csp, 0, sizeof(csp)); 378 if (use_outputbuffers) 379 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 380 if (mac != 0) { 381 csp.csp_auth_alg = mac; 382 csp.csp_auth_klen = sop->mackeylen; 383 } 384 if (sop->cipher != 0) { 385 csp.csp_cipher_alg = sop->cipher; 386 csp.csp_cipher_klen = sop->keylen; 387 } 388 thash = crypto_auth_hash(&csp); 389 txform = crypto_cipher(&csp); 390 391 if (txform != NULL && txform->macsize != 0) { 392 if (mac != 0) { 393 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 394 return (EINVAL); 395 } 396 csp.csp_mode = CSP_MODE_AEAD; 397 } else if (txform != NULL && thash != NULL) { 398 csp.csp_mode = CSP_MODE_ETA; 399 } else if (txform != NULL) { 400 csp.csp_mode = CSP_MODE_CIPHER; 401 } else if (thash != NULL) { 402 csp.csp_mode = CSP_MODE_DIGEST; 403 } else { 404 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 405 return (EINVAL); 406 } 407 408 switch (csp.csp_mode) { 409 case CSP_MODE_AEAD: 410 case CSP_MODE_ETA: 411 if (use_separate_aad) 412 csp.csp_flags |= CSP_F_SEPARATE_AAD; 413 break; 414 } 415 416 if (txform != NULL) { 417 if (sop->keylen > txform->maxkey || 418 sop->keylen < txform->minkey) { 419 CRYPTDEB("invalid cipher parameters"); 420 error = EINVAL; 421 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 422 goto bail; 423 } 424 425 key = malloc(csp.csp_cipher_klen, M_CRYPTODEV, M_WAITOK); 426 error = copyin(sop->key, key, csp.csp_cipher_klen); 427 if (error) { 428 CRYPTDEB("invalid key"); 429 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 430 goto bail; 431 } 432 csp.csp_cipher_key = key; 433 csp.csp_ivlen = txform->ivsize; 434 } 435 436 if (thash != NULL) { 437 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { 438 CRYPTDEB("invalid mac key length"); 439 error = EINVAL; 440 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 441 goto bail; 442 } 443 444 if (csp.csp_auth_klen != 0) { 445 mackey = malloc(csp.csp_auth_klen, M_CRYPTODEV, 446 M_WAITOK); 447 error = copyin(sop->mackey, mackey, csp.csp_auth_klen); 448 if (error) { 449 CRYPTDEB("invalid mac key"); 450 SDT_PROBE1(opencrypto, dev, ioctl, error, 451 __LINE__); 452 goto bail; 453 } 454 csp.csp_auth_key = mackey; 455 } 456 457 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 458 csp.csp_ivlen = AES_GCM_IV_LEN; 459 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 460 csp.csp_ivlen = AES_CCM_IV_LEN; 461 } 462 463 if (sop->ivlen != 0) { 464 if (csp.csp_ivlen == 0) { 465 CRYPTDEB("does not support an IV"); 466 error = EINVAL; 467 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 468 goto bail; 469 } 470 csp.csp_ivlen = sop->ivlen; 471 } 472 if (sop->maclen != 0) { 473 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) { 474 CRYPTDEB("does not support a MAC"); 475 error = EINVAL; 476 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 477 goto bail; 478 } 479 csp.csp_auth_mlen = sop->maclen; 480 } 481 482 crid = sop->crid; 483 error = checkforsoftware(&crid); 484 if (error) { 485 CRYPTDEB("checkforsoftware"); 486 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 487 goto bail; 488 } 489 error = crypto_newsession(&cses, &csp, crid); 490 if (error) { 491 CRYPTDEB("crypto_newsession"); 492 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 493 goto bail; 494 } 495 496 cse = malloc(sizeof(struct csession), M_CRYPTODEV, M_WAITOK | M_ZERO); 497 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 498 refcount_init(&cse->refs, 1); 499 cse->key = key; 500 cse->mackey = mackey; 501 cse->cses = cses; 502 if (sop->maclen != 0) 503 cse->hashsize = sop->maclen; 504 else if (thash != NULL) 505 cse->hashsize = thash->hashsize; 506 else if (csp.csp_mode == CSP_MODE_AEAD) 507 cse->hashsize = txform->macsize; 508 cse->ivsize = csp.csp_ivlen; 509 510 /* 511 * NB: This isn't necessarily the block size of the underlying 512 * MAC or cipher but is instead a restriction on valid input 513 * sizes. 514 */ 515 if (txform != NULL) 516 cse->blocksize = txform->blocksize; 517 else 518 cse->blocksize = 1; 519 520 mtx_lock(&fcr->lock); 521 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 522 cse->ses = fcr->sesn++; 523 mtx_unlock(&fcr->lock); 524 525 sop->ses = cse->ses; 526 527 /* return hardware/driver id */ 528 sop->crid = crypto_ses2hid(cse->cses); 529 bail: 530 if (error) { 531 free(key, M_CRYPTODEV); 532 free(mackey, M_CRYPTODEV); 533 } 534 return (error); 535 } 536 537 static struct csession * 538 cse_find(struct fcrypt *fcr, u_int ses) 539 { 540 struct csession *cse; 541 542 mtx_lock(&fcr->lock); 543 TAILQ_FOREACH(cse, &fcr->csessions, next) { 544 if (cse->ses == ses) { 545 refcount_acquire(&cse->refs); 546 mtx_unlock(&fcr->lock); 547 return (cse); 548 } 549 } 550 mtx_unlock(&fcr->lock); 551 return (NULL); 552 } 553 554 static void 555 cse_free(struct csession *cse) 556 { 557 558 if (!refcount_release(&cse->refs)) 559 return; 560 crypto_freesession(cse->cses); 561 mtx_destroy(&cse->lock); 562 if (cse->key) 563 free(cse->key, M_CRYPTODEV); 564 if (cse->mackey) 565 free(cse->mackey, M_CRYPTODEV); 566 free(cse, M_CRYPTODEV); 567 } 568 569 static bool 570 cse_delete(struct fcrypt *fcr, u_int ses) 571 { 572 struct csession *cse; 573 574 mtx_lock(&fcr->lock); 575 TAILQ_FOREACH(cse, &fcr->csessions, next) { 576 if (cse->ses == ses) { 577 TAILQ_REMOVE(&fcr->csessions, cse, next); 578 mtx_unlock(&fcr->lock); 579 cse_free(cse); 580 return (true); 581 } 582 } 583 mtx_unlock(&fcr->lock); 584 return (false); 585 } 586 587 static struct cryptop_data * 588 cod_alloc(struct csession *cse, size_t aad_len, size_t len) 589 { 590 struct cryptop_data *cod; 591 592 cod = malloc(sizeof(struct cryptop_data), M_CRYPTODEV, M_WAITOK | 593 M_ZERO); 594 595 cod->cse = cse; 596 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 597 if (aad_len != 0) 598 cod->aad = malloc(aad_len, M_CRYPTODEV, M_WAITOK); 599 cod->buf = malloc(len, M_CRYPTODEV, M_WAITOK); 600 } else 601 cod->buf = malloc(aad_len + len, M_CRYPTODEV, M_WAITOK); 602 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 603 cod->obuf = malloc(len, M_CRYPTODEV, M_WAITOK); 604 return (cod); 605 } 606 607 static void 608 cod_free(struct cryptop_data *cod) 609 { 610 611 free(cod->aad, M_CRYPTODEV); 612 free(cod->obuf, M_CRYPTODEV); 613 free(cod->buf, M_CRYPTODEV); 614 free(cod, M_CRYPTODEV); 615 } 616 617 static int 618 cryptodev_cb(struct cryptop *crp) 619 { 620 struct cryptop_data *cod = crp->crp_opaque; 621 622 /* 623 * Lock to ensure the wakeup() is not missed by the loops 624 * waiting on cod->done in cryptodev_op() and 625 * cryptodev_aead(). 626 */ 627 mtx_lock(&cod->cse->lock); 628 cod->done = true; 629 mtx_unlock(&cod->cse->lock); 630 wakeup(cod); 631 return (0); 632 } 633 634 static int 635 cryptodev_op(struct csession *cse, const struct crypt_op *cop) 636 { 637 const struct crypto_session_params *csp; 638 struct cryptop_data *cod = NULL; 639 struct cryptop *crp = NULL; 640 char *dst; 641 int error; 642 643 if (cop->len > 256*1024-4) { 644 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 645 return (E2BIG); 646 } 647 648 if ((cop->len % cse->blocksize) != 0) { 649 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 650 return (EINVAL); 651 } 652 653 if (cop->mac && cse->hashsize == 0) { 654 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 655 return (EINVAL); 656 } 657 658 /* 659 * The COP_F_CIPHER_FIRST flag predates explicit session 660 * modes, but the only way it was used was for EtA so allow it 661 * as long as it is consistent with EtA. 662 */ 663 if (cop->flags & COP_F_CIPHER_FIRST) { 664 if (cop->op != COP_ENCRYPT) { 665 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 666 return (EINVAL); 667 } 668 } 669 670 cod = cod_alloc(cse, 0, cop->len + cse->hashsize); 671 dst = cop->dst; 672 673 crp = crypto_getreq(cse->cses, M_WAITOK); 674 675 error = copyin(cop->src, cod->buf, cop->len); 676 if (error) { 677 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 678 goto bail; 679 } 680 crp->crp_payload_start = 0; 681 crp->crp_payload_length = cop->len; 682 if (cse->hashsize) 683 crp->crp_digest_start = cop->len; 684 685 csp = crypto_get_params(cse->cses); 686 switch (csp->csp_mode) { 687 case CSP_MODE_COMPRESS: 688 switch (cop->op) { 689 case COP_ENCRYPT: 690 crp->crp_op = CRYPTO_OP_COMPRESS; 691 break; 692 case COP_DECRYPT: 693 crp->crp_op = CRYPTO_OP_DECOMPRESS; 694 break; 695 default: 696 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 697 error = EINVAL; 698 goto bail; 699 } 700 break; 701 case CSP_MODE_CIPHER: 702 if (cop->len == 0 || 703 (cop->iv == NULL && cop->len == cse->ivsize)) { 704 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 705 error = EINVAL; 706 goto bail; 707 } 708 switch (cop->op) { 709 case COP_ENCRYPT: 710 crp->crp_op = CRYPTO_OP_ENCRYPT; 711 break; 712 case COP_DECRYPT: 713 crp->crp_op = CRYPTO_OP_DECRYPT; 714 break; 715 default: 716 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 717 error = EINVAL; 718 goto bail; 719 } 720 break; 721 case CSP_MODE_DIGEST: 722 switch (cop->op) { 723 case 0: 724 case COP_ENCRYPT: 725 case COP_DECRYPT: 726 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 727 if (cod->obuf != NULL) 728 crp->crp_digest_start = 0; 729 break; 730 default: 731 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 732 error = EINVAL; 733 goto bail; 734 } 735 break; 736 case CSP_MODE_AEAD: 737 if (cse->ivsize != 0 && cop->iv == NULL) { 738 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 739 error = EINVAL; 740 goto bail; 741 } 742 /* FALLTHROUGH */ 743 case CSP_MODE_ETA: 744 switch (cop->op) { 745 case COP_ENCRYPT: 746 crp->crp_op = CRYPTO_OP_ENCRYPT | 747 CRYPTO_OP_COMPUTE_DIGEST; 748 break; 749 case COP_DECRYPT: 750 crp->crp_op = CRYPTO_OP_DECRYPT | 751 CRYPTO_OP_VERIFY_DIGEST; 752 break; 753 default: 754 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 755 error = EINVAL; 756 goto bail; 757 } 758 break; 759 default: 760 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 761 error = EINVAL; 762 goto bail; 763 } 764 765 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 766 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 767 if (cod->obuf) 768 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 769 crp->crp_callback = cryptodev_cb; 770 crp->crp_opaque = cod; 771 772 if (cop->iv) { 773 if (cse->ivsize == 0) { 774 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 775 error = EINVAL; 776 goto bail; 777 } 778 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 779 if (error) { 780 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 781 goto bail; 782 } 783 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 784 } else if (cse->ivsize != 0) { 785 if (crp->crp_payload_length < cse->ivsize) { 786 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 787 error = EINVAL; 788 goto bail; 789 } 790 crp->crp_iv_start = 0; 791 crp->crp_payload_length -= cse->ivsize; 792 if (crp->crp_payload_length != 0) 793 crp->crp_payload_start = cse->ivsize; 794 dst += cse->ivsize; 795 } 796 797 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 798 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 799 cse->hashsize); 800 if (error) { 801 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 802 goto bail; 803 } 804 } 805 again: 806 /* 807 * Let the dispatch run unlocked, then, interlock against the 808 * callback before checking if the operation completed and going 809 * to sleep. This insures drivers don't inherit our lock which 810 * results in a lock order reversal between crypto_dispatch forced 811 * entry and the crypto_done callback into us. 812 */ 813 error = crypto_dispatch(crp); 814 if (error != 0) { 815 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 816 goto bail; 817 } 818 819 mtx_lock(&cse->lock); 820 while (!cod->done) 821 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 822 mtx_unlock(&cse->lock); 823 824 if (crp->crp_etype == EAGAIN) { 825 crp->crp_etype = 0; 826 crp->crp_flags &= ~CRYPTO_F_DONE; 827 cod->done = false; 828 goto again; 829 } 830 831 if (crp->crp_etype != 0) { 832 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 833 error = crp->crp_etype; 834 goto bail; 835 } 836 837 if (cop->dst != NULL) { 838 error = copyout(cod->obuf != NULL ? cod->obuf : 839 cod->buf + crp->crp_payload_start, dst, 840 crp->crp_payload_length); 841 if (error) { 842 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 843 goto bail; 844 } 845 } 846 847 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 848 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 849 crp->crp_digest_start, cop->mac, cse->hashsize); 850 if (error) { 851 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 852 goto bail; 853 } 854 } 855 856 bail: 857 crypto_freereq(crp); 858 cod_free(cod); 859 860 return (error); 861 } 862 863 static int 864 cryptodev_aead(struct csession *cse, struct crypt_aead *caead) 865 { 866 const struct crypto_session_params *csp; 867 struct cryptop_data *cod = NULL; 868 struct cryptop *crp = NULL; 869 char *dst; 870 int error; 871 872 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 873 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 874 return (E2BIG); 875 } 876 877 if ((caead->len % cse->blocksize) != 0) { 878 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 879 return (EINVAL); 880 } 881 882 if (cse->hashsize == 0 || caead->tag == NULL) { 883 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 884 return (EINVAL); 885 } 886 887 /* 888 * The COP_F_CIPHER_FIRST flag predates explicit session 889 * modes, but the only way it was used was for EtA so allow it 890 * as long as it is consistent with EtA. 891 */ 892 if (caead->flags & COP_F_CIPHER_FIRST) { 893 if (caead->op != COP_ENCRYPT) { 894 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 895 return (EINVAL); 896 } 897 } 898 899 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize); 900 dst = caead->dst; 901 902 crp = crypto_getreq(cse->cses, M_WAITOK); 903 904 if (cod->aad != NULL) 905 error = copyin(caead->aad, cod->aad, caead->aadlen); 906 else 907 error = copyin(caead->aad, cod->buf, caead->aadlen); 908 if (error) { 909 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 910 goto bail; 911 } 912 crp->crp_aad = cod->aad; 913 crp->crp_aad_start = 0; 914 crp->crp_aad_length = caead->aadlen; 915 916 if (cod->aad != NULL) 917 crp->crp_payload_start = 0; 918 else 919 crp->crp_payload_start = caead->aadlen; 920 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 921 caead->len); 922 if (error) { 923 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 924 goto bail; 925 } 926 crp->crp_payload_length = caead->len; 927 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 928 crp->crp_digest_start = crp->crp_payload_output_start + 929 caead->len; 930 else 931 crp->crp_digest_start = crp->crp_payload_start + caead->len; 932 933 csp = crypto_get_params(cse->cses); 934 switch (csp->csp_mode) { 935 case CSP_MODE_AEAD: 936 case CSP_MODE_ETA: 937 switch (caead->op) { 938 case COP_ENCRYPT: 939 crp->crp_op = CRYPTO_OP_ENCRYPT | 940 CRYPTO_OP_COMPUTE_DIGEST; 941 break; 942 case COP_DECRYPT: 943 crp->crp_op = CRYPTO_OP_DECRYPT | 944 CRYPTO_OP_VERIFY_DIGEST; 945 break; 946 default: 947 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 948 error = EINVAL; 949 goto bail; 950 } 951 break; 952 default: 953 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 954 error = EINVAL; 955 goto bail; 956 } 957 958 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 959 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 960 cse->hashsize); 961 if (cod->obuf != NULL) 962 crypto_use_output_buf(crp, cod->obuf, caead->len + 963 cse->hashsize); 964 crp->crp_callback = cryptodev_cb; 965 crp->crp_opaque = cod; 966 967 if (caead->iv) { 968 /* 969 * Permit a 16-byte IV for AES-XTS, but only use the 970 * first 8 bytes as a block number. 971 */ 972 if (csp->csp_mode == CSP_MODE_ETA && 973 csp->csp_cipher_alg == CRYPTO_AES_XTS && 974 caead->ivlen == AES_BLOCK_LEN) 975 caead->ivlen = AES_XTS_IV_LEN; 976 977 if (cse->ivsize == 0) { 978 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 979 error = EINVAL; 980 goto bail; 981 } 982 if (caead->ivlen != cse->ivsize) { 983 error = EINVAL; 984 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 985 goto bail; 986 } 987 988 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 989 if (error) { 990 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 991 goto bail; 992 } 993 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 994 } else { 995 error = EINVAL; 996 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 997 goto bail; 998 } 999 1000 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1001 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1002 cse->hashsize); 1003 if (error) { 1004 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1005 goto bail; 1006 } 1007 } 1008 again: 1009 /* 1010 * Let the dispatch run unlocked, then, interlock against the 1011 * callback before checking if the operation completed and going 1012 * to sleep. This insures drivers don't inherit our lock which 1013 * results in a lock order reversal between crypto_dispatch forced 1014 * entry and the crypto_done callback into us. 1015 */ 1016 error = crypto_dispatch(crp); 1017 if (error != 0) { 1018 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1019 goto bail; 1020 } 1021 1022 mtx_lock(&cse->lock); 1023 while (!cod->done) 1024 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1025 mtx_unlock(&cse->lock); 1026 1027 if (crp->crp_etype == EAGAIN) { 1028 crp->crp_etype = 0; 1029 crp->crp_flags &= ~CRYPTO_F_DONE; 1030 cod->done = false; 1031 goto again; 1032 } 1033 1034 if (crp->crp_etype != 0) { 1035 error = crp->crp_etype; 1036 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1037 goto bail; 1038 } 1039 1040 if (caead->dst != NULL) { 1041 error = copyout(cod->obuf != NULL ? cod->obuf : 1042 cod->buf + crp->crp_payload_start, dst, 1043 crp->crp_payload_length); 1044 if (error) { 1045 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1046 goto bail; 1047 } 1048 } 1049 1050 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1051 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1052 crp->crp_digest_start, caead->tag, cse->hashsize); 1053 if (error) { 1054 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1055 goto bail; 1056 } 1057 } 1058 1059 bail: 1060 crypto_freereq(crp); 1061 cod_free(cod); 1062 1063 return (error); 1064 } 1065 1066 static int 1067 cryptodev_find(struct crypt_find_op *find) 1068 { 1069 device_t dev; 1070 size_t fnlen = sizeof find->name; 1071 1072 if (find->crid != -1) { 1073 dev = crypto_find_device_byhid(find->crid); 1074 if (dev == NULL) 1075 return (ENOENT); 1076 strncpy(find->name, device_get_nameunit(dev), fnlen); 1077 find->name[fnlen - 1] = '\x0'; 1078 } else { 1079 find->name[fnlen - 1] = '\x0'; 1080 find->crid = crypto_find_driver(find->name); 1081 if (find->crid == -1) 1082 return (ENOENT); 1083 } 1084 return (0); 1085 } 1086 1087 static void 1088 fcrypt_dtor(void *data) 1089 { 1090 struct fcrypt *fcr = data; 1091 struct csession *cse; 1092 1093 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1094 TAILQ_REMOVE(&fcr->csessions, cse, next); 1095 KASSERT(refcount_load(&cse->refs) == 1, 1096 ("%s: crypto session %p with %d refs", __func__, cse, 1097 refcount_load(&cse->refs))); 1098 cse_free(cse); 1099 } 1100 mtx_destroy(&fcr->lock); 1101 free(fcr, M_CRYPTODEV); 1102 } 1103 1104 static int 1105 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1106 { 1107 struct fcrypt *fcr; 1108 int error; 1109 1110 fcr = malloc(sizeof(struct fcrypt), M_CRYPTODEV, M_WAITOK | M_ZERO); 1111 TAILQ_INIT(&fcr->csessions); 1112 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1113 error = devfs_set_cdevpriv(fcr, fcrypt_dtor); 1114 if (error) 1115 fcrypt_dtor(fcr); 1116 return (error); 1117 } 1118 1119 static int 1120 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, 1121 struct thread *td) 1122 { 1123 struct fcrypt *fcr; 1124 struct csession *cse; 1125 struct session2_op *sop; 1126 struct crypt_op *cop; 1127 struct crypt_aead *caead; 1128 uint32_t ses; 1129 int error = 0; 1130 union { 1131 struct session2_op sopc; 1132 #ifdef COMPAT_FREEBSD32 1133 struct crypt_op copc; 1134 struct crypt_aead aeadc; 1135 #endif 1136 } thunk; 1137 #ifdef COMPAT_FREEBSD32 1138 u_long cmd32; 1139 void *data32; 1140 1141 cmd32 = 0; 1142 data32 = NULL; 1143 switch (cmd) { 1144 case CIOCGSESSION32: 1145 cmd32 = cmd; 1146 data32 = data; 1147 cmd = CIOCGSESSION; 1148 data = (void *)&thunk.sopc; 1149 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 1150 break; 1151 case CIOCGSESSION232: 1152 cmd32 = cmd; 1153 data32 = data; 1154 cmd = CIOCGSESSION2; 1155 data = (void *)&thunk.sopc; 1156 session2_op_from_32((struct session2_op32 *)data32, 1157 &thunk.sopc); 1158 break; 1159 case CIOCCRYPT32: 1160 cmd32 = cmd; 1161 data32 = data; 1162 cmd = CIOCCRYPT; 1163 data = (void *)&thunk.copc; 1164 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 1165 break; 1166 case CIOCCRYPTAEAD32: 1167 cmd32 = cmd; 1168 data32 = data; 1169 cmd = CIOCCRYPTAEAD; 1170 data = (void *)&thunk.aeadc; 1171 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 1172 break; 1173 } 1174 #endif 1175 1176 devfs_get_cdevpriv((void **)&fcr); 1177 1178 switch (cmd) { 1179 #ifdef COMPAT_FREEBSD12 1180 case CRIOGET: 1181 /* 1182 * NB: This may fail in cases that the old 1183 * implementation did not if the current process has 1184 * restricted filesystem access (e.g. running in a 1185 * jail that does not expose /dev/crypto or in 1186 * capability mode). 1187 */ 1188 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE, 1189 O_RDWR, 0); 1190 if (error == 0) 1191 *(uint32_t *)data = td->td_retval[0]; 1192 break; 1193 #endif 1194 case CIOCGSESSION: 1195 case CIOCGSESSION2: 1196 if (cmd == CIOCGSESSION) { 1197 session2_op_from_op((void *)data, &thunk.sopc); 1198 sop = &thunk.sopc; 1199 } else 1200 sop = (struct session2_op *)data; 1201 1202 error = cse_create(fcr, sop); 1203 if (cmd == CIOCGSESSION && error == 0) 1204 session2_op_to_op(sop, (void *)data); 1205 break; 1206 case CIOCFSESSION: 1207 ses = *(uint32_t *)data; 1208 if (!cse_delete(fcr, ses)) { 1209 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1210 return (EINVAL); 1211 } 1212 break; 1213 case CIOCCRYPT: 1214 cop = (struct crypt_op *)data; 1215 cse = cse_find(fcr, cop->ses); 1216 if (cse == NULL) { 1217 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1218 return (EINVAL); 1219 } 1220 error = cryptodev_op(cse, cop); 1221 cse_free(cse); 1222 break; 1223 case CIOCFINDDEV: 1224 error = cryptodev_find((struct crypt_find_op *)data); 1225 break; 1226 case CIOCCRYPTAEAD: 1227 caead = (struct crypt_aead *)data; 1228 cse = cse_find(fcr, caead->ses); 1229 if (cse == NULL) { 1230 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1231 return (EINVAL); 1232 } 1233 error = cryptodev_aead(cse, caead); 1234 cse_free(cse); 1235 break; 1236 default: 1237 error = EINVAL; 1238 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1239 break; 1240 } 1241 1242 #ifdef COMPAT_FREEBSD32 1243 switch (cmd32) { 1244 case CIOCGSESSION32: 1245 if (error == 0) 1246 session_op_to_32((void *)data, data32); 1247 break; 1248 case CIOCGSESSION232: 1249 if (error == 0) 1250 session2_op_to_32((void *)data, data32); 1251 break; 1252 case CIOCCRYPT32: 1253 if (error == 0) 1254 crypt_op_to_32((void *)data, data32); 1255 break; 1256 case CIOCCRYPTAEAD32: 1257 if (error == 0) 1258 crypt_aead_to_32((void *)data, data32); 1259 break; 1260 } 1261 #endif 1262 return (error); 1263 } 1264 1265 static struct cdevsw crypto_cdevsw = { 1266 .d_version = D_VERSION, 1267 .d_open = crypto_open, 1268 .d_ioctl = crypto_ioctl, 1269 .d_name = "crypto", 1270 }; 1271 static struct cdev *crypto_dev; 1272 1273 /* 1274 * Initialization code, both for static and dynamic loading. 1275 */ 1276 static int 1277 cryptodev_modevent(module_t mod, int type, void *unused) 1278 { 1279 switch (type) { 1280 case MOD_LOAD: 1281 if (bootverbose) 1282 printf("crypto: <crypto device>\n"); 1283 crypto_dev = make_dev(&crypto_cdevsw, 0, 1284 UID_ROOT, GID_WHEEL, 0666, 1285 "crypto"); 1286 return 0; 1287 case MOD_UNLOAD: 1288 /*XXX disallow if active sessions */ 1289 destroy_dev(crypto_dev); 1290 return 0; 1291 } 1292 return EINVAL; 1293 } 1294 1295 static moduledata_t cryptodev_mod = { 1296 "cryptodev", 1297 cryptodev_modevent, 1298 0 1299 }; 1300 MODULE_VERSION(cryptodev, 1); 1301 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1302 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1303 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1304