1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/endian.h> 48 #include <sys/limits.h> 49 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform.h> 55 56 #include <sys/kobj.h> 57 #include <sys/bus.h> 58 #include "cryptodev_if.h" 59 60 struct swcr_auth { 61 void *sw_ictx; 62 void *sw_octx; 63 const struct auth_hash *sw_axf; 64 uint16_t sw_mlen; 65 bool sw_hmac; 66 }; 67 68 struct swcr_encdec { 69 void *sw_ctx; 70 const struct enc_xform *sw_exf; 71 }; 72 73 struct swcr_compdec { 74 const struct comp_algo *sw_cxf; 75 }; 76 77 struct swcr_session { 78 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79 80 struct swcr_auth swcr_auth; 81 struct swcr_encdec swcr_encdec; 82 struct swcr_compdec swcr_compdec; 83 }; 84 85 static int32_t swcr_id; 86 87 static void swcr_freesession(device_t dev, crypto_session_t cses); 88 89 /* Used for CRYPTO_NULL_CBC. */ 90 static int 91 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92 { 93 94 return (0); 95 } 96 97 /* 98 * Apply a symmetric encryption/decryption algorithm. 99 */ 100 static int 101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102 { 103 unsigned char blk[EALG_MAX_BLOCK_LEN]; 104 const struct crypto_session_params *csp; 105 const struct enc_xform *exf; 106 const struct swcr_encdec *sw; 107 void *ctx; 108 size_t inlen, outlen, todo; 109 int blksz, resid; 110 struct crypto_buffer_cursor cc_in, cc_out; 111 const unsigned char *inblk; 112 unsigned char *outblk; 113 int error; 114 bool encrypting; 115 116 error = 0; 117 118 sw = &ses->swcr_encdec; 119 exf = sw->sw_exf; 120 csp = crypto_get_params(crp->crp_session); 121 122 if (exf->native_blocksize == 0) { 123 /* Check for non-padded data */ 124 if ((crp->crp_payload_length % exf->blocksize) != 0) 125 return (EINVAL); 126 127 blksz = exf->blocksize; 128 } else 129 blksz = exf->native_blocksize; 130 131 if (exf == &enc_xform_aes_icm && 132 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 133 return (EINVAL); 134 135 ctx = __builtin_alloca(exf->ctxsize); 136 if (crp->crp_cipher_key != NULL) { 137 error = exf->setkey(ctx, crp->crp_cipher_key, 138 csp->csp_cipher_klen); 139 if (error) 140 return (error); 141 } else 142 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143 144 crypto_read_iv(crp, blk); 145 exf->reinit(ctx, blk, csp->csp_ivlen); 146 147 crypto_cursor_init(&cc_in, &crp->crp_buf); 148 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 149 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 150 crypto_cursor_init(&cc_out, &crp->crp_obuf); 151 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 152 } else 153 cc_out = cc_in; 154 155 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 156 157 /* 158 * Loop through encrypting blocks. 'inlen' is the remaining 159 * length of the current segment in the input buffer. 160 * 'outlen' is the remaining length of current segment in the 161 * output buffer. 162 */ 163 inlen = outlen = 0; 164 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 165 if (inlen == 0) 166 inblk = crypto_cursor_segment(&cc_in, &inlen); 167 if (outlen == 0) 168 outblk = crypto_cursor_segment(&cc_out, &outlen); 169 170 /* 171 * If the current block is not contained within the 172 * current input/output segment, use 'blk' as a local 173 * buffer. 174 */ 175 if (inlen < blksz) { 176 crypto_cursor_copydata(&cc_in, blksz, blk); 177 inblk = blk; 178 inlen = blksz; 179 } 180 if (outlen < blksz) { 181 outblk = blk; 182 outlen = blksz; 183 } 184 185 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz); 186 187 if (encrypting) 188 exf->encrypt_multi(ctx, inblk, outblk, todo); 189 else 190 exf->decrypt_multi(ctx, inblk, outblk, todo); 191 192 if (inblk == blk) { 193 inblk = crypto_cursor_segment(&cc_in, &inlen); 194 } else { 195 crypto_cursor_advance(&cc_in, todo); 196 inlen -= todo; 197 inblk += todo; 198 } 199 200 if (outblk == blk) { 201 crypto_cursor_copyback(&cc_out, blksz, blk); 202 outblk = crypto_cursor_segment(&cc_out, &outlen); 203 } else { 204 crypto_cursor_advance(&cc_out, todo); 205 outlen -= todo; 206 outblk += todo; 207 } 208 } 209 210 /* Handle trailing partial block for stream ciphers. */ 211 if (resid > 0) { 212 KASSERT(exf->native_blocksize != 0, 213 ("%s: partial block of %d bytes for cipher %s", 214 __func__, resid, exf->name)); 215 KASSERT(resid < blksz, ("%s: partial block too big", __func__)); 216 217 inblk = crypto_cursor_segment(&cc_in, &inlen); 218 outblk = crypto_cursor_segment(&cc_out, &outlen); 219 if (inlen < resid) { 220 crypto_cursor_copydata(&cc_in, resid, blk); 221 inblk = blk; 222 } 223 if (outlen < resid) 224 outblk = blk; 225 if (encrypting) 226 exf->encrypt_last(ctx, inblk, outblk, 227 resid); 228 else 229 exf->decrypt_last(ctx, inblk, outblk, 230 resid); 231 if (outlen < resid) 232 crypto_cursor_copyback(&cc_out, resid, blk); 233 } 234 235 explicit_bzero(ctx, exf->ctxsize); 236 explicit_bzero(blk, sizeof(blk)); 237 return (0); 238 } 239 240 /* 241 * Compute or verify hash. 242 */ 243 static int 244 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 245 { 246 struct { 247 union authctx ctx; 248 u_char aalg[HASH_MAX_LEN]; 249 u_char uaalg[HASH_MAX_LEN]; 250 } s; 251 const struct crypto_session_params *csp; 252 const struct swcr_auth *sw; 253 const struct auth_hash *axf; 254 int err; 255 256 sw = &ses->swcr_auth; 257 258 axf = sw->sw_axf; 259 260 csp = crypto_get_params(crp->crp_session); 261 if (crp->crp_auth_key != NULL) { 262 if (sw->sw_hmac) { 263 hmac_init_ipad(axf, crp->crp_auth_key, 264 csp->csp_auth_klen, &s.ctx); 265 } else { 266 axf->Init(&s.ctx); 267 axf->Setkey(&s.ctx, crp->crp_auth_key, 268 csp->csp_auth_klen); 269 } 270 } else 271 memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize); 272 273 if (crp->crp_aad != NULL) 274 err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length); 275 else 276 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 277 axf->Update, &s.ctx); 278 if (err) 279 goto out; 280 281 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 282 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 283 err = crypto_apply_buf(&crp->crp_obuf, 284 crp->crp_payload_output_start, crp->crp_payload_length, 285 axf->Update, &s.ctx); 286 else 287 err = crypto_apply(crp, crp->crp_payload_start, 288 crp->crp_payload_length, axf->Update, &s.ctx); 289 if (err) 290 goto out; 291 292 if (csp->csp_flags & CSP_F_ESN) 293 axf->Update(&s.ctx, crp->crp_esn, 4); 294 295 axf->Final(s.aalg, &s.ctx); 296 if (sw->sw_hmac) { 297 if (crp->crp_auth_key != NULL) 298 hmac_init_opad(axf, crp->crp_auth_key, 299 csp->csp_auth_klen, &s.ctx); 300 else 301 memcpy(&s.ctx, sw->sw_octx, axf->ctxsize); 302 axf->Update(&s.ctx, s.aalg, axf->hashsize); 303 axf->Final(s.aalg, &s.ctx); 304 } 305 306 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 307 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg); 308 if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0) 309 err = EBADMSG; 310 } else { 311 /* Inject the authentication data */ 312 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg); 313 } 314 out: 315 explicit_bzero(&s, sizeof(s)); 316 return (err); 317 } 318 319 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 320 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 321 322 static int 323 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 324 { 325 struct { 326 union authctx ctx; 327 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 328 u_char tag[GMAC_DIGEST_LEN]; 329 u_char tag2[GMAC_DIGEST_LEN]; 330 } s; 331 u_char *blk = (u_char *)s.blkbuf; 332 struct crypto_buffer_cursor cc; 333 const u_char *inblk; 334 const struct swcr_auth *swa; 335 const struct auth_hash *axf; 336 uint32_t *blkp; 337 size_t len; 338 int blksz, error, ivlen, resid; 339 340 swa = &ses->swcr_auth; 341 axf = swa->sw_axf; 342 blksz = GMAC_BLOCK_LEN; 343 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 344 __func__)); 345 346 if (crp->crp_auth_key != NULL) { 347 axf->Init(&s.ctx); 348 axf->Setkey(&s.ctx, crp->crp_auth_key, 349 crypto_get_params(crp->crp_session)->csp_auth_klen); 350 } else 351 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 352 353 /* Initialize the IV */ 354 ivlen = AES_GCM_IV_LEN; 355 crypto_read_iv(crp, blk); 356 357 axf->Reinit(&s.ctx, blk, ivlen); 358 crypto_cursor_init(&cc, &crp->crp_buf); 359 crypto_cursor_advance(&cc, crp->crp_payload_start); 360 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 361 inblk = crypto_cursor_segment(&cc, &len); 362 if (len >= blksz) { 363 len = rounddown(MIN(len, resid), blksz); 364 crypto_cursor_advance(&cc, len); 365 } else { 366 len = blksz; 367 crypto_cursor_copydata(&cc, len, blk); 368 inblk = blk; 369 } 370 axf->Update(&s.ctx, inblk, len); 371 } 372 if (resid > 0) { 373 memset(blk, 0, blksz); 374 crypto_cursor_copydata(&cc, resid, blk); 375 axf->Update(&s.ctx, blk, blksz); 376 } 377 378 /* length block */ 379 memset(blk, 0, blksz); 380 blkp = (uint32_t *)blk + 1; 381 *blkp = htobe32(crp->crp_payload_length * 8); 382 axf->Update(&s.ctx, blk, blksz); 383 384 /* Finalize MAC */ 385 axf->Final(s.tag, &s.ctx); 386 387 error = 0; 388 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 389 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 390 s.tag2); 391 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 392 error = EBADMSG; 393 } else { 394 /* Inject the authentication data */ 395 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag); 396 } 397 explicit_bzero(&s, sizeof(s)); 398 return (error); 399 } 400 401 static int 402 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 403 { 404 struct { 405 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 406 u_char tag[GMAC_DIGEST_LEN]; 407 u_char tag2[GMAC_DIGEST_LEN]; 408 } s; 409 u_char *blk = (u_char *)s.blkbuf; 410 struct crypto_buffer_cursor cc_in, cc_out; 411 const u_char *inblk; 412 u_char *outblk; 413 size_t inlen, outlen, todo; 414 const struct swcr_auth *swa; 415 const struct swcr_encdec *swe; 416 const struct enc_xform *exf; 417 void *ctx; 418 uint32_t *blkp; 419 int blksz, error, ivlen, r, resid; 420 421 swa = &ses->swcr_auth; 422 swe = &ses->swcr_encdec; 423 exf = swe->sw_exf; 424 blksz = GMAC_BLOCK_LEN; 425 KASSERT(blksz == exf->native_blocksize, 426 ("%s: blocksize mismatch", __func__)); 427 428 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 429 return (EINVAL); 430 431 ivlen = AES_GCM_IV_LEN; 432 433 ctx = __builtin_alloca(exf->ctxsize); 434 if (crp->crp_cipher_key != NULL) 435 exf->setkey(ctx, crp->crp_cipher_key, 436 crypto_get_params(crp->crp_session)->csp_cipher_klen); 437 else 438 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 439 exf->reinit(ctx, crp->crp_iv, ivlen); 440 441 /* Supply MAC with AAD */ 442 if (crp->crp_aad != NULL) { 443 inlen = rounddown2(crp->crp_aad_length, blksz); 444 if (inlen != 0) 445 exf->update(ctx, crp->crp_aad, inlen); 446 if (crp->crp_aad_length != inlen) { 447 memset(blk, 0, blksz); 448 memcpy(blk, (char *)crp->crp_aad + inlen, 449 crp->crp_aad_length - inlen); 450 exf->update(ctx, blk, blksz); 451 } 452 } else { 453 crypto_cursor_init(&cc_in, &crp->crp_buf); 454 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 455 for (resid = crp->crp_aad_length; resid >= blksz; 456 resid -= inlen) { 457 inblk = crypto_cursor_segment(&cc_in, &inlen); 458 if (inlen >= blksz) { 459 inlen = rounddown2(MIN(inlen, resid), blksz); 460 crypto_cursor_advance(&cc_in, inlen); 461 } else { 462 inlen = blksz; 463 crypto_cursor_copydata(&cc_in, inlen, blk); 464 inblk = blk; 465 } 466 exf->update(ctx, inblk, inlen); 467 } 468 if (resid > 0) { 469 memset(blk, 0, blksz); 470 crypto_cursor_copydata(&cc_in, resid, blk); 471 exf->update(ctx, blk, blksz); 472 } 473 } 474 475 /* Do encryption with MAC */ 476 crypto_cursor_init(&cc_in, &crp->crp_buf); 477 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 478 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 479 crypto_cursor_init(&cc_out, &crp->crp_obuf); 480 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 481 } else 482 cc_out = cc_in; 483 484 inlen = outlen = 0; 485 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 486 if (inlen == 0) 487 inblk = crypto_cursor_segment(&cc_in, &inlen); 488 if (outlen == 0) 489 outblk = crypto_cursor_segment(&cc_out, &outlen); 490 491 if (inlen < blksz) { 492 crypto_cursor_copydata(&cc_in, blksz, blk); 493 inblk = blk; 494 inlen = blksz; 495 } 496 497 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 498 if (outlen < blksz) { 499 outblk = blk; 500 outlen = blksz; 501 } 502 503 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 504 blksz); 505 506 exf->encrypt_multi(ctx, inblk, outblk, todo); 507 exf->update(ctx, outblk, todo); 508 509 if (outblk == blk) { 510 crypto_cursor_copyback(&cc_out, blksz, blk); 511 outblk = crypto_cursor_segment(&cc_out, &outlen); 512 } else { 513 crypto_cursor_advance(&cc_out, todo); 514 outlen -= todo; 515 outblk += todo; 516 } 517 } else { 518 todo = rounddown2(MIN(resid, inlen), blksz); 519 exf->update(ctx, inblk, todo); 520 } 521 522 if (inblk == blk) { 523 inblk = crypto_cursor_segment(&cc_in, &inlen); 524 } else { 525 crypto_cursor_advance(&cc_in, todo); 526 inlen -= todo; 527 inblk += todo; 528 } 529 } 530 if (resid > 0) { 531 crypto_cursor_copydata(&cc_in, resid, blk); 532 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 533 exf->encrypt_last(ctx, blk, blk, resid); 534 crypto_cursor_copyback(&cc_out, resid, blk); 535 } 536 exf->update(ctx, blk, resid); 537 } 538 539 /* length block */ 540 memset(blk, 0, blksz); 541 blkp = (uint32_t *)blk + 1; 542 *blkp = htobe32(crp->crp_aad_length * 8); 543 blkp = (uint32_t *)blk + 3; 544 *blkp = htobe32(crp->crp_payload_length * 8); 545 exf->update(ctx, blk, blksz); 546 547 /* Finalize MAC */ 548 exf->final(s.tag, ctx); 549 550 /* Validate tag */ 551 error = 0; 552 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 553 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 554 s.tag2); 555 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 556 if (r != 0) { 557 error = EBADMSG; 558 goto out; 559 } 560 561 /* tag matches, decrypt data */ 562 crypto_cursor_init(&cc_in, &crp->crp_buf); 563 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 564 565 inlen = 0; 566 for (resid = crp->crp_payload_length; resid > blksz; 567 resid -= todo) { 568 if (inlen == 0) 569 inblk = crypto_cursor_segment(&cc_in, &inlen); 570 if (outlen == 0) 571 outblk = crypto_cursor_segment(&cc_out, &outlen); 572 if (inlen < blksz) { 573 crypto_cursor_copydata(&cc_in, blksz, blk); 574 inblk = blk; 575 inlen = blksz; 576 } 577 if (outlen < blksz) { 578 outblk = blk; 579 outlen = blksz; 580 } 581 582 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 583 blksz); 584 585 exf->decrypt_multi(ctx, inblk, outblk, todo); 586 587 if (inblk == blk) { 588 inblk = crypto_cursor_segment(&cc_in, &inlen); 589 } else { 590 crypto_cursor_advance(&cc_in, todo); 591 inlen -= todo; 592 inblk += todo; 593 } 594 595 if (outblk == blk) { 596 crypto_cursor_copyback(&cc_out, blksz, blk); 597 outblk = crypto_cursor_segment(&cc_out, 598 &outlen); 599 } else { 600 crypto_cursor_advance(&cc_out, todo); 601 outlen -= todo; 602 outblk += todo; 603 } 604 } 605 if (resid > 0) { 606 crypto_cursor_copydata(&cc_in, resid, blk); 607 exf->decrypt_last(ctx, blk, blk, resid); 608 crypto_cursor_copyback(&cc_out, resid, blk); 609 } 610 } else { 611 /* Inject the authentication data */ 612 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 613 s.tag); 614 } 615 616 out: 617 explicit_bzero(ctx, exf->ctxsize); 618 explicit_bzero(&s, sizeof(s)); 619 620 return (error); 621 } 622 623 static void 624 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 625 u_int data_length, u_int tag_length, uint8_t *b0) 626 { 627 uint8_t *bp; 628 uint8_t flags, L; 629 630 KASSERT(nonce_length >= 7 && nonce_length <= 13, 631 ("nonce_length must be between 7 and 13 bytes")); 632 633 /* 634 * Need to determine the L field value. This is the number of 635 * bytes needed to specify the length of the message; the length 636 * is whatever is left in the 16 bytes after specifying flags and 637 * the nonce. 638 */ 639 L = 15 - nonce_length; 640 641 flags = ((aad_length > 0) << 6) + 642 (((tag_length - 2) / 2) << 3) + 643 L - 1; 644 645 /* 646 * Now we need to set up the first block, which has flags, nonce, 647 * and the message length. 648 */ 649 b0[0] = flags; 650 memcpy(b0 + 1, nonce, nonce_length); 651 bp = b0 + 1 + nonce_length; 652 653 /* Need to copy L' [aka L-1] bytes of data_length */ 654 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 655 *dst = data_length; 656 data_length >>= 8; 657 } 658 } 659 660 /* NB: OCF only supports AAD lengths < 2^32. */ 661 static int 662 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 663 { 664 if (aad_length < ((1 << 16) - (1 << 8))) { 665 be16enc(blk, aad_length); 666 return (sizeof(uint16_t)); 667 } else { 668 blk[0] = 0xff; 669 blk[1] = 0xfe; 670 be32enc(blk + 2, aad_length); 671 return (2 + sizeof(uint32_t)); 672 } 673 } 674 675 static int 676 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 677 { 678 struct { 679 union authctx ctx; 680 u_char blk[CCM_CBC_BLOCK_LEN]; 681 u_char tag[AES_CBC_MAC_HASH_LEN]; 682 u_char tag2[AES_CBC_MAC_HASH_LEN]; 683 } s; 684 const struct crypto_session_params *csp; 685 const struct swcr_auth *swa; 686 const struct auth_hash *axf; 687 int error, ivlen, len; 688 689 csp = crypto_get_params(crp->crp_session); 690 swa = &ses->swcr_auth; 691 axf = swa->sw_axf; 692 693 if (crp->crp_auth_key != NULL) { 694 axf->Init(&s.ctx); 695 axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen); 696 } else 697 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 698 699 /* Initialize the IV */ 700 ivlen = csp->csp_ivlen; 701 702 /* Supply MAC with IV */ 703 axf->Reinit(&s.ctx, crp->crp_iv, ivlen); 704 705 /* Supply MAC with b0. */ 706 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 707 swa->sw_mlen, s.blk); 708 axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN); 709 710 len = build_ccm_aad_length(crp->crp_payload_length, s.blk); 711 axf->Update(&s.ctx, s.blk, len); 712 713 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 714 axf->Update, &s.ctx); 715 716 /* Finalize MAC */ 717 axf->Final(s.tag, &s.ctx); 718 719 error = 0; 720 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 721 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 722 s.tag2); 723 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 724 error = EBADMSG; 725 } else { 726 /* Inject the authentication data */ 727 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 728 s.tag); 729 } 730 explicit_bzero(&s, sizeof(s)); 731 return (error); 732 } 733 734 static int 735 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 736 { 737 const struct crypto_session_params *csp; 738 struct { 739 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 740 u_char tag[AES_CBC_MAC_HASH_LEN]; 741 u_char tag2[AES_CBC_MAC_HASH_LEN]; 742 } s; 743 u_char *blk = (u_char *)s.blkbuf; 744 struct crypto_buffer_cursor cc_in, cc_out; 745 const u_char *inblk; 746 u_char *outblk; 747 size_t inlen, outlen, todo; 748 const struct swcr_auth *swa; 749 const struct swcr_encdec *swe; 750 const struct enc_xform *exf; 751 void *ctx; 752 size_t len; 753 int blksz, error, ivlen, r, resid; 754 755 csp = crypto_get_params(crp->crp_session); 756 swa = &ses->swcr_auth; 757 swe = &ses->swcr_encdec; 758 exf = swe->sw_exf; 759 blksz = AES_BLOCK_LEN; 760 KASSERT(blksz == exf->native_blocksize, 761 ("%s: blocksize mismatch", __func__)); 762 763 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 764 return (EMSGSIZE); 765 766 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 767 return (EINVAL); 768 769 ivlen = csp->csp_ivlen; 770 771 ctx = __builtin_alloca(exf->ctxsize); 772 if (crp->crp_cipher_key != NULL) 773 exf->setkey(ctx, crp->crp_cipher_key, 774 crypto_get_params(crp->crp_session)->csp_cipher_klen); 775 else 776 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 777 exf->reinit(ctx, crp->crp_iv, ivlen); 778 779 /* Supply MAC with b0. */ 780 _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN, 781 "blkbuf too small for b0"); 782 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 783 crp->crp_payload_length, swa->sw_mlen, blk); 784 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 785 786 /* Supply MAC with AAD */ 787 if (crp->crp_aad_length != 0) { 788 len = build_ccm_aad_length(crp->crp_aad_length, blk); 789 exf->update(ctx, blk, len); 790 if (crp->crp_aad != NULL) 791 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 792 else 793 crypto_apply(crp, crp->crp_aad_start, 794 crp->crp_aad_length, exf->update, ctx); 795 796 /* Pad the AAD (including length field) to a full block. */ 797 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 798 if (len != 0) { 799 len = CCM_CBC_BLOCK_LEN - len; 800 memset(blk, 0, CCM_CBC_BLOCK_LEN); 801 exf->update(ctx, blk, len); 802 } 803 } 804 805 /* Do encryption/decryption with MAC */ 806 crypto_cursor_init(&cc_in, &crp->crp_buf); 807 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 808 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 809 crypto_cursor_init(&cc_out, &crp->crp_obuf); 810 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 811 } else 812 cc_out = cc_in; 813 814 inlen = outlen = 0; 815 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 816 if (inlen == 0) 817 inblk = crypto_cursor_segment(&cc_in, &inlen); 818 if (outlen == 0) 819 outblk = crypto_cursor_segment(&cc_out, &outlen); 820 821 if (inlen < blksz) { 822 crypto_cursor_copydata(&cc_in, blksz, blk); 823 inblk = blk; 824 inlen = blksz; 825 } 826 827 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 828 if (outlen < blksz) { 829 outblk = blk; 830 outlen = blksz; 831 } 832 833 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 834 blksz); 835 836 exf->update(ctx, inblk, todo); 837 exf->encrypt_multi(ctx, inblk, outblk, todo); 838 839 if (outblk == blk) { 840 crypto_cursor_copyback(&cc_out, blksz, blk); 841 outblk = crypto_cursor_segment(&cc_out, &outlen); 842 } else { 843 crypto_cursor_advance(&cc_out, todo); 844 outlen -= todo; 845 outblk += todo; 846 } 847 } else { 848 /* 849 * One of the problems with CCM+CBC is that 850 * the authentication is done on the 851 * unencrypted data. As a result, we have to 852 * decrypt the data twice: once to generate 853 * the tag and a second time after the tag is 854 * verified. 855 */ 856 todo = blksz; 857 exf->decrypt(ctx, inblk, blk); 858 exf->update(ctx, blk, todo); 859 } 860 861 if (inblk == blk) { 862 inblk = crypto_cursor_segment(&cc_in, &inlen); 863 } else { 864 crypto_cursor_advance(&cc_in, todo); 865 inlen -= todo; 866 inblk += todo; 867 } 868 } 869 if (resid > 0) { 870 crypto_cursor_copydata(&cc_in, resid, blk); 871 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 872 exf->update(ctx, blk, resid); 873 exf->encrypt_last(ctx, blk, blk, resid); 874 crypto_cursor_copyback(&cc_out, resid, blk); 875 } else { 876 exf->decrypt_last(ctx, blk, blk, resid); 877 exf->update(ctx, blk, resid); 878 } 879 } 880 881 /* Finalize MAC */ 882 exf->final(s.tag, ctx); 883 884 /* Validate tag */ 885 error = 0; 886 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 887 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 888 s.tag2); 889 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 890 if (r != 0) { 891 error = EBADMSG; 892 goto out; 893 } 894 895 /* tag matches, decrypt data */ 896 exf->reinit(ctx, crp->crp_iv, ivlen); 897 crypto_cursor_init(&cc_in, &crp->crp_buf); 898 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 899 900 inlen = 0; 901 for (resid = crp->crp_payload_length; resid >= blksz; 902 resid -= todo) { 903 if (inlen == 0) 904 inblk = crypto_cursor_segment(&cc_in, &inlen); 905 if (outlen == 0) 906 outblk = crypto_cursor_segment(&cc_out, 907 &outlen); 908 909 if (inlen < blksz) { 910 crypto_cursor_copydata(&cc_in, blksz, blk); 911 inblk = blk; 912 inlen = blksz; 913 } 914 if (outlen < blksz) { 915 outblk = blk; 916 outlen = blksz; 917 } 918 919 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 920 blksz); 921 922 exf->decrypt_multi(ctx, inblk, outblk, todo); 923 924 if (inblk == blk) { 925 inblk = crypto_cursor_segment(&cc_in, &inlen); 926 } else { 927 crypto_cursor_advance(&cc_in, todo); 928 inlen -= todo; 929 inblk += todo; 930 } 931 932 if (outblk == blk) { 933 crypto_cursor_copyback(&cc_out, blksz, blk); 934 outblk = crypto_cursor_segment(&cc_out, 935 &outlen); 936 } else { 937 crypto_cursor_advance(&cc_out, todo); 938 outlen -= todo; 939 outblk += todo; 940 } 941 } 942 if (resid > 0) { 943 crypto_cursor_copydata(&cc_in, resid, blk); 944 exf->decrypt_last(ctx, blk, blk, resid); 945 crypto_cursor_copyback(&cc_out, resid, blk); 946 } 947 } else { 948 /* Inject the authentication data */ 949 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 950 s.tag); 951 } 952 953 out: 954 explicit_bzero(ctx, exf->ctxsize); 955 explicit_bzero(&s, sizeof(s)); 956 return (error); 957 } 958 959 static int 960 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 961 { 962 const struct crypto_session_params *csp; 963 struct { 964 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 965 u_char tag[POLY1305_HASH_LEN]; 966 u_char tag2[POLY1305_HASH_LEN]; 967 } s; 968 u_char *blk = (u_char *)s.blkbuf; 969 struct crypto_buffer_cursor cc_in, cc_out; 970 const u_char *inblk; 971 u_char *outblk; 972 size_t inlen, outlen, todo; 973 uint64_t *blkp; 974 const struct swcr_auth *swa; 975 const struct swcr_encdec *swe; 976 const struct enc_xform *exf; 977 void *ctx; 978 int blksz, error, r, resid; 979 980 swa = &ses->swcr_auth; 981 swe = &ses->swcr_encdec; 982 exf = swe->sw_exf; 983 blksz = exf->native_blocksize; 984 KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__)); 985 986 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 987 return (EINVAL); 988 989 csp = crypto_get_params(crp->crp_session); 990 991 ctx = __builtin_alloca(exf->ctxsize); 992 if (crp->crp_cipher_key != NULL) 993 exf->setkey(ctx, crp->crp_cipher_key, 994 csp->csp_cipher_klen); 995 else 996 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 997 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 998 999 /* Supply MAC with AAD */ 1000 if (crp->crp_aad != NULL) 1001 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 1002 else 1003 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 1004 exf->update, ctx); 1005 if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) { 1006 /* padding1 */ 1007 memset(blk, 0, POLY1305_BLOCK_LEN); 1008 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1009 crp->crp_aad_length % POLY1305_BLOCK_LEN); 1010 } 1011 1012 /* Do encryption with MAC */ 1013 crypto_cursor_init(&cc_in, &crp->crp_buf); 1014 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1015 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1016 crypto_cursor_init(&cc_out, &crp->crp_obuf); 1017 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1018 } else 1019 cc_out = cc_in; 1020 1021 inlen = outlen = 0; 1022 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1023 for (resid = crp->crp_payload_length; resid >= blksz; 1024 resid -= todo) { 1025 if (inlen == 0) 1026 inblk = crypto_cursor_segment(&cc_in, &inlen); 1027 if (outlen == 0) 1028 outblk = crypto_cursor_segment(&cc_out, 1029 &outlen); 1030 1031 if (inlen < blksz) { 1032 crypto_cursor_copydata(&cc_in, blksz, blk); 1033 inblk = blk; 1034 inlen = blksz; 1035 } 1036 1037 if (outlen < blksz) { 1038 outblk = blk; 1039 outlen = blksz; 1040 } 1041 1042 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 1043 blksz); 1044 1045 exf->encrypt_multi(ctx, inblk, outblk, todo); 1046 exf->update(ctx, outblk, todo); 1047 1048 if (inblk == blk) { 1049 inblk = crypto_cursor_segment(&cc_in, &inlen); 1050 } else { 1051 crypto_cursor_advance(&cc_in, todo); 1052 inlen -= todo; 1053 inblk += todo; 1054 } 1055 1056 if (outblk == blk) { 1057 crypto_cursor_copyback(&cc_out, blksz, blk); 1058 outblk = crypto_cursor_segment(&cc_out, &outlen); 1059 } else { 1060 crypto_cursor_advance(&cc_out, todo); 1061 outlen -= todo; 1062 outblk += todo; 1063 } 1064 } 1065 if (resid > 0) { 1066 crypto_cursor_copydata(&cc_in, resid, blk); 1067 exf->encrypt_last(ctx, blk, blk, resid); 1068 crypto_cursor_copyback(&cc_out, resid, blk); 1069 exf->update(ctx, blk, resid); 1070 } 1071 } else 1072 crypto_apply(crp, crp->crp_payload_start, 1073 crp->crp_payload_length, exf->update, ctx); 1074 if (crp->crp_payload_length % POLY1305_BLOCK_LEN != 0) { 1075 /* padding2 */ 1076 memset(blk, 0, POLY1305_BLOCK_LEN); 1077 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1078 crp->crp_payload_length % POLY1305_BLOCK_LEN); 1079 } 1080 1081 /* lengths */ 1082 blkp = (uint64_t *)blk; 1083 blkp[0] = htole64(crp->crp_aad_length); 1084 blkp[1] = htole64(crp->crp_payload_length); 1085 exf->update(ctx, blk, sizeof(uint64_t) * 2); 1086 1087 /* Finalize MAC */ 1088 exf->final(s.tag, ctx); 1089 1090 /* Validate tag */ 1091 error = 0; 1092 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1093 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 1094 s.tag2); 1095 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 1096 if (r != 0) { 1097 error = EBADMSG; 1098 goto out; 1099 } 1100 1101 /* tag matches, decrypt data */ 1102 crypto_cursor_init(&cc_in, &crp->crp_buf); 1103 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1104 1105 inlen = 0; 1106 for (resid = crp->crp_payload_length; resid > blksz; 1107 resid -= todo) { 1108 if (inlen == 0) 1109 inblk = crypto_cursor_segment(&cc_in, &inlen); 1110 if (outlen == 0) 1111 outblk = crypto_cursor_segment(&cc_out, 1112 &outlen); 1113 if (inlen < blksz) { 1114 crypto_cursor_copydata(&cc_in, blksz, blk); 1115 inblk = blk; 1116 inlen = blksz; 1117 } 1118 if (outlen < blksz) { 1119 outblk = blk; 1120 outlen = blksz; 1121 } 1122 1123 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 1124 blksz); 1125 1126 exf->decrypt_multi(ctx, inblk, outblk, todo); 1127 1128 if (inblk == blk) { 1129 inblk = crypto_cursor_segment(&cc_in, &inlen); 1130 } else { 1131 crypto_cursor_advance(&cc_in, todo); 1132 inlen -= todo; 1133 inblk += todo; 1134 } 1135 1136 if (outblk == blk) { 1137 crypto_cursor_copyback(&cc_out, blksz, blk); 1138 outblk = crypto_cursor_segment(&cc_out, 1139 &outlen); 1140 } else { 1141 crypto_cursor_advance(&cc_out, todo); 1142 outlen -= todo; 1143 outblk += todo; 1144 } 1145 } 1146 if (resid > 0) { 1147 crypto_cursor_copydata(&cc_in, resid, blk); 1148 exf->decrypt_last(ctx, blk, blk, resid); 1149 crypto_cursor_copyback(&cc_out, resid, blk); 1150 } 1151 } else { 1152 /* Inject the authentication data */ 1153 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 1154 s.tag); 1155 } 1156 1157 out: 1158 explicit_bzero(ctx, exf->ctxsize); 1159 explicit_bzero(&s, sizeof(s)); 1160 return (error); 1161 } 1162 1163 /* 1164 * Apply a cipher and a digest to perform EtA. 1165 */ 1166 static int 1167 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1168 { 1169 int error; 1170 1171 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1172 error = swcr_encdec(ses, crp); 1173 if (error == 0) 1174 error = swcr_authcompute(ses, crp); 1175 } else { 1176 error = swcr_authcompute(ses, crp); 1177 if (error == 0) 1178 error = swcr_encdec(ses, crp); 1179 } 1180 return (error); 1181 } 1182 1183 /* 1184 * Apply a compression/decompression algorithm 1185 */ 1186 static int 1187 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1188 { 1189 const struct comp_algo *cxf; 1190 uint8_t *data, *out; 1191 int adj; 1192 uint32_t result; 1193 1194 cxf = ses->swcr_compdec.sw_cxf; 1195 1196 /* We must handle the whole buffer of data in one time 1197 * then if there is not all the data in the mbuf, we must 1198 * copy in a buffer. 1199 */ 1200 1201 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1202 if (data == NULL) 1203 return (EINVAL); 1204 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1205 data); 1206 1207 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1208 result = cxf->compress(data, crp->crp_payload_length, &out); 1209 else 1210 result = cxf->decompress(data, crp->crp_payload_length, &out); 1211 1212 free(data, M_CRYPTO_DATA); 1213 if (result == 0) 1214 return (EINVAL); 1215 crp->crp_olen = result; 1216 1217 /* Check the compressed size when doing compression */ 1218 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1219 if (result >= crp->crp_payload_length) { 1220 /* Compression was useless, we lost time */ 1221 free(out, M_CRYPTO_DATA); 1222 return (0); 1223 } 1224 } 1225 1226 /* Copy back the (de)compressed data. m_copyback is 1227 * extending the mbuf as necessary. 1228 */ 1229 crypto_copyback(crp, crp->crp_payload_start, result, out); 1230 if (result < crp->crp_payload_length) { 1231 switch (crp->crp_buf.cb_type) { 1232 case CRYPTO_BUF_MBUF: 1233 case CRYPTO_BUF_SINGLE_MBUF: 1234 adj = result - crp->crp_payload_length; 1235 m_adj(crp->crp_buf.cb_mbuf, adj); 1236 break; 1237 case CRYPTO_BUF_UIO: { 1238 struct uio *uio = crp->crp_buf.cb_uio; 1239 int ind; 1240 1241 adj = crp->crp_payload_length - result; 1242 ind = uio->uio_iovcnt - 1; 1243 1244 while (adj > 0 && ind >= 0) { 1245 if (adj < uio->uio_iov[ind].iov_len) { 1246 uio->uio_iov[ind].iov_len -= adj; 1247 break; 1248 } 1249 1250 adj -= uio->uio_iov[ind].iov_len; 1251 uio->uio_iov[ind].iov_len = 0; 1252 ind--; 1253 uio->uio_iovcnt--; 1254 } 1255 } 1256 break; 1257 case CRYPTO_BUF_VMPAGE: 1258 adj = crp->crp_payload_length - result; 1259 crp->crp_buf.cb_vm_page_len -= adj; 1260 break; 1261 default: 1262 break; 1263 } 1264 } 1265 free(out, M_CRYPTO_DATA); 1266 return 0; 1267 } 1268 1269 static int 1270 swcr_setup_cipher(struct swcr_session *ses, 1271 const struct crypto_session_params *csp) 1272 { 1273 struct swcr_encdec *swe; 1274 const struct enc_xform *txf; 1275 int error; 1276 1277 swe = &ses->swcr_encdec; 1278 txf = crypto_cipher(csp); 1279 if (csp->csp_cipher_key != NULL) { 1280 if (txf->ctxsize != 0) { 1281 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1282 M_NOWAIT); 1283 if (swe->sw_ctx == NULL) 1284 return (ENOMEM); 1285 } 1286 error = txf->setkey(swe->sw_ctx, 1287 csp->csp_cipher_key, csp->csp_cipher_klen); 1288 if (error) 1289 return (error); 1290 } 1291 swe->sw_exf = txf; 1292 return (0); 1293 } 1294 1295 static int 1296 swcr_setup_auth(struct swcr_session *ses, 1297 const struct crypto_session_params *csp) 1298 { 1299 struct swcr_auth *swa; 1300 const struct auth_hash *axf; 1301 1302 swa = &ses->swcr_auth; 1303 1304 axf = crypto_auth_hash(csp); 1305 swa->sw_axf = axf; 1306 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1307 return (EINVAL); 1308 if (csp->csp_auth_mlen == 0) 1309 swa->sw_mlen = axf->hashsize; 1310 else 1311 swa->sw_mlen = csp->csp_auth_mlen; 1312 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1313 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1314 M_NOWAIT); 1315 if (swa->sw_ictx == NULL) 1316 return (ENOBUFS); 1317 } 1318 1319 switch (csp->csp_auth_alg) { 1320 case CRYPTO_SHA1_HMAC: 1321 case CRYPTO_SHA2_224_HMAC: 1322 case CRYPTO_SHA2_256_HMAC: 1323 case CRYPTO_SHA2_384_HMAC: 1324 case CRYPTO_SHA2_512_HMAC: 1325 case CRYPTO_RIPEMD160_HMAC: 1326 swa->sw_hmac = true; 1327 if (csp->csp_auth_key != NULL) { 1328 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1329 M_NOWAIT); 1330 if (swa->sw_octx == NULL) 1331 return (ENOBUFS); 1332 hmac_init_ipad(axf, csp->csp_auth_key, 1333 csp->csp_auth_klen, swa->sw_ictx); 1334 hmac_init_opad(axf, csp->csp_auth_key, 1335 csp->csp_auth_klen, swa->sw_octx); 1336 } 1337 break; 1338 case CRYPTO_RIPEMD160: 1339 case CRYPTO_SHA1: 1340 case CRYPTO_SHA2_224: 1341 case CRYPTO_SHA2_256: 1342 case CRYPTO_SHA2_384: 1343 case CRYPTO_SHA2_512: 1344 case CRYPTO_NULL_HMAC: 1345 axf->Init(swa->sw_ictx); 1346 break; 1347 case CRYPTO_AES_NIST_GMAC: 1348 case CRYPTO_AES_CCM_CBC_MAC: 1349 case CRYPTO_POLY1305: 1350 if (csp->csp_auth_key != NULL) { 1351 axf->Init(swa->sw_ictx); 1352 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1353 csp->csp_auth_klen); 1354 } 1355 break; 1356 case CRYPTO_BLAKE2B: 1357 case CRYPTO_BLAKE2S: 1358 /* 1359 * Blake2b and Blake2s support an optional key but do 1360 * not require one. 1361 */ 1362 if (csp->csp_auth_klen == 0) 1363 axf->Init(swa->sw_ictx); 1364 else if (csp->csp_auth_key != NULL) 1365 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1366 csp->csp_auth_klen); 1367 break; 1368 } 1369 1370 if (csp->csp_mode == CSP_MODE_DIGEST) { 1371 switch (csp->csp_auth_alg) { 1372 case CRYPTO_AES_NIST_GMAC: 1373 ses->swcr_process = swcr_gmac; 1374 break; 1375 case CRYPTO_AES_CCM_CBC_MAC: 1376 ses->swcr_process = swcr_ccm_cbc_mac; 1377 break; 1378 default: 1379 ses->swcr_process = swcr_authcompute; 1380 } 1381 } 1382 1383 return (0); 1384 } 1385 1386 static int 1387 swcr_setup_aead(struct swcr_session *ses, 1388 const struct crypto_session_params *csp) 1389 { 1390 struct swcr_auth *swa; 1391 int error; 1392 1393 error = swcr_setup_cipher(ses, csp); 1394 if (error) 1395 return (error); 1396 1397 swa = &ses->swcr_auth; 1398 if (csp->csp_auth_mlen == 0) 1399 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1400 else 1401 swa->sw_mlen = csp->csp_auth_mlen; 1402 return (0); 1403 } 1404 1405 static bool 1406 swcr_auth_supported(const struct crypto_session_params *csp) 1407 { 1408 const struct auth_hash *axf; 1409 1410 axf = crypto_auth_hash(csp); 1411 if (axf == NULL) 1412 return (false); 1413 switch (csp->csp_auth_alg) { 1414 case CRYPTO_SHA1_HMAC: 1415 case CRYPTO_SHA2_224_HMAC: 1416 case CRYPTO_SHA2_256_HMAC: 1417 case CRYPTO_SHA2_384_HMAC: 1418 case CRYPTO_SHA2_512_HMAC: 1419 case CRYPTO_NULL_HMAC: 1420 case CRYPTO_RIPEMD160_HMAC: 1421 break; 1422 case CRYPTO_AES_NIST_GMAC: 1423 switch (csp->csp_auth_klen * 8) { 1424 case 128: 1425 case 192: 1426 case 256: 1427 break; 1428 default: 1429 return (false); 1430 } 1431 if (csp->csp_auth_key == NULL) 1432 return (false); 1433 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1434 return (false); 1435 break; 1436 case CRYPTO_POLY1305: 1437 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1438 return (false); 1439 break; 1440 case CRYPTO_AES_CCM_CBC_MAC: 1441 switch (csp->csp_auth_klen * 8) { 1442 case 128: 1443 case 192: 1444 case 256: 1445 break; 1446 default: 1447 return (false); 1448 } 1449 if (csp->csp_auth_key == NULL) 1450 return (false); 1451 break; 1452 } 1453 return (true); 1454 } 1455 1456 static bool 1457 swcr_cipher_supported(const struct crypto_session_params *csp) 1458 { 1459 const struct enc_xform *txf; 1460 1461 txf = crypto_cipher(csp); 1462 if (txf == NULL) 1463 return (false); 1464 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1465 txf->ivsize != csp->csp_ivlen) 1466 return (false); 1467 return (true); 1468 } 1469 1470 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1471 1472 static int 1473 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1474 { 1475 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1476 return (EINVAL); 1477 switch (csp->csp_mode) { 1478 case CSP_MODE_COMPRESS: 1479 switch (csp->csp_cipher_alg) { 1480 case CRYPTO_DEFLATE_COMP: 1481 break; 1482 default: 1483 return (EINVAL); 1484 } 1485 break; 1486 case CSP_MODE_CIPHER: 1487 switch (csp->csp_cipher_alg) { 1488 case CRYPTO_AES_NIST_GCM_16: 1489 case CRYPTO_AES_CCM_16: 1490 case CRYPTO_CHACHA20_POLY1305: 1491 case CRYPTO_XCHACHA20_POLY1305: 1492 return (EINVAL); 1493 default: 1494 if (!swcr_cipher_supported(csp)) 1495 return (EINVAL); 1496 break; 1497 } 1498 break; 1499 case CSP_MODE_DIGEST: 1500 if (!swcr_auth_supported(csp)) 1501 return (EINVAL); 1502 break; 1503 case CSP_MODE_AEAD: 1504 switch (csp->csp_cipher_alg) { 1505 case CRYPTO_AES_NIST_GCM_16: 1506 case CRYPTO_AES_CCM_16: 1507 switch (csp->csp_cipher_klen * 8) { 1508 case 128: 1509 case 192: 1510 case 256: 1511 break; 1512 default: 1513 return (EINVAL); 1514 } 1515 break; 1516 case CRYPTO_CHACHA20_POLY1305: 1517 case CRYPTO_XCHACHA20_POLY1305: 1518 break; 1519 default: 1520 return (EINVAL); 1521 } 1522 break; 1523 case CSP_MODE_ETA: 1524 /* AEAD algorithms cannot be used for EtA. */ 1525 switch (csp->csp_cipher_alg) { 1526 case CRYPTO_AES_NIST_GCM_16: 1527 case CRYPTO_AES_CCM_16: 1528 case CRYPTO_CHACHA20_POLY1305: 1529 case CRYPTO_XCHACHA20_POLY1305: 1530 return (EINVAL); 1531 } 1532 switch (csp->csp_auth_alg) { 1533 case CRYPTO_AES_NIST_GMAC: 1534 case CRYPTO_AES_CCM_CBC_MAC: 1535 return (EINVAL); 1536 } 1537 1538 if (!swcr_cipher_supported(csp) || 1539 !swcr_auth_supported(csp)) 1540 return (EINVAL); 1541 break; 1542 default: 1543 return (EINVAL); 1544 } 1545 1546 return (CRYPTODEV_PROBE_SOFTWARE); 1547 } 1548 1549 /* 1550 * Generate a new software session. 1551 */ 1552 static int 1553 swcr_newsession(device_t dev, crypto_session_t cses, 1554 const struct crypto_session_params *csp) 1555 { 1556 struct swcr_session *ses; 1557 const struct comp_algo *cxf; 1558 int error; 1559 1560 ses = crypto_get_driver_session(cses); 1561 1562 error = 0; 1563 switch (csp->csp_mode) { 1564 case CSP_MODE_COMPRESS: 1565 switch (csp->csp_cipher_alg) { 1566 case CRYPTO_DEFLATE_COMP: 1567 cxf = &comp_algo_deflate; 1568 break; 1569 #ifdef INVARIANTS 1570 default: 1571 panic("bad compression algo"); 1572 #endif 1573 } 1574 ses->swcr_compdec.sw_cxf = cxf; 1575 ses->swcr_process = swcr_compdec; 1576 break; 1577 case CSP_MODE_CIPHER: 1578 switch (csp->csp_cipher_alg) { 1579 case CRYPTO_NULL_CBC: 1580 ses->swcr_process = swcr_null; 1581 break; 1582 #ifdef INVARIANTS 1583 case CRYPTO_AES_NIST_GCM_16: 1584 case CRYPTO_AES_CCM_16: 1585 case CRYPTO_CHACHA20_POLY1305: 1586 case CRYPTO_XCHACHA20_POLY1305: 1587 panic("bad cipher algo"); 1588 #endif 1589 default: 1590 error = swcr_setup_cipher(ses, csp); 1591 if (error == 0) 1592 ses->swcr_process = swcr_encdec; 1593 } 1594 break; 1595 case CSP_MODE_DIGEST: 1596 error = swcr_setup_auth(ses, csp); 1597 break; 1598 case CSP_MODE_AEAD: 1599 switch (csp->csp_cipher_alg) { 1600 case CRYPTO_AES_NIST_GCM_16: 1601 error = swcr_setup_aead(ses, csp); 1602 if (error == 0) 1603 ses->swcr_process = swcr_gcm; 1604 break; 1605 case CRYPTO_AES_CCM_16: 1606 error = swcr_setup_aead(ses, csp); 1607 if (error == 0) 1608 ses->swcr_process = swcr_ccm; 1609 break; 1610 case CRYPTO_CHACHA20_POLY1305: 1611 case CRYPTO_XCHACHA20_POLY1305: 1612 error = swcr_setup_aead(ses, csp); 1613 if (error == 0) 1614 ses->swcr_process = swcr_chacha20_poly1305; 1615 break; 1616 #ifdef INVARIANTS 1617 default: 1618 panic("bad aead algo"); 1619 #endif 1620 } 1621 break; 1622 case CSP_MODE_ETA: 1623 #ifdef INVARIANTS 1624 switch (csp->csp_cipher_alg) { 1625 case CRYPTO_AES_NIST_GCM_16: 1626 case CRYPTO_AES_CCM_16: 1627 case CRYPTO_CHACHA20_POLY1305: 1628 case CRYPTO_XCHACHA20_POLY1305: 1629 panic("bad eta cipher algo"); 1630 } 1631 switch (csp->csp_auth_alg) { 1632 case CRYPTO_AES_NIST_GMAC: 1633 case CRYPTO_AES_CCM_CBC_MAC: 1634 panic("bad eta auth algo"); 1635 } 1636 #endif 1637 1638 error = swcr_setup_auth(ses, csp); 1639 if (error) 1640 break; 1641 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1642 /* Effectively degrade to digest mode. */ 1643 ses->swcr_process = swcr_authcompute; 1644 break; 1645 } 1646 1647 error = swcr_setup_cipher(ses, csp); 1648 if (error == 0) 1649 ses->swcr_process = swcr_eta; 1650 break; 1651 default: 1652 error = EINVAL; 1653 } 1654 1655 if (error) 1656 swcr_freesession(dev, cses); 1657 return (error); 1658 } 1659 1660 static void 1661 swcr_freesession(device_t dev, crypto_session_t cses) 1662 { 1663 struct swcr_session *ses; 1664 1665 ses = crypto_get_driver_session(cses); 1666 1667 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1668 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1669 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1670 } 1671 1672 /* 1673 * Process a software request. 1674 */ 1675 static int 1676 swcr_process(device_t dev, struct cryptop *crp, int hint) 1677 { 1678 struct swcr_session *ses; 1679 1680 ses = crypto_get_driver_session(crp->crp_session); 1681 1682 crp->crp_etype = ses->swcr_process(ses, crp); 1683 1684 crypto_done(crp); 1685 return (0); 1686 } 1687 1688 static void 1689 swcr_identify(driver_t *drv, device_t parent) 1690 { 1691 /* NB: order 10 is so we get attached after h/w devices */ 1692 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1693 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1694 panic("cryptosoft: could not attach"); 1695 } 1696 1697 static int 1698 swcr_probe(device_t dev) 1699 { 1700 device_set_desc(dev, "software crypto"); 1701 device_quiet(dev); 1702 return (BUS_PROBE_NOWILDCARD); 1703 } 1704 1705 static int 1706 swcr_attach(device_t dev) 1707 { 1708 1709 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1710 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1711 if (swcr_id < 0) { 1712 device_printf(dev, "cannot initialize!"); 1713 return (ENXIO); 1714 } 1715 1716 return (0); 1717 } 1718 1719 static int 1720 swcr_detach(device_t dev) 1721 { 1722 crypto_unregister_all(swcr_id); 1723 return 0; 1724 } 1725 1726 static device_method_t swcr_methods[] = { 1727 DEVMETHOD(device_identify, swcr_identify), 1728 DEVMETHOD(device_probe, swcr_probe), 1729 DEVMETHOD(device_attach, swcr_attach), 1730 DEVMETHOD(device_detach, swcr_detach), 1731 1732 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1733 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1734 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1735 DEVMETHOD(cryptodev_process, swcr_process), 1736 1737 {0, 0}, 1738 }; 1739 1740 static driver_t swcr_driver = { 1741 "cryptosoft", 1742 swcr_methods, 1743 0, /* NB: no softc */ 1744 }; 1745 1746 /* 1747 * NB: We explicitly reference the crypto module so we 1748 * get the necessary ordering when built as a loadable 1749 * module. This is required because we bundle the crypto 1750 * module code together with the cryptosoft driver (otherwise 1751 * normal module dependencies would handle things). 1752 */ 1753 extern int crypto_modevent(struct module *, int, void *); 1754 /* XXX where to attach */ 1755 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, crypto_modevent, NULL); 1756 MODULE_VERSION(cryptosoft, 1); 1757 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1758