1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/malloc.h> 37 #include <sys/mbuf.h> 38 #include <sys/module.h> 39 #include <sys/sysctl.h> 40 #include <sys/errno.h> 41 #include <sys/random.h> 42 #include <sys/kernel.h> 43 #include <sys/uio.h> 44 #include <sys/endian.h> 45 #include <sys/limits.h> 46 47 #include <crypto/sha1.h> 48 #include <opencrypto/rmd160.h> 49 50 #include <opencrypto/cryptodev.h> 51 #include <opencrypto/xform.h> 52 53 #include <sys/kobj.h> 54 #include <sys/bus.h> 55 #include "cryptodev_if.h" 56 57 struct swcr_auth { 58 void *sw_ictx; 59 void *sw_octx; 60 const struct auth_hash *sw_axf; 61 uint16_t sw_mlen; 62 bool sw_hmac; 63 }; 64 65 struct swcr_encdec { 66 void *sw_ctx; 67 const struct enc_xform *sw_exf; 68 }; 69 70 struct swcr_compdec { 71 const struct comp_algo *sw_cxf; 72 }; 73 74 struct swcr_session { 75 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 76 77 struct swcr_auth swcr_auth; 78 struct swcr_encdec swcr_encdec; 79 struct swcr_compdec swcr_compdec; 80 }; 81 82 static int32_t swcr_id; 83 84 static void swcr_freesession(device_t dev, crypto_session_t cses); 85 86 /* Used for CRYPTO_NULL_CBC. */ 87 static int 88 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 89 { 90 91 return (0); 92 } 93 94 /* 95 * Apply a symmetric encryption/decryption algorithm. 96 */ 97 static int 98 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 99 { 100 unsigned char blk[EALG_MAX_BLOCK_LEN]; 101 const struct crypto_session_params *csp; 102 const struct enc_xform *exf; 103 const struct swcr_encdec *sw; 104 void *ctx; 105 size_t inlen, outlen, todo; 106 int blksz, resid; 107 struct crypto_buffer_cursor cc_in, cc_out; 108 const unsigned char *inblk; 109 unsigned char *outblk; 110 int error; 111 bool encrypting; 112 113 error = 0; 114 115 sw = &ses->swcr_encdec; 116 exf = sw->sw_exf; 117 csp = crypto_get_params(crp->crp_session); 118 119 if (exf->native_blocksize == 0) { 120 /* Check for non-padded data */ 121 if ((crp->crp_payload_length % exf->blocksize) != 0) 122 return (EINVAL); 123 124 blksz = exf->blocksize; 125 } else 126 blksz = exf->native_blocksize; 127 128 if (exf == &enc_xform_aes_icm && 129 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 130 return (EINVAL); 131 132 ctx = __builtin_alloca(exf->ctxsize); 133 if (crp->crp_cipher_key != NULL) { 134 error = exf->setkey(ctx, crp->crp_cipher_key, 135 csp->csp_cipher_klen); 136 if (error) 137 return (error); 138 } else 139 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 140 141 crypto_read_iv(crp, blk); 142 exf->reinit(ctx, blk, csp->csp_ivlen); 143 144 crypto_cursor_init(&cc_in, &crp->crp_buf); 145 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 146 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 147 crypto_cursor_init(&cc_out, &crp->crp_obuf); 148 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 149 } else 150 cc_out = cc_in; 151 152 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 153 154 /* 155 * Loop through encrypting blocks. 'inlen' is the remaining 156 * length of the current segment in the input buffer. 157 * 'outlen' is the remaining length of current segment in the 158 * output buffer. 159 */ 160 inlen = outlen = 0; 161 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 162 if (inlen == 0) 163 inblk = crypto_cursor_segment(&cc_in, &inlen); 164 if (outlen == 0) 165 outblk = crypto_cursor_segment(&cc_out, &outlen); 166 167 /* 168 * If the current block is not contained within the 169 * current input/output segment, use 'blk' as a local 170 * buffer. 171 */ 172 if (inlen < blksz) { 173 crypto_cursor_copydata(&cc_in, blksz, blk); 174 inblk = blk; 175 inlen = blksz; 176 } 177 if (outlen < blksz) { 178 outblk = blk; 179 outlen = blksz; 180 } 181 182 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz); 183 184 if (encrypting) 185 exf->encrypt_multi(ctx, inblk, outblk, todo); 186 else 187 exf->decrypt_multi(ctx, inblk, outblk, todo); 188 189 if (inblk == blk) { 190 inblk = crypto_cursor_segment(&cc_in, &inlen); 191 } else { 192 crypto_cursor_advance(&cc_in, todo); 193 inlen -= todo; 194 inblk += todo; 195 } 196 197 if (outblk == blk) { 198 crypto_cursor_copyback(&cc_out, blksz, blk); 199 outblk = crypto_cursor_segment(&cc_out, &outlen); 200 } else { 201 crypto_cursor_advance(&cc_out, todo); 202 outlen -= todo; 203 outblk += todo; 204 } 205 } 206 207 /* Handle trailing partial block for stream ciphers. */ 208 if (resid > 0) { 209 KASSERT(exf->native_blocksize != 0, 210 ("%s: partial block of %d bytes for cipher %s", 211 __func__, resid, exf->name)); 212 KASSERT(resid < blksz, ("%s: partial block too big", __func__)); 213 214 inblk = crypto_cursor_segment(&cc_in, &inlen); 215 outblk = crypto_cursor_segment(&cc_out, &outlen); 216 if (inlen < resid) { 217 crypto_cursor_copydata(&cc_in, resid, blk); 218 inblk = blk; 219 } 220 if (outlen < resid) 221 outblk = blk; 222 if (encrypting) 223 exf->encrypt_last(ctx, inblk, outblk, 224 resid); 225 else 226 exf->decrypt_last(ctx, inblk, outblk, 227 resid); 228 if (outlen < resid) 229 crypto_cursor_copyback(&cc_out, resid, blk); 230 } 231 232 explicit_bzero(ctx, exf->ctxsize); 233 explicit_bzero(blk, sizeof(blk)); 234 return (0); 235 } 236 237 /* 238 * Compute or verify hash. 239 */ 240 static int 241 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 242 { 243 struct { 244 union authctx ctx; 245 u_char aalg[HASH_MAX_LEN]; 246 u_char uaalg[HASH_MAX_LEN]; 247 } s; 248 const struct crypto_session_params *csp; 249 const struct swcr_auth *sw; 250 const struct auth_hash *axf; 251 int err; 252 253 sw = &ses->swcr_auth; 254 255 axf = sw->sw_axf; 256 257 csp = crypto_get_params(crp->crp_session); 258 if (crp->crp_auth_key != NULL) { 259 if (sw->sw_hmac) { 260 hmac_init_ipad(axf, crp->crp_auth_key, 261 csp->csp_auth_klen, &s.ctx); 262 } else { 263 axf->Init(&s.ctx); 264 axf->Setkey(&s.ctx, crp->crp_auth_key, 265 csp->csp_auth_klen); 266 } 267 } else 268 memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize); 269 270 if (crp->crp_aad != NULL) 271 err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length); 272 else 273 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 274 axf->Update, &s.ctx); 275 if (err) 276 goto out; 277 278 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 279 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 280 err = crypto_apply_buf(&crp->crp_obuf, 281 crp->crp_payload_output_start, crp->crp_payload_length, 282 axf->Update, &s.ctx); 283 else 284 err = crypto_apply(crp, crp->crp_payload_start, 285 crp->crp_payload_length, axf->Update, &s.ctx); 286 if (err) 287 goto out; 288 289 if (csp->csp_flags & CSP_F_ESN) 290 axf->Update(&s.ctx, crp->crp_esn, 4); 291 292 axf->Final(s.aalg, &s.ctx); 293 if (sw->sw_hmac) { 294 if (crp->crp_auth_key != NULL) 295 hmac_init_opad(axf, crp->crp_auth_key, 296 csp->csp_auth_klen, &s.ctx); 297 else 298 memcpy(&s.ctx, sw->sw_octx, axf->ctxsize); 299 axf->Update(&s.ctx, s.aalg, axf->hashsize); 300 axf->Final(s.aalg, &s.ctx); 301 } 302 303 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 304 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg); 305 if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0) 306 err = EBADMSG; 307 } else { 308 /* Inject the authentication data */ 309 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg); 310 } 311 out: 312 explicit_bzero(&s, sizeof(s)); 313 return (err); 314 } 315 316 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 317 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 318 319 static int 320 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 321 { 322 struct { 323 union authctx ctx; 324 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 325 u_char tag[GMAC_DIGEST_LEN]; 326 u_char tag2[GMAC_DIGEST_LEN]; 327 } s; 328 u_char *blk = (u_char *)s.blkbuf; 329 struct crypto_buffer_cursor cc; 330 const u_char *inblk; 331 const struct swcr_auth *swa; 332 const struct auth_hash *axf; 333 uint32_t *blkp; 334 size_t len; 335 int blksz, error, ivlen, resid; 336 337 swa = &ses->swcr_auth; 338 axf = swa->sw_axf; 339 blksz = GMAC_BLOCK_LEN; 340 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 341 __func__)); 342 343 if (crp->crp_auth_key != NULL) { 344 axf->Init(&s.ctx); 345 axf->Setkey(&s.ctx, crp->crp_auth_key, 346 crypto_get_params(crp->crp_session)->csp_auth_klen); 347 } else 348 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 349 350 /* Initialize the IV */ 351 ivlen = AES_GCM_IV_LEN; 352 crypto_read_iv(crp, blk); 353 354 axf->Reinit(&s.ctx, blk, ivlen); 355 crypto_cursor_init(&cc, &crp->crp_buf); 356 crypto_cursor_advance(&cc, crp->crp_payload_start); 357 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 358 inblk = crypto_cursor_segment(&cc, &len); 359 if (len >= blksz) { 360 len = rounddown(MIN(len, resid), blksz); 361 crypto_cursor_advance(&cc, len); 362 } else { 363 len = blksz; 364 crypto_cursor_copydata(&cc, len, blk); 365 inblk = blk; 366 } 367 axf->Update(&s.ctx, inblk, len); 368 } 369 if (resid > 0) { 370 memset(blk, 0, blksz); 371 crypto_cursor_copydata(&cc, resid, blk); 372 axf->Update(&s.ctx, blk, blksz); 373 } 374 375 /* length block */ 376 memset(blk, 0, blksz); 377 blkp = (uint32_t *)blk + 1; 378 *blkp = htobe32(crp->crp_payload_length * 8); 379 axf->Update(&s.ctx, blk, blksz); 380 381 /* Finalize MAC */ 382 axf->Final(s.tag, &s.ctx); 383 384 error = 0; 385 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 386 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 387 s.tag2); 388 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 389 error = EBADMSG; 390 } else { 391 /* Inject the authentication data */ 392 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag); 393 } 394 explicit_bzero(&s, sizeof(s)); 395 return (error); 396 } 397 398 static int 399 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 400 { 401 struct { 402 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 403 u_char tag[GMAC_DIGEST_LEN]; 404 u_char tag2[GMAC_DIGEST_LEN]; 405 } s; 406 u_char *blk = (u_char *)s.blkbuf; 407 struct crypto_buffer_cursor cc_in, cc_out; 408 const u_char *inblk; 409 u_char *outblk; 410 size_t inlen, outlen, todo; 411 const struct swcr_auth *swa; 412 const struct swcr_encdec *swe; 413 const struct enc_xform *exf; 414 void *ctx; 415 uint32_t *blkp; 416 int blksz, error, ivlen, r, resid; 417 418 swa = &ses->swcr_auth; 419 swe = &ses->swcr_encdec; 420 exf = swe->sw_exf; 421 blksz = GMAC_BLOCK_LEN; 422 KASSERT(blksz == exf->native_blocksize, 423 ("%s: blocksize mismatch", __func__)); 424 425 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 426 return (EINVAL); 427 428 ivlen = AES_GCM_IV_LEN; 429 430 ctx = __builtin_alloca(exf->ctxsize); 431 if (crp->crp_cipher_key != NULL) 432 exf->setkey(ctx, crp->crp_cipher_key, 433 crypto_get_params(crp->crp_session)->csp_cipher_klen); 434 else 435 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 436 exf->reinit(ctx, crp->crp_iv, ivlen); 437 438 /* Supply MAC with AAD */ 439 if (crp->crp_aad != NULL) { 440 inlen = rounddown2(crp->crp_aad_length, blksz); 441 if (inlen != 0) 442 exf->update(ctx, crp->crp_aad, inlen); 443 if (crp->crp_aad_length != inlen) { 444 memset(blk, 0, blksz); 445 memcpy(blk, (char *)crp->crp_aad + inlen, 446 crp->crp_aad_length - inlen); 447 exf->update(ctx, blk, blksz); 448 } 449 } else { 450 crypto_cursor_init(&cc_in, &crp->crp_buf); 451 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 452 for (resid = crp->crp_aad_length; resid >= blksz; 453 resid -= inlen) { 454 inblk = crypto_cursor_segment(&cc_in, &inlen); 455 if (inlen >= blksz) { 456 inlen = rounddown2(MIN(inlen, resid), blksz); 457 crypto_cursor_advance(&cc_in, inlen); 458 } else { 459 inlen = blksz; 460 crypto_cursor_copydata(&cc_in, inlen, blk); 461 inblk = blk; 462 } 463 exf->update(ctx, inblk, inlen); 464 } 465 if (resid > 0) { 466 memset(blk, 0, blksz); 467 crypto_cursor_copydata(&cc_in, resid, blk); 468 exf->update(ctx, blk, blksz); 469 } 470 } 471 472 /* Do encryption with MAC */ 473 crypto_cursor_init(&cc_in, &crp->crp_buf); 474 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 475 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 476 crypto_cursor_init(&cc_out, &crp->crp_obuf); 477 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 478 } else 479 cc_out = cc_in; 480 481 inlen = outlen = 0; 482 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 483 if (inlen == 0) 484 inblk = crypto_cursor_segment(&cc_in, &inlen); 485 if (outlen == 0) 486 outblk = crypto_cursor_segment(&cc_out, &outlen); 487 488 if (inlen < blksz) { 489 crypto_cursor_copydata(&cc_in, blksz, blk); 490 inblk = blk; 491 inlen = blksz; 492 } 493 494 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 495 if (outlen < blksz) { 496 outblk = blk; 497 outlen = blksz; 498 } 499 500 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 501 blksz); 502 503 exf->encrypt_multi(ctx, inblk, outblk, todo); 504 exf->update(ctx, outblk, todo); 505 506 if (outblk == blk) { 507 crypto_cursor_copyback(&cc_out, blksz, blk); 508 outblk = crypto_cursor_segment(&cc_out, &outlen); 509 } else { 510 crypto_cursor_advance(&cc_out, todo); 511 outlen -= todo; 512 outblk += todo; 513 } 514 } else { 515 todo = rounddown2(MIN(resid, inlen), blksz); 516 exf->update(ctx, inblk, todo); 517 } 518 519 if (inblk == blk) { 520 inblk = crypto_cursor_segment(&cc_in, &inlen); 521 } else { 522 crypto_cursor_advance(&cc_in, todo); 523 inlen -= todo; 524 inblk += todo; 525 } 526 } 527 if (resid > 0) { 528 crypto_cursor_copydata(&cc_in, resid, blk); 529 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 530 exf->encrypt_last(ctx, blk, blk, resid); 531 crypto_cursor_copyback(&cc_out, resid, blk); 532 } 533 exf->update(ctx, blk, resid); 534 } 535 536 /* length block */ 537 memset(blk, 0, blksz); 538 blkp = (uint32_t *)blk + 1; 539 *blkp = htobe32(crp->crp_aad_length * 8); 540 blkp = (uint32_t *)blk + 3; 541 *blkp = htobe32(crp->crp_payload_length * 8); 542 exf->update(ctx, blk, blksz); 543 544 /* Finalize MAC */ 545 exf->final(s.tag, ctx); 546 547 /* Validate tag */ 548 error = 0; 549 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 550 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 551 s.tag2); 552 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 553 if (r != 0) { 554 error = EBADMSG; 555 goto out; 556 } 557 558 /* tag matches, decrypt data */ 559 crypto_cursor_init(&cc_in, &crp->crp_buf); 560 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 561 562 inlen = 0; 563 for (resid = crp->crp_payload_length; resid > blksz; 564 resid -= todo) { 565 if (inlen == 0) 566 inblk = crypto_cursor_segment(&cc_in, &inlen); 567 if (outlen == 0) 568 outblk = crypto_cursor_segment(&cc_out, &outlen); 569 if (inlen < blksz) { 570 crypto_cursor_copydata(&cc_in, blksz, blk); 571 inblk = blk; 572 inlen = blksz; 573 } 574 if (outlen < blksz) { 575 outblk = blk; 576 outlen = blksz; 577 } 578 579 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 580 blksz); 581 582 exf->decrypt_multi(ctx, inblk, outblk, todo); 583 584 if (inblk == blk) { 585 inblk = crypto_cursor_segment(&cc_in, &inlen); 586 } else { 587 crypto_cursor_advance(&cc_in, todo); 588 inlen -= todo; 589 inblk += todo; 590 } 591 592 if (outblk == blk) { 593 crypto_cursor_copyback(&cc_out, blksz, blk); 594 outblk = crypto_cursor_segment(&cc_out, 595 &outlen); 596 } else { 597 crypto_cursor_advance(&cc_out, todo); 598 outlen -= todo; 599 outblk += todo; 600 } 601 } 602 if (resid > 0) { 603 crypto_cursor_copydata(&cc_in, resid, blk); 604 exf->decrypt_last(ctx, blk, blk, resid); 605 crypto_cursor_copyback(&cc_out, resid, blk); 606 } 607 } else { 608 /* Inject the authentication data */ 609 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 610 s.tag); 611 } 612 613 out: 614 explicit_bzero(ctx, exf->ctxsize); 615 explicit_bzero(&s, sizeof(s)); 616 617 return (error); 618 } 619 620 static void 621 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 622 u_int data_length, u_int tag_length, uint8_t *b0) 623 { 624 uint8_t *bp; 625 uint8_t flags, L; 626 627 KASSERT(nonce_length >= 7 && nonce_length <= 13, 628 ("nonce_length must be between 7 and 13 bytes")); 629 630 /* 631 * Need to determine the L field value. This is the number of 632 * bytes needed to specify the length of the message; the length 633 * is whatever is left in the 16 bytes after specifying flags and 634 * the nonce. 635 */ 636 L = 15 - nonce_length; 637 638 flags = ((aad_length > 0) << 6) + 639 (((tag_length - 2) / 2) << 3) + 640 L - 1; 641 642 /* 643 * Now we need to set up the first block, which has flags, nonce, 644 * and the message length. 645 */ 646 b0[0] = flags; 647 memcpy(b0 + 1, nonce, nonce_length); 648 bp = b0 + 1 + nonce_length; 649 650 /* Need to copy L' [aka L-1] bytes of data_length */ 651 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 652 *dst = data_length; 653 data_length >>= 8; 654 } 655 } 656 657 /* NB: OCF only supports AAD lengths < 2^32. */ 658 static int 659 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 660 { 661 if (aad_length < ((1 << 16) - (1 << 8))) { 662 be16enc(blk, aad_length); 663 return (sizeof(uint16_t)); 664 } else { 665 blk[0] = 0xff; 666 blk[1] = 0xfe; 667 be32enc(blk + 2, aad_length); 668 return (2 + sizeof(uint32_t)); 669 } 670 } 671 672 static int 673 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 674 { 675 struct { 676 union authctx ctx; 677 u_char blk[CCM_CBC_BLOCK_LEN]; 678 u_char tag[AES_CBC_MAC_HASH_LEN]; 679 u_char tag2[AES_CBC_MAC_HASH_LEN]; 680 } s; 681 const struct crypto_session_params *csp; 682 const struct swcr_auth *swa; 683 const struct auth_hash *axf; 684 int error, ivlen, len; 685 686 csp = crypto_get_params(crp->crp_session); 687 swa = &ses->swcr_auth; 688 axf = swa->sw_axf; 689 690 if (crp->crp_auth_key != NULL) { 691 axf->Init(&s.ctx); 692 axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen); 693 } else 694 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 695 696 /* Initialize the IV */ 697 ivlen = csp->csp_ivlen; 698 699 /* Supply MAC with IV */ 700 axf->Reinit(&s.ctx, crp->crp_iv, ivlen); 701 702 /* Supply MAC with b0. */ 703 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 704 swa->sw_mlen, s.blk); 705 axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN); 706 707 len = build_ccm_aad_length(crp->crp_payload_length, s.blk); 708 axf->Update(&s.ctx, s.blk, len); 709 710 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 711 axf->Update, &s.ctx); 712 713 /* Finalize MAC */ 714 axf->Final(s.tag, &s.ctx); 715 716 error = 0; 717 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 718 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 719 s.tag2); 720 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 721 error = EBADMSG; 722 } else { 723 /* Inject the authentication data */ 724 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 725 s.tag); 726 } 727 explicit_bzero(&s, sizeof(s)); 728 return (error); 729 } 730 731 static int 732 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 733 { 734 const struct crypto_session_params *csp; 735 struct { 736 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 737 u_char tag[AES_CBC_MAC_HASH_LEN]; 738 u_char tag2[AES_CBC_MAC_HASH_LEN]; 739 } s; 740 u_char *blk = (u_char *)s.blkbuf; 741 struct crypto_buffer_cursor cc_in, cc_out; 742 const u_char *inblk; 743 u_char *outblk; 744 size_t inlen, outlen, todo; 745 const struct swcr_auth *swa; 746 const struct swcr_encdec *swe; 747 const struct enc_xform *exf; 748 void *ctx; 749 size_t len; 750 int blksz, error, ivlen, r, resid; 751 752 csp = crypto_get_params(crp->crp_session); 753 swa = &ses->swcr_auth; 754 swe = &ses->swcr_encdec; 755 exf = swe->sw_exf; 756 blksz = AES_BLOCK_LEN; 757 KASSERT(blksz == exf->native_blocksize, 758 ("%s: blocksize mismatch", __func__)); 759 760 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 761 return (EMSGSIZE); 762 763 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 764 return (EINVAL); 765 766 ivlen = csp->csp_ivlen; 767 768 ctx = __builtin_alloca(exf->ctxsize); 769 if (crp->crp_cipher_key != NULL) 770 exf->setkey(ctx, crp->crp_cipher_key, 771 crypto_get_params(crp->crp_session)->csp_cipher_klen); 772 else 773 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 774 exf->reinit(ctx, crp->crp_iv, ivlen); 775 776 /* Supply MAC with b0. */ 777 _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN, 778 "blkbuf too small for b0"); 779 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 780 crp->crp_payload_length, swa->sw_mlen, blk); 781 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 782 783 /* Supply MAC with AAD */ 784 if (crp->crp_aad_length != 0) { 785 len = build_ccm_aad_length(crp->crp_aad_length, blk); 786 exf->update(ctx, blk, len); 787 if (crp->crp_aad != NULL) 788 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 789 else 790 crypto_apply(crp, crp->crp_aad_start, 791 crp->crp_aad_length, exf->update, ctx); 792 793 /* Pad the AAD (including length field) to a full block. */ 794 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 795 if (len != 0) { 796 len = CCM_CBC_BLOCK_LEN - len; 797 memset(blk, 0, CCM_CBC_BLOCK_LEN); 798 exf->update(ctx, blk, len); 799 } 800 } 801 802 /* Do encryption/decryption with MAC */ 803 crypto_cursor_init(&cc_in, &crp->crp_buf); 804 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 805 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 806 crypto_cursor_init(&cc_out, &crp->crp_obuf); 807 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 808 } else 809 cc_out = cc_in; 810 811 inlen = outlen = 0; 812 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 813 if (inlen == 0) 814 inblk = crypto_cursor_segment(&cc_in, &inlen); 815 if (outlen == 0) 816 outblk = crypto_cursor_segment(&cc_out, &outlen); 817 818 if (inlen < blksz) { 819 crypto_cursor_copydata(&cc_in, blksz, blk); 820 inblk = blk; 821 inlen = blksz; 822 } 823 824 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 825 if (outlen < blksz) { 826 outblk = blk; 827 outlen = blksz; 828 } 829 830 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 831 blksz); 832 833 exf->update(ctx, inblk, todo); 834 exf->encrypt_multi(ctx, inblk, outblk, todo); 835 836 if (outblk == blk) { 837 crypto_cursor_copyback(&cc_out, blksz, blk); 838 outblk = crypto_cursor_segment(&cc_out, &outlen); 839 } else { 840 crypto_cursor_advance(&cc_out, todo); 841 outlen -= todo; 842 outblk += todo; 843 } 844 } else { 845 /* 846 * One of the problems with CCM+CBC is that 847 * the authentication is done on the 848 * unencrypted data. As a result, we have to 849 * decrypt the data twice: once to generate 850 * the tag and a second time after the tag is 851 * verified. 852 */ 853 todo = blksz; 854 exf->decrypt(ctx, inblk, blk); 855 exf->update(ctx, blk, todo); 856 } 857 858 if (inblk == blk) { 859 inblk = crypto_cursor_segment(&cc_in, &inlen); 860 } else { 861 crypto_cursor_advance(&cc_in, todo); 862 inlen -= todo; 863 inblk += todo; 864 } 865 } 866 if (resid > 0) { 867 crypto_cursor_copydata(&cc_in, resid, blk); 868 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 869 exf->update(ctx, blk, resid); 870 exf->encrypt_last(ctx, blk, blk, resid); 871 crypto_cursor_copyback(&cc_out, resid, blk); 872 } else { 873 exf->decrypt_last(ctx, blk, blk, resid); 874 exf->update(ctx, blk, resid); 875 } 876 } 877 878 /* Finalize MAC */ 879 exf->final(s.tag, ctx); 880 881 /* Validate tag */ 882 error = 0; 883 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 884 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 885 s.tag2); 886 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 887 if (r != 0) { 888 error = EBADMSG; 889 goto out; 890 } 891 892 /* tag matches, decrypt data */ 893 exf->reinit(ctx, crp->crp_iv, ivlen); 894 crypto_cursor_init(&cc_in, &crp->crp_buf); 895 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 896 897 inlen = 0; 898 for (resid = crp->crp_payload_length; resid >= blksz; 899 resid -= todo) { 900 if (inlen == 0) 901 inblk = crypto_cursor_segment(&cc_in, &inlen); 902 if (outlen == 0) 903 outblk = crypto_cursor_segment(&cc_out, 904 &outlen); 905 906 if (inlen < blksz) { 907 crypto_cursor_copydata(&cc_in, blksz, blk); 908 inblk = blk; 909 inlen = blksz; 910 } 911 if (outlen < blksz) { 912 outblk = blk; 913 outlen = blksz; 914 } 915 916 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 917 blksz); 918 919 exf->decrypt_multi(ctx, inblk, outblk, todo); 920 921 if (inblk == blk) { 922 inblk = crypto_cursor_segment(&cc_in, &inlen); 923 } else { 924 crypto_cursor_advance(&cc_in, todo); 925 inlen -= todo; 926 inblk += todo; 927 } 928 929 if (outblk == blk) { 930 crypto_cursor_copyback(&cc_out, blksz, blk); 931 outblk = crypto_cursor_segment(&cc_out, 932 &outlen); 933 } else { 934 crypto_cursor_advance(&cc_out, todo); 935 outlen -= todo; 936 outblk += todo; 937 } 938 } 939 if (resid > 0) { 940 crypto_cursor_copydata(&cc_in, resid, blk); 941 exf->decrypt_last(ctx, blk, blk, resid); 942 crypto_cursor_copyback(&cc_out, resid, blk); 943 } 944 } else { 945 /* Inject the authentication data */ 946 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 947 s.tag); 948 } 949 950 out: 951 explicit_bzero(ctx, exf->ctxsize); 952 explicit_bzero(&s, sizeof(s)); 953 return (error); 954 } 955 956 static int 957 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 958 { 959 const struct crypto_session_params *csp; 960 struct { 961 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 962 u_char tag[POLY1305_HASH_LEN]; 963 u_char tag2[POLY1305_HASH_LEN]; 964 } s; 965 u_char *blk = (u_char *)s.blkbuf; 966 struct crypto_buffer_cursor cc_in, cc_out; 967 const u_char *inblk; 968 u_char *outblk; 969 size_t inlen, outlen, todo; 970 uint64_t *blkp; 971 const struct swcr_auth *swa; 972 const struct swcr_encdec *swe; 973 const struct enc_xform *exf; 974 void *ctx; 975 int blksz, error, r, resid; 976 977 swa = &ses->swcr_auth; 978 swe = &ses->swcr_encdec; 979 exf = swe->sw_exf; 980 blksz = exf->native_blocksize; 981 KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__)); 982 983 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 984 return (EINVAL); 985 986 csp = crypto_get_params(crp->crp_session); 987 988 ctx = __builtin_alloca(exf->ctxsize); 989 if (crp->crp_cipher_key != NULL) 990 exf->setkey(ctx, crp->crp_cipher_key, 991 csp->csp_cipher_klen); 992 else 993 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 994 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 995 996 /* Supply MAC with AAD */ 997 if (crp->crp_aad != NULL) 998 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 999 else 1000 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 1001 exf->update, ctx); 1002 if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) { 1003 /* padding1 */ 1004 memset(blk, 0, POLY1305_BLOCK_LEN); 1005 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1006 crp->crp_aad_length % POLY1305_BLOCK_LEN); 1007 } 1008 1009 /* Do encryption with MAC */ 1010 crypto_cursor_init(&cc_in, &crp->crp_buf); 1011 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1012 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1013 crypto_cursor_init(&cc_out, &crp->crp_obuf); 1014 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1015 } else 1016 cc_out = cc_in; 1017 1018 inlen = outlen = 0; 1019 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1020 for (resid = crp->crp_payload_length; resid >= blksz; 1021 resid -= todo) { 1022 if (inlen == 0) 1023 inblk = crypto_cursor_segment(&cc_in, &inlen); 1024 if (outlen == 0) 1025 outblk = crypto_cursor_segment(&cc_out, 1026 &outlen); 1027 1028 if (inlen < blksz) { 1029 crypto_cursor_copydata(&cc_in, blksz, blk); 1030 inblk = blk; 1031 inlen = blksz; 1032 } 1033 1034 if (outlen < blksz) { 1035 outblk = blk; 1036 outlen = blksz; 1037 } 1038 1039 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 1040 blksz); 1041 1042 exf->encrypt_multi(ctx, inblk, outblk, todo); 1043 exf->update(ctx, outblk, todo); 1044 1045 if (inblk == blk) { 1046 inblk = crypto_cursor_segment(&cc_in, &inlen); 1047 } else { 1048 crypto_cursor_advance(&cc_in, todo); 1049 inlen -= todo; 1050 inblk += todo; 1051 } 1052 1053 if (outblk == blk) { 1054 crypto_cursor_copyback(&cc_out, blksz, blk); 1055 outblk = crypto_cursor_segment(&cc_out, &outlen); 1056 } else { 1057 crypto_cursor_advance(&cc_out, todo); 1058 outlen -= todo; 1059 outblk += todo; 1060 } 1061 } 1062 if (resid > 0) { 1063 crypto_cursor_copydata(&cc_in, resid, blk); 1064 exf->encrypt_last(ctx, blk, blk, resid); 1065 crypto_cursor_copyback(&cc_out, resid, blk); 1066 exf->update(ctx, blk, resid); 1067 } 1068 } else 1069 crypto_apply(crp, crp->crp_payload_start, 1070 crp->crp_payload_length, exf->update, ctx); 1071 if (crp->crp_payload_length % POLY1305_BLOCK_LEN != 0) { 1072 /* padding2 */ 1073 memset(blk, 0, POLY1305_BLOCK_LEN); 1074 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1075 crp->crp_payload_length % POLY1305_BLOCK_LEN); 1076 } 1077 1078 /* lengths */ 1079 blkp = (uint64_t *)blk; 1080 blkp[0] = htole64(crp->crp_aad_length); 1081 blkp[1] = htole64(crp->crp_payload_length); 1082 exf->update(ctx, blk, sizeof(uint64_t) * 2); 1083 1084 /* Finalize MAC */ 1085 exf->final(s.tag, ctx); 1086 1087 /* Validate tag */ 1088 error = 0; 1089 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1090 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 1091 s.tag2); 1092 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 1093 if (r != 0) { 1094 error = EBADMSG; 1095 goto out; 1096 } 1097 1098 /* tag matches, decrypt data */ 1099 crypto_cursor_init(&cc_in, &crp->crp_buf); 1100 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1101 1102 inlen = 0; 1103 for (resid = crp->crp_payload_length; resid > blksz; 1104 resid -= todo) { 1105 if (inlen == 0) 1106 inblk = crypto_cursor_segment(&cc_in, &inlen); 1107 if (outlen == 0) 1108 outblk = crypto_cursor_segment(&cc_out, 1109 &outlen); 1110 if (inlen < blksz) { 1111 crypto_cursor_copydata(&cc_in, blksz, blk); 1112 inblk = blk; 1113 inlen = blksz; 1114 } 1115 if (outlen < blksz) { 1116 outblk = blk; 1117 outlen = blksz; 1118 } 1119 1120 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 1121 blksz); 1122 1123 exf->decrypt_multi(ctx, inblk, outblk, todo); 1124 1125 if (inblk == blk) { 1126 inblk = crypto_cursor_segment(&cc_in, &inlen); 1127 } else { 1128 crypto_cursor_advance(&cc_in, todo); 1129 inlen -= todo; 1130 inblk += todo; 1131 } 1132 1133 if (outblk == blk) { 1134 crypto_cursor_copyback(&cc_out, blksz, blk); 1135 outblk = crypto_cursor_segment(&cc_out, 1136 &outlen); 1137 } else { 1138 crypto_cursor_advance(&cc_out, todo); 1139 outlen -= todo; 1140 outblk += todo; 1141 } 1142 } 1143 if (resid > 0) { 1144 crypto_cursor_copydata(&cc_in, resid, blk); 1145 exf->decrypt_last(ctx, blk, blk, resid); 1146 crypto_cursor_copyback(&cc_out, resid, blk); 1147 } 1148 } else { 1149 /* Inject the authentication data */ 1150 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 1151 s.tag); 1152 } 1153 1154 out: 1155 explicit_bzero(ctx, exf->ctxsize); 1156 explicit_bzero(&s, sizeof(s)); 1157 return (error); 1158 } 1159 1160 /* 1161 * Apply a cipher and a digest to perform EtA. 1162 */ 1163 static int 1164 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1165 { 1166 int error; 1167 1168 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1169 error = swcr_encdec(ses, crp); 1170 if (error == 0) 1171 error = swcr_authcompute(ses, crp); 1172 } else { 1173 error = swcr_authcompute(ses, crp); 1174 if (error == 0) 1175 error = swcr_encdec(ses, crp); 1176 } 1177 return (error); 1178 } 1179 1180 /* 1181 * Apply a compression/decompression algorithm 1182 */ 1183 static int 1184 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1185 { 1186 const struct comp_algo *cxf; 1187 uint8_t *data, *out; 1188 int adj; 1189 uint32_t result; 1190 1191 cxf = ses->swcr_compdec.sw_cxf; 1192 1193 /* We must handle the whole buffer of data in one time 1194 * then if there is not all the data in the mbuf, we must 1195 * copy in a buffer. 1196 */ 1197 1198 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1199 if (data == NULL) 1200 return (EINVAL); 1201 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1202 data); 1203 1204 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1205 result = cxf->compress(data, crp->crp_payload_length, &out); 1206 else 1207 result = cxf->decompress(data, crp->crp_payload_length, &out); 1208 1209 free(data, M_CRYPTO_DATA); 1210 if (result == 0) 1211 return (EINVAL); 1212 crp->crp_olen = result; 1213 1214 /* Check the compressed size when doing compression */ 1215 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1216 if (result >= crp->crp_payload_length) { 1217 /* Compression was useless, we lost time */ 1218 free(out, M_CRYPTO_DATA); 1219 return (0); 1220 } 1221 } 1222 1223 /* Copy back the (de)compressed data. m_copyback is 1224 * extending the mbuf as necessary. 1225 */ 1226 crypto_copyback(crp, crp->crp_payload_start, result, out); 1227 if (result < crp->crp_payload_length) { 1228 switch (crp->crp_buf.cb_type) { 1229 case CRYPTO_BUF_MBUF: 1230 case CRYPTO_BUF_SINGLE_MBUF: 1231 adj = result - crp->crp_payload_length; 1232 m_adj(crp->crp_buf.cb_mbuf, adj); 1233 break; 1234 case CRYPTO_BUF_UIO: { 1235 struct uio *uio = crp->crp_buf.cb_uio; 1236 int ind; 1237 1238 adj = crp->crp_payload_length - result; 1239 ind = uio->uio_iovcnt - 1; 1240 1241 while (adj > 0 && ind >= 0) { 1242 if (adj < uio->uio_iov[ind].iov_len) { 1243 uio->uio_iov[ind].iov_len -= adj; 1244 break; 1245 } 1246 1247 adj -= uio->uio_iov[ind].iov_len; 1248 uio->uio_iov[ind].iov_len = 0; 1249 ind--; 1250 uio->uio_iovcnt--; 1251 } 1252 } 1253 break; 1254 case CRYPTO_BUF_VMPAGE: 1255 adj = crp->crp_payload_length - result; 1256 crp->crp_buf.cb_vm_page_len -= adj; 1257 break; 1258 default: 1259 break; 1260 } 1261 } 1262 free(out, M_CRYPTO_DATA); 1263 return 0; 1264 } 1265 1266 static int 1267 swcr_setup_cipher(struct swcr_session *ses, 1268 const struct crypto_session_params *csp) 1269 { 1270 struct swcr_encdec *swe; 1271 const struct enc_xform *txf; 1272 int error; 1273 1274 swe = &ses->swcr_encdec; 1275 txf = crypto_cipher(csp); 1276 if (csp->csp_cipher_key != NULL) { 1277 if (txf->ctxsize != 0) { 1278 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1279 M_NOWAIT); 1280 if (swe->sw_ctx == NULL) 1281 return (ENOMEM); 1282 } 1283 error = txf->setkey(swe->sw_ctx, 1284 csp->csp_cipher_key, csp->csp_cipher_klen); 1285 if (error) 1286 return (error); 1287 } 1288 swe->sw_exf = txf; 1289 return (0); 1290 } 1291 1292 static int 1293 swcr_setup_auth(struct swcr_session *ses, 1294 const struct crypto_session_params *csp) 1295 { 1296 struct swcr_auth *swa; 1297 const struct auth_hash *axf; 1298 1299 swa = &ses->swcr_auth; 1300 1301 axf = crypto_auth_hash(csp); 1302 swa->sw_axf = axf; 1303 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1304 return (EINVAL); 1305 if (csp->csp_auth_mlen == 0) 1306 swa->sw_mlen = axf->hashsize; 1307 else 1308 swa->sw_mlen = csp->csp_auth_mlen; 1309 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1310 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1311 M_NOWAIT); 1312 if (swa->sw_ictx == NULL) 1313 return (ENOBUFS); 1314 } 1315 1316 switch (csp->csp_auth_alg) { 1317 case CRYPTO_SHA1_HMAC: 1318 case CRYPTO_SHA2_224_HMAC: 1319 case CRYPTO_SHA2_256_HMAC: 1320 case CRYPTO_SHA2_384_HMAC: 1321 case CRYPTO_SHA2_512_HMAC: 1322 case CRYPTO_RIPEMD160_HMAC: 1323 swa->sw_hmac = true; 1324 if (csp->csp_auth_key != NULL) { 1325 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1326 M_NOWAIT); 1327 if (swa->sw_octx == NULL) 1328 return (ENOBUFS); 1329 hmac_init_ipad(axf, csp->csp_auth_key, 1330 csp->csp_auth_klen, swa->sw_ictx); 1331 hmac_init_opad(axf, csp->csp_auth_key, 1332 csp->csp_auth_klen, swa->sw_octx); 1333 } 1334 break; 1335 case CRYPTO_RIPEMD160: 1336 case CRYPTO_SHA1: 1337 case CRYPTO_SHA2_224: 1338 case CRYPTO_SHA2_256: 1339 case CRYPTO_SHA2_384: 1340 case CRYPTO_SHA2_512: 1341 case CRYPTO_NULL_HMAC: 1342 axf->Init(swa->sw_ictx); 1343 break; 1344 case CRYPTO_AES_NIST_GMAC: 1345 case CRYPTO_AES_CCM_CBC_MAC: 1346 case CRYPTO_POLY1305: 1347 if (csp->csp_auth_key != NULL) { 1348 axf->Init(swa->sw_ictx); 1349 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1350 csp->csp_auth_klen); 1351 } 1352 break; 1353 case CRYPTO_BLAKE2B: 1354 case CRYPTO_BLAKE2S: 1355 /* 1356 * Blake2b and Blake2s support an optional key but do 1357 * not require one. 1358 */ 1359 if (csp->csp_auth_klen == 0) 1360 axf->Init(swa->sw_ictx); 1361 else if (csp->csp_auth_key != NULL) 1362 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1363 csp->csp_auth_klen); 1364 break; 1365 } 1366 1367 if (csp->csp_mode == CSP_MODE_DIGEST) { 1368 switch (csp->csp_auth_alg) { 1369 case CRYPTO_AES_NIST_GMAC: 1370 ses->swcr_process = swcr_gmac; 1371 break; 1372 case CRYPTO_AES_CCM_CBC_MAC: 1373 ses->swcr_process = swcr_ccm_cbc_mac; 1374 break; 1375 default: 1376 ses->swcr_process = swcr_authcompute; 1377 } 1378 } 1379 1380 return (0); 1381 } 1382 1383 static int 1384 swcr_setup_aead(struct swcr_session *ses, 1385 const struct crypto_session_params *csp) 1386 { 1387 struct swcr_auth *swa; 1388 int error; 1389 1390 error = swcr_setup_cipher(ses, csp); 1391 if (error) 1392 return (error); 1393 1394 swa = &ses->swcr_auth; 1395 if (csp->csp_auth_mlen == 0) 1396 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1397 else 1398 swa->sw_mlen = csp->csp_auth_mlen; 1399 return (0); 1400 } 1401 1402 static bool 1403 swcr_auth_supported(const struct crypto_session_params *csp) 1404 { 1405 const struct auth_hash *axf; 1406 1407 axf = crypto_auth_hash(csp); 1408 if (axf == NULL) 1409 return (false); 1410 switch (csp->csp_auth_alg) { 1411 case CRYPTO_SHA1_HMAC: 1412 case CRYPTO_SHA2_224_HMAC: 1413 case CRYPTO_SHA2_256_HMAC: 1414 case CRYPTO_SHA2_384_HMAC: 1415 case CRYPTO_SHA2_512_HMAC: 1416 case CRYPTO_NULL_HMAC: 1417 case CRYPTO_RIPEMD160_HMAC: 1418 break; 1419 case CRYPTO_AES_NIST_GMAC: 1420 switch (csp->csp_auth_klen * 8) { 1421 case 128: 1422 case 192: 1423 case 256: 1424 break; 1425 default: 1426 return (false); 1427 } 1428 if (csp->csp_auth_key == NULL) 1429 return (false); 1430 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1431 return (false); 1432 break; 1433 case CRYPTO_POLY1305: 1434 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1435 return (false); 1436 break; 1437 case CRYPTO_AES_CCM_CBC_MAC: 1438 switch (csp->csp_auth_klen * 8) { 1439 case 128: 1440 case 192: 1441 case 256: 1442 break; 1443 default: 1444 return (false); 1445 } 1446 if (csp->csp_auth_key == NULL) 1447 return (false); 1448 break; 1449 } 1450 return (true); 1451 } 1452 1453 static bool 1454 swcr_cipher_supported(const struct crypto_session_params *csp) 1455 { 1456 const struct enc_xform *txf; 1457 1458 txf = crypto_cipher(csp); 1459 if (txf == NULL) 1460 return (false); 1461 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1462 txf->ivsize != csp->csp_ivlen) 1463 return (false); 1464 return (true); 1465 } 1466 1467 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1468 1469 static int 1470 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1471 { 1472 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1473 return (EINVAL); 1474 switch (csp->csp_mode) { 1475 case CSP_MODE_COMPRESS: 1476 switch (csp->csp_cipher_alg) { 1477 case CRYPTO_DEFLATE_COMP: 1478 break; 1479 default: 1480 return (EINVAL); 1481 } 1482 break; 1483 case CSP_MODE_CIPHER: 1484 switch (csp->csp_cipher_alg) { 1485 case CRYPTO_AES_NIST_GCM_16: 1486 case CRYPTO_AES_CCM_16: 1487 case CRYPTO_CHACHA20_POLY1305: 1488 case CRYPTO_XCHACHA20_POLY1305: 1489 return (EINVAL); 1490 default: 1491 if (!swcr_cipher_supported(csp)) 1492 return (EINVAL); 1493 break; 1494 } 1495 break; 1496 case CSP_MODE_DIGEST: 1497 if (!swcr_auth_supported(csp)) 1498 return (EINVAL); 1499 break; 1500 case CSP_MODE_AEAD: 1501 switch (csp->csp_cipher_alg) { 1502 case CRYPTO_AES_NIST_GCM_16: 1503 case CRYPTO_AES_CCM_16: 1504 switch (csp->csp_cipher_klen * 8) { 1505 case 128: 1506 case 192: 1507 case 256: 1508 break; 1509 default: 1510 return (EINVAL); 1511 } 1512 break; 1513 case CRYPTO_CHACHA20_POLY1305: 1514 case CRYPTO_XCHACHA20_POLY1305: 1515 break; 1516 default: 1517 return (EINVAL); 1518 } 1519 break; 1520 case CSP_MODE_ETA: 1521 /* AEAD algorithms cannot be used for EtA. */ 1522 switch (csp->csp_cipher_alg) { 1523 case CRYPTO_AES_NIST_GCM_16: 1524 case CRYPTO_AES_CCM_16: 1525 case CRYPTO_CHACHA20_POLY1305: 1526 case CRYPTO_XCHACHA20_POLY1305: 1527 return (EINVAL); 1528 } 1529 switch (csp->csp_auth_alg) { 1530 case CRYPTO_AES_NIST_GMAC: 1531 case CRYPTO_AES_CCM_CBC_MAC: 1532 return (EINVAL); 1533 } 1534 1535 if (!swcr_cipher_supported(csp) || 1536 !swcr_auth_supported(csp)) 1537 return (EINVAL); 1538 break; 1539 default: 1540 return (EINVAL); 1541 } 1542 1543 return (CRYPTODEV_PROBE_SOFTWARE); 1544 } 1545 1546 /* 1547 * Generate a new software session. 1548 */ 1549 static int 1550 swcr_newsession(device_t dev, crypto_session_t cses, 1551 const struct crypto_session_params *csp) 1552 { 1553 struct swcr_session *ses; 1554 const struct comp_algo *cxf; 1555 int error; 1556 1557 ses = crypto_get_driver_session(cses); 1558 1559 error = 0; 1560 switch (csp->csp_mode) { 1561 case CSP_MODE_COMPRESS: 1562 switch (csp->csp_cipher_alg) { 1563 case CRYPTO_DEFLATE_COMP: 1564 cxf = &comp_algo_deflate; 1565 break; 1566 #ifdef INVARIANTS 1567 default: 1568 panic("bad compression algo"); 1569 #endif 1570 } 1571 ses->swcr_compdec.sw_cxf = cxf; 1572 ses->swcr_process = swcr_compdec; 1573 break; 1574 case CSP_MODE_CIPHER: 1575 switch (csp->csp_cipher_alg) { 1576 case CRYPTO_NULL_CBC: 1577 ses->swcr_process = swcr_null; 1578 break; 1579 #ifdef INVARIANTS 1580 case CRYPTO_AES_NIST_GCM_16: 1581 case CRYPTO_AES_CCM_16: 1582 case CRYPTO_CHACHA20_POLY1305: 1583 case CRYPTO_XCHACHA20_POLY1305: 1584 panic("bad cipher algo"); 1585 #endif 1586 default: 1587 error = swcr_setup_cipher(ses, csp); 1588 if (error == 0) 1589 ses->swcr_process = swcr_encdec; 1590 } 1591 break; 1592 case CSP_MODE_DIGEST: 1593 error = swcr_setup_auth(ses, csp); 1594 break; 1595 case CSP_MODE_AEAD: 1596 switch (csp->csp_cipher_alg) { 1597 case CRYPTO_AES_NIST_GCM_16: 1598 error = swcr_setup_aead(ses, csp); 1599 if (error == 0) 1600 ses->swcr_process = swcr_gcm; 1601 break; 1602 case CRYPTO_AES_CCM_16: 1603 error = swcr_setup_aead(ses, csp); 1604 if (error == 0) 1605 ses->swcr_process = swcr_ccm; 1606 break; 1607 case CRYPTO_CHACHA20_POLY1305: 1608 case CRYPTO_XCHACHA20_POLY1305: 1609 error = swcr_setup_aead(ses, csp); 1610 if (error == 0) 1611 ses->swcr_process = swcr_chacha20_poly1305; 1612 break; 1613 #ifdef INVARIANTS 1614 default: 1615 panic("bad aead algo"); 1616 #endif 1617 } 1618 break; 1619 case CSP_MODE_ETA: 1620 #ifdef INVARIANTS 1621 switch (csp->csp_cipher_alg) { 1622 case CRYPTO_AES_NIST_GCM_16: 1623 case CRYPTO_AES_CCM_16: 1624 case CRYPTO_CHACHA20_POLY1305: 1625 case CRYPTO_XCHACHA20_POLY1305: 1626 panic("bad eta cipher algo"); 1627 } 1628 switch (csp->csp_auth_alg) { 1629 case CRYPTO_AES_NIST_GMAC: 1630 case CRYPTO_AES_CCM_CBC_MAC: 1631 panic("bad eta auth algo"); 1632 } 1633 #endif 1634 1635 error = swcr_setup_auth(ses, csp); 1636 if (error) 1637 break; 1638 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1639 /* Effectively degrade to digest mode. */ 1640 ses->swcr_process = swcr_authcompute; 1641 break; 1642 } 1643 1644 error = swcr_setup_cipher(ses, csp); 1645 if (error == 0) 1646 ses->swcr_process = swcr_eta; 1647 break; 1648 default: 1649 error = EINVAL; 1650 } 1651 1652 if (error) 1653 swcr_freesession(dev, cses); 1654 return (error); 1655 } 1656 1657 static void 1658 swcr_freesession(device_t dev, crypto_session_t cses) 1659 { 1660 struct swcr_session *ses; 1661 1662 ses = crypto_get_driver_session(cses); 1663 1664 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1665 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1666 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1667 } 1668 1669 /* 1670 * Process a software request. 1671 */ 1672 static int 1673 swcr_process(device_t dev, struct cryptop *crp, int hint) 1674 { 1675 struct swcr_session *ses; 1676 1677 ses = crypto_get_driver_session(crp->crp_session); 1678 1679 crp->crp_etype = ses->swcr_process(ses, crp); 1680 1681 crypto_done(crp); 1682 return (0); 1683 } 1684 1685 static void 1686 swcr_identify(driver_t *drv, device_t parent) 1687 { 1688 /* NB: order 10 is so we get attached after h/w devices */ 1689 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1690 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1691 panic("cryptosoft: could not attach"); 1692 } 1693 1694 static int 1695 swcr_probe(device_t dev) 1696 { 1697 device_set_desc(dev, "software crypto"); 1698 device_quiet(dev); 1699 return (BUS_PROBE_NOWILDCARD); 1700 } 1701 1702 static int 1703 swcr_attach(device_t dev) 1704 { 1705 1706 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1707 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1708 if (swcr_id < 0) { 1709 device_printf(dev, "cannot initialize!"); 1710 return (ENXIO); 1711 } 1712 1713 return (0); 1714 } 1715 1716 static int 1717 swcr_detach(device_t dev) 1718 { 1719 crypto_unregister_all(swcr_id); 1720 return 0; 1721 } 1722 1723 static device_method_t swcr_methods[] = { 1724 DEVMETHOD(device_identify, swcr_identify), 1725 DEVMETHOD(device_probe, swcr_probe), 1726 DEVMETHOD(device_attach, swcr_attach), 1727 DEVMETHOD(device_detach, swcr_detach), 1728 1729 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1730 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1731 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1732 DEVMETHOD(cryptodev_process, swcr_process), 1733 1734 {0, 0}, 1735 }; 1736 1737 static driver_t swcr_driver = { 1738 "cryptosoft", 1739 swcr_methods, 1740 0, /* NB: no softc */ 1741 }; 1742 1743 /* 1744 * NB: We explicitly reference the crypto module so we 1745 * get the necessary ordering when built as a loadable 1746 * module. This is required because we bundle the crypto 1747 * module code together with the cryptosoft driver (otherwise 1748 * normal module dependencies would handle things). 1749 */ 1750 extern int crypto_modevent(struct module *, int, void *); 1751 /* XXX where to attach */ 1752 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, crypto_modevent, NULL); 1753 MODULE_VERSION(cryptosoft, 1); 1754 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1755