1 /* 2 * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c 28 */ 29 30 #include <sys/types.h> 31 #include <sys/errno.h> 32 33 #ifdef _KERNEL 34 #include <sys/libkern.h> 35 #include <sys/malloc.h> 36 #include <sys/sysctl.h> 37 #include <opencrypto/cryptodev.h> 38 #include <opencrypto/xform.h> 39 #endif 40 41 #include <sys/zio_crypt.h> 42 #include <sys/fs/zfs.h> 43 #include <sys/zio.h> 44 45 #include <sys/freebsd_crypto.h> 46 47 #define SHA512_HMAC_BLOCK_SIZE 128 48 49 static int crypt_sessions = 0; 50 SYSCTL_DECL(_vfs_zfs); 51 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD, 52 &crypt_sessions, 0, "Number of cryptographic sessions created"); 53 54 void 55 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key) 56 { 57 uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE], 58 k_opad[SHA512_HMAC_BLOCK_SIZE], 59 key[SHA512_HMAC_BLOCK_SIZE]; 60 SHA512_CTX lctx; 61 int i; 62 size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length); 63 64 /* 65 * This code is based on the similar code in geom/eli/g_eli_hmac.c 66 */ 67 memset(key, 0, sizeof (key)); 68 if (c_key->ck_length == 0) 69 /* do nothing */; 70 else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE) 71 memcpy(key, c_key->ck_data, cl_bytes); 72 else { 73 /* 74 * If key is longer than 128 bytes reset it to 75 * key = SHA512(key). 76 */ 77 SHA512_Init(&lctx); 78 SHA512_Update(&lctx, c_key->ck_data, cl_bytes); 79 SHA512_Final(key, &lctx); 80 } 81 82 /* XOR key with ipad and opad values. */ 83 for (i = 0; i < sizeof (key); i++) { 84 k_ipad[i] = key[i] ^ 0x36; 85 k_opad[i] = key[i] ^ 0x5c; 86 } 87 memset(key, 0, sizeof (key)); 88 89 /* Start inner SHA512. */ 90 SHA512_Init(&ctx->innerctx); 91 SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad)); 92 memset(k_ipad, 0, sizeof (k_ipad)); 93 /* Start outer SHA512. */ 94 SHA512_Init(&ctx->outerctx); 95 SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad)); 96 memset(k_opad, 0, sizeof (k_opad)); 97 } 98 99 void 100 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize) 101 { 102 SHA512_Update(&ctx->innerctx, data, datasize); 103 } 104 105 void 106 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize) 107 { 108 uint8_t digest[SHA512_DIGEST_LENGTH]; 109 110 /* Complete inner hash */ 111 SHA512_Final(digest, &ctx->innerctx); 112 113 /* Complete outer hash */ 114 SHA512_Update(&ctx->outerctx, digest, sizeof (digest)); 115 SHA512_Final(digest, &ctx->outerctx); 116 117 memset(ctx, 0, sizeof (*ctx)); 118 /* mdsize == 0 means "Give me the whole hash!" */ 119 if (mdsize == 0) 120 mdsize = SHA512_DIGEST_LENGTH; 121 memcpy(md, digest, mdsize); 122 memset(digest, 0, sizeof (digest)); 123 } 124 125 void 126 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size, 127 void *out_data, size_t out_data_size) 128 { 129 struct hmac_ctx ctx; 130 131 crypto_mac_init(&ctx, key); 132 crypto_mac_update(&ctx, in_data, in_data_size); 133 crypto_mac_final(&ctx, out_data, out_data_size); 134 } 135 136 static int 137 freebsd_zfs_crypt_done(struct cryptop *crp) 138 { 139 freebsd_crypt_session_t *ses; 140 141 ses = crp->crp_opaque; 142 mtx_lock(&ses->fs_lock); 143 ses->fs_done = true; 144 mtx_unlock(&ses->fs_lock); 145 wakeup(crp); 146 return (0); 147 } 148 149 static int 150 freebsd_zfs_crypt_done_sync(struct cryptop *crp) 151 { 152 153 return (0); 154 } 155 156 void 157 freebsd_crypt_freesession(freebsd_crypt_session_t *sess) 158 { 159 mtx_destroy(&sess->fs_lock); 160 crypto_freesession(sess->fs_sid); 161 memset(sess, 0, sizeof (*sess)); 162 } 163 164 static int 165 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp) 166 { 167 int error; 168 169 crp->crp_opaque = session; 170 for (;;) { 171 #if __FreeBSD_version < 1400004 172 boolean_t async = ((crypto_ses2caps(crp->crp_session) & 173 CRYPTOCAP_F_SYNC) == 0); 174 #else 175 boolean_t async = !CRYPTO_SESS_SYNC(crp->crp_session); 176 #endif 177 crp->crp_callback = async ? freebsd_zfs_crypt_done : 178 freebsd_zfs_crypt_done_sync; 179 error = crypto_dispatch(crp); 180 if (error == 0) { 181 if (async) { 182 mtx_lock(&session->fs_lock); 183 while (session->fs_done == false) { 184 msleep(crp, &session->fs_lock, 0, 185 "zfs_crypto", 0); 186 } 187 mtx_unlock(&session->fs_lock); 188 } 189 error = crp->crp_etype; 190 } 191 192 if (error == ENOMEM) { 193 pause("zcrnomem", 1); 194 } else if (error != EAGAIN) { 195 break; 196 } 197 crp->crp_etype = 0; 198 crp->crp_flags &= ~CRYPTO_F_DONE; 199 session->fs_done = false; 200 #if __FreeBSD_version < 1300087 201 /* 202 * Session ID changed, so we should record that, 203 * and try again 204 */ 205 session->fs_sid = crp->crp_session; 206 #endif 207 } 208 return (error); 209 } 210 static void 211 freebsd_crypt_uio_debug_log(boolean_t encrypt, 212 freebsd_crypt_session_t *input_sessionp, 213 const struct zio_crypt_info *c_info, 214 zfs_uio_t *data_uio, 215 crypto_key_t *key, 216 uint8_t *ivbuf, 217 size_t datalen, 218 size_t auth_len) 219 { 220 #ifdef FCRYPTO_DEBUG 221 struct cryptodesc *crd; 222 uint8_t *p = NULL; 223 size_t total = 0; 224 225 printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, " 226 "%p, %u, %u)\n", 227 __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp, 228 c_info->ci_algname, c_info->ci_crypt_type, 229 (unsigned int)c_info->ci_keylen, c_info->ci_name, 230 data_uio, key->ck_data, 231 (unsigned int)key->ck_length, 232 ivbuf, (unsigned int)datalen, (unsigned int)auth_len); 233 printf("\tkey = { "); 234 for (int i = 0; i < key->ck_length / 8; i++) { 235 uint8_t *b = (uint8_t *)key->ck_data; 236 printf("%02x ", b[i]); 237 } 238 printf("}\n"); 239 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) { 240 printf("\tiovec #%d: <%p, %u>\n", i, 241 zfs_uio_iovbase(data_uio, i), 242 (unsigned int)zfs_uio_iovlen(data_uio, i)); 243 total += zfs_uio_iovlen(data_uio, i); 244 } 245 zfs_uio_resid(data_uio) = total; 246 #endif 247 } 248 /* 249 * Create a new cryptographic session. This should 250 * happen every time the key changes (including when 251 * it's first loaded). 252 */ 253 #if __FreeBSD_version >= 1300087 254 int 255 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 256 const struct zio_crypt_info *c_info, crypto_key_t *key) 257 { 258 struct crypto_session_params csp = {0}; 259 int error = 0; 260 261 #ifdef FCRYPTO_DEBUG 262 printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n", 263 __FUNCTION__, sessp, 264 c_info->ci_algname, c_info->ci_crypt_type, 265 (unsigned int)c_info->ci_keylen, c_info->ci_name, 266 key->ck_data, (unsigned int)key->ck_length); 267 printf("\tkey = { "); 268 for (int i = 0; i < key->ck_length / 8; i++) { 269 uint8_t *b = (uint8_t *)key->ck_data; 270 printf("%02x ", b[i]); 271 } 272 printf("}\n"); 273 #endif 274 csp.csp_mode = CSP_MODE_AEAD; 275 csp.csp_cipher_key = key->ck_data; 276 csp.csp_cipher_klen = key->ck_length / 8; 277 switch (c_info->ci_crypt_type) { 278 case ZC_TYPE_GCM: 279 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 280 csp.csp_ivlen = AES_GCM_IV_LEN; 281 switch (key->ck_length/8) { 282 case AES_128_GMAC_KEY_LEN: 283 case AES_192_GMAC_KEY_LEN: 284 case AES_256_GMAC_KEY_LEN: 285 break; 286 default: 287 error = EINVAL; 288 goto bad; 289 } 290 break; 291 case ZC_TYPE_CCM: 292 csp.csp_cipher_alg = CRYPTO_AES_CCM_16; 293 csp.csp_ivlen = AES_CCM_IV_LEN; 294 switch (key->ck_length/8) { 295 case AES_128_CBC_MAC_KEY_LEN: 296 case AES_192_CBC_MAC_KEY_LEN: 297 case AES_256_CBC_MAC_KEY_LEN: 298 break; 299 default: 300 error = EINVAL; 301 goto bad; 302 break; 303 } 304 break; 305 default: 306 error = ENOTSUP; 307 goto bad; 308 } 309 310 /* 311 * Disable the use of hardware drivers on FreeBSD 13 and later since 312 * common crypto offload drivers impose constraints on AES-GCM AAD 313 * lengths that make them unusable for ZFS, and we currently do not have 314 * a mechanism to fall back to a software driver for requests not 315 * handled by a hardware driver. 316 * 317 * On 12 we continue to permit the use of hardware drivers since 318 * CPU-accelerated drivers such as aesni(4) register themselves as 319 * hardware drivers. 320 */ 321 error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE); 322 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 323 NULL, MTX_DEF); 324 crypt_sessions++; 325 bad: 326 #ifdef FCRYPTO_DEBUG 327 if (error) 328 printf("%s: returning error %d\n", __FUNCTION__, error); 329 #endif 330 return (error); 331 } 332 333 int 334 freebsd_crypt_uio(boolean_t encrypt, 335 freebsd_crypt_session_t *input_sessionp, 336 const struct zio_crypt_info *c_info, 337 zfs_uio_t *data_uio, 338 crypto_key_t *key, 339 uint8_t *ivbuf, 340 size_t datalen, 341 size_t auth_len) 342 { 343 struct cryptop *crp; 344 freebsd_crypt_session_t *session = NULL; 345 int error = 0; 346 size_t total = 0; 347 348 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 349 key, ivbuf, datalen, auth_len); 350 for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) 351 total += zfs_uio_iovlen(data_uio, i); 352 zfs_uio_resid(data_uio) = total; 353 if (input_sessionp == NULL) { 354 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 355 error = freebsd_crypt_newsession(session, c_info, key); 356 if (error) 357 goto out; 358 } else 359 session = input_sessionp; 360 361 crp = crypto_getreq(session->fs_sid, M_WAITOK); 362 if (encrypt) { 363 crp->crp_op = CRYPTO_OP_ENCRYPT | 364 CRYPTO_OP_COMPUTE_DIGEST; 365 } else { 366 crp->crp_op = CRYPTO_OP_DECRYPT | 367 CRYPTO_OP_VERIFY_DIGEST; 368 } 369 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE; 370 crypto_use_uio(crp, GET_UIO_STRUCT(data_uio)); 371 372 crp->crp_aad_start = 0; 373 crp->crp_aad_length = auth_len; 374 crp->crp_payload_start = auth_len; 375 crp->crp_payload_length = datalen; 376 crp->crp_digest_start = auth_len + datalen; 377 378 memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN); 379 error = zfs_crypto_dispatch(session, crp); 380 crypto_freereq(crp); 381 out: 382 #ifdef FCRYPTO_DEBUG 383 if (error) 384 printf("%s: returning error %d\n", __FUNCTION__, error); 385 #endif 386 if (input_sessionp == NULL) { 387 freebsd_crypt_freesession(session); 388 kmem_free(session, sizeof (*session)); 389 } 390 return (error); 391 } 392 393 #else 394 int 395 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp, 396 const struct zio_crypt_info *c_info, crypto_key_t *key) 397 { 398 struct cryptoini cria = {0}, crie = {0}, *crip; 399 struct enc_xform *xform; 400 struct auth_hash *xauth; 401 int error = 0; 402 crypto_session_t sid; 403 404 #ifdef FCRYPTO_DEBUG 405 printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n", 406 __FUNCTION__, sessp, 407 c_info->ci_algname, c_info->ci_crypt_type, 408 (unsigned int)c_info->ci_keylen, c_info->ci_name, 409 key->ck_data, (unsigned int)key->ck_length); 410 printf("\tkey = { "); 411 for (int i = 0; i < key->ck_length / 8; i++) { 412 uint8_t *b = (uint8_t *)key->ck_data; 413 printf("%02x ", b[i]); 414 } 415 printf("}\n"); 416 #endif 417 switch (c_info->ci_crypt_type) { 418 case ZC_TYPE_GCM: 419 xform = &enc_xform_aes_nist_gcm; 420 switch (key->ck_length/8) { 421 case AES_128_GMAC_KEY_LEN: 422 xauth = &auth_hash_nist_gmac_aes_128; 423 break; 424 case AES_192_GMAC_KEY_LEN: 425 xauth = &auth_hash_nist_gmac_aes_192; 426 break; 427 case AES_256_GMAC_KEY_LEN: 428 xauth = &auth_hash_nist_gmac_aes_256; 429 break; 430 default: 431 error = EINVAL; 432 goto bad; 433 } 434 break; 435 case ZC_TYPE_CCM: 436 xform = &enc_xform_ccm; 437 switch (key->ck_length/8) { 438 case AES_128_CBC_MAC_KEY_LEN: 439 xauth = &auth_hash_ccm_cbc_mac_128; 440 break; 441 case AES_192_CBC_MAC_KEY_LEN: 442 xauth = &auth_hash_ccm_cbc_mac_192; 443 break; 444 case AES_256_CBC_MAC_KEY_LEN: 445 xauth = &auth_hash_ccm_cbc_mac_256; 446 break; 447 default: 448 error = EINVAL; 449 goto bad; 450 break; 451 } 452 break; 453 default: 454 error = ENOTSUP; 455 goto bad; 456 } 457 #ifdef FCRYPTO_DEBUG 458 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 459 "auth %s (key length %d)\n", 460 __FUNCTION__, __LINE__, 461 xform->name, (unsigned int)key->ck_length, 462 (unsigned int)key->ck_length/8, 463 xauth->name, xauth->keysize); 464 #endif 465 466 crie.cri_alg = xform->type; 467 crie.cri_key = key->ck_data; 468 crie.cri_klen = key->ck_length; 469 470 cria.cri_alg = xauth->type; 471 cria.cri_key = key->ck_data; 472 cria.cri_klen = key->ck_length; 473 474 cria.cri_next = &crie; 475 crie.cri_next = NULL; 476 crip = &cria; 477 // Everything else is zero-initialised 478 479 error = crypto_newsession(&sid, crip, 480 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 481 if (error != 0) { 482 printf("%s(%d): crypto_newsession failed with %d\n", 483 __FUNCTION__, __LINE__, error); 484 goto bad; 485 } 486 sessp->fs_sid = sid; 487 mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock", 488 NULL, MTX_DEF); 489 crypt_sessions++; 490 bad: 491 return (error); 492 } 493 494 /* 495 * The meat of encryption/decryption. 496 * If sessp is NULL, then it will create a 497 * temporary cryptographic session, and release 498 * it when done. 499 */ 500 int 501 freebsd_crypt_uio(boolean_t encrypt, 502 freebsd_crypt_session_t *input_sessionp, 503 const struct zio_crypt_info *c_info, 504 zfs_uio_t *data_uio, 505 crypto_key_t *key, 506 uint8_t *ivbuf, 507 size_t datalen, 508 size_t auth_len) 509 { 510 struct cryptop *crp; 511 struct cryptodesc *enc_desc, *auth_desc; 512 struct enc_xform *xform; 513 struct auth_hash *xauth; 514 freebsd_crypt_session_t *session = NULL; 515 int error; 516 517 freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio, 518 key, ivbuf, datalen, auth_len); 519 switch (c_info->ci_crypt_type) { 520 case ZC_TYPE_GCM: 521 xform = &enc_xform_aes_nist_gcm; 522 switch (key->ck_length/8) { 523 case AES_128_GMAC_KEY_LEN: 524 xauth = &auth_hash_nist_gmac_aes_128; 525 break; 526 case AES_192_GMAC_KEY_LEN: 527 xauth = &auth_hash_nist_gmac_aes_192; 528 break; 529 case AES_256_GMAC_KEY_LEN: 530 xauth = &auth_hash_nist_gmac_aes_256; 531 break; 532 default: 533 error = EINVAL; 534 goto bad; 535 } 536 break; 537 case ZC_TYPE_CCM: 538 xform = &enc_xform_ccm; 539 switch (key->ck_length/8) { 540 case AES_128_CBC_MAC_KEY_LEN: 541 xauth = &auth_hash_ccm_cbc_mac_128; 542 break; 543 case AES_192_CBC_MAC_KEY_LEN: 544 xauth = &auth_hash_ccm_cbc_mac_192; 545 break; 546 case AES_256_CBC_MAC_KEY_LEN: 547 xauth = &auth_hash_ccm_cbc_mac_256; 548 break; 549 default: 550 error = EINVAL; 551 goto bad; 552 break; 553 } 554 break; 555 default: 556 error = ENOTSUP; 557 goto bad; 558 } 559 560 #ifdef FCRYPTO_DEBUG 561 printf("%s(%d): Using crypt %s (key length %u [%u bytes]), " 562 "auth %s (key length %d)\n", 563 __FUNCTION__, __LINE__, 564 xform->name, (unsigned int)key->ck_length, 565 (unsigned int)key->ck_length/8, 566 xauth->name, xauth->keysize); 567 #endif 568 569 if (input_sessionp == NULL) { 570 session = kmem_zalloc(sizeof (*session), KM_SLEEP); 571 error = freebsd_crypt_newsession(session, c_info, key); 572 if (error) 573 goto out; 574 } else 575 session = input_sessionp; 576 577 crp = crypto_getreq(2); 578 if (crp == NULL) { 579 error = ENOMEM; 580 goto bad; 581 } 582 583 auth_desc = crp->crp_desc; 584 enc_desc = auth_desc->crd_next; 585 586 crp->crp_session = session->fs_sid; 587 crp->crp_ilen = auth_len + datalen; 588 crp->crp_buf = (void*)GET_UIO_STRUCT(data_uio); 589 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC; 590 591 auth_desc->crd_skip = 0; 592 auth_desc->crd_len = auth_len; 593 auth_desc->crd_inject = auth_len + datalen; 594 auth_desc->crd_alg = xauth->type; 595 #ifdef FCRYPTO_DEBUG 596 printf("%s: auth: skip = %u, len = %u, inject = %u\n", 597 __FUNCTION__, auth_desc->crd_skip, auth_desc->crd_len, 598 auth_desc->crd_inject); 599 #endif 600 601 enc_desc->crd_skip = auth_len; 602 enc_desc->crd_len = datalen; 603 enc_desc->crd_inject = auth_len; 604 enc_desc->crd_alg = xform->type; 605 enc_desc->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; 606 memcpy(enc_desc->crd_iv, ivbuf, ZIO_DATA_IV_LEN); 607 enc_desc->crd_next = NULL; 608 609 #ifdef FCRYPTO_DEBUG 610 printf("%s: enc: skip = %u, len = %u, inject = %u\n", 611 __FUNCTION__, enc_desc->crd_skip, enc_desc->crd_len, 612 enc_desc->crd_inject); 613 #endif 614 615 if (encrypt) 616 enc_desc->crd_flags |= CRD_F_ENCRYPT; 617 618 error = zfs_crypto_dispatch(session, crp); 619 crypto_freereq(crp); 620 out: 621 if (input_sessionp == NULL) { 622 freebsd_crypt_freesession(session); 623 kmem_free(session, sizeof (*session)); 624 } 625 bad: 626 #ifdef FCRYPTO_DEBUG 627 if (error) 628 printf("%s: returning error %d\n", __FUNCTION__, error); 629 #endif 630 return (error); 631 } 632 #endif 633