1 /* 2 * CDDL HEADER START 3 * 4 * This file and its contents are supplied under the terms of the 5 * Common Development and Distribution License ("CDDL"), version 1.0. 6 * You may only use this file in accordance with the terms of version 7 * 1.0 of the CDDL. 8 * 9 * A full copy of the text of the CDDL should have accompanied this 10 * source. A copy of the CDDL is also available via the Internet at 11 * http://www.illumos.org/license/CDDL. 12 * 13 * CDDL HEADER END 14 */ 15 16 /* 17 * Copyright (c) 2017, Datto, Inc. All rights reserved. 18 */ 19 20 #include <sys/dsl_crypt.h> 21 #include <sys/dsl_pool.h> 22 #include <sys/zap.h> 23 #include <sys/zil.h> 24 #include <sys/dsl_dir.h> 25 #include <sys/dsl_prop.h> 26 #include <sys/spa_impl.h> 27 #include <sys/dmu_objset.h> 28 #include <sys/zvol.h> 29 30 /* 31 * This file's primary purpose is for managing master encryption keys in 32 * memory and on disk. For more info on how these keys are used, see the 33 * block comment in zio_crypt.c. 34 * 35 * All master keys are stored encrypted on disk in the form of the DSL 36 * Crypto Key ZAP object. The binary key data in this object is always 37 * randomly generated and is encrypted with the user's wrapping key. This 38 * layer of indirection allows the user to change their key without 39 * needing to re-encrypt the entire dataset. The ZAP also holds on to the 40 * (non-encrypted) encryption algorithm identifier, IV, and MAC needed to 41 * safely decrypt the master key. For more info on the user's key see the 42 * block comment in libzfs_crypto.c 43 * 44 * In-memory encryption keys are managed through the spa_keystore. The 45 * keystore consists of 3 AVL trees, which are as follows: 46 * 47 * The Wrapping Key Tree: 48 * The wrapping key (wkey) tree stores the user's keys that are fed into the 49 * kernel through 'zfs load-key' and related commands. Datasets inherit their 50 * parent's wkey by default, so these structures are refcounted. The wrapping 51 * keys remain in memory until they are explicitly unloaded (with 52 * "zfs unload-key"). Unloading is only possible when no datasets are using 53 * them (refcount=0). 54 * 55 * The DSL Crypto Key Tree: 56 * The DSL Crypto Keys (DCK) are the in-memory representation of decrypted 57 * master keys. They are used by the functions in zio_crypt.c to perform 58 * encryption, decryption, and authentication. Snapshots and clones of a given 59 * dataset will share a DSL Crypto Key, so they are also refcounted. Once the 60 * refcount on a key hits zero, it is immediately zeroed out and freed. 61 * 62 * The Crypto Key Mapping Tree: 63 * The zio layer needs to lookup master keys by their dataset object id. Since 64 * the DSL Crypto Keys can belong to multiple datasets, we maintain a tree of 65 * dsl_key_mapping_t's which essentially just map the dataset object id to its 66 * appropriate DSL Crypto Key. The management for creating and destroying these 67 * mappings hooks into the code for owning and disowning datasets. Usually, 68 * there will only be one active dataset owner, but there are times 69 * (particularly during dataset creation and destruction) when this may not be 70 * true or the dataset may not be initialized enough to own. As a result, this 71 * object is also refcounted. 72 */ 73 74 /* 75 * This tunable allows datasets to be raw received even if the stream does 76 * not include IVset guids or if the guids don't match. This is used as part 77 * of the resolution for ZPOOL_ERRATA_ZOL_8308_ENCRYPTION. 78 */ 79 int zfs_disable_ivset_guid_check = 0; 80 81 static void 82 dsl_wrapping_key_hold(dsl_wrapping_key_t *wkey, void *tag) 83 { 84 (void) zfs_refcount_add(&wkey->wk_refcnt, tag); 85 } 86 87 static void 88 dsl_wrapping_key_rele(dsl_wrapping_key_t *wkey, void *tag) 89 { 90 (void) zfs_refcount_remove(&wkey->wk_refcnt, tag); 91 } 92 93 static void 94 dsl_wrapping_key_free(dsl_wrapping_key_t *wkey) 95 { 96 ASSERT0(zfs_refcount_count(&wkey->wk_refcnt)); 97 98 if (wkey->wk_key.ck_data) { 99 bzero(wkey->wk_key.ck_data, 100 CRYPTO_BITS2BYTES(wkey->wk_key.ck_length)); 101 kmem_free(wkey->wk_key.ck_data, 102 CRYPTO_BITS2BYTES(wkey->wk_key.ck_length)); 103 } 104 105 zfs_refcount_destroy(&wkey->wk_refcnt); 106 kmem_free(wkey, sizeof (dsl_wrapping_key_t)); 107 } 108 109 static int 110 dsl_wrapping_key_create(uint8_t *wkeydata, zfs_keyformat_t keyformat, 111 uint64_t salt, uint64_t iters, dsl_wrapping_key_t **wkey_out) 112 { 113 int ret; 114 dsl_wrapping_key_t *wkey; 115 116 /* allocate the wrapping key */ 117 wkey = kmem_alloc(sizeof (dsl_wrapping_key_t), KM_SLEEP); 118 if (!wkey) 119 return (SET_ERROR(ENOMEM)); 120 121 /* allocate and initialize the underlying crypto key */ 122 wkey->wk_key.ck_data = kmem_alloc(WRAPPING_KEY_LEN, KM_SLEEP); 123 if (!wkey->wk_key.ck_data) { 124 ret = SET_ERROR(ENOMEM); 125 goto error; 126 } 127 128 wkey->wk_key.ck_format = CRYPTO_KEY_RAW; 129 wkey->wk_key.ck_length = CRYPTO_BYTES2BITS(WRAPPING_KEY_LEN); 130 bcopy(wkeydata, wkey->wk_key.ck_data, WRAPPING_KEY_LEN); 131 132 /* initialize the rest of the struct */ 133 zfs_refcount_create(&wkey->wk_refcnt); 134 wkey->wk_keyformat = keyformat; 135 wkey->wk_salt = salt; 136 wkey->wk_iters = iters; 137 138 *wkey_out = wkey; 139 return (0); 140 141 error: 142 dsl_wrapping_key_free(wkey); 143 144 *wkey_out = NULL; 145 return (ret); 146 } 147 148 int 149 dsl_crypto_params_create_nvlist(dcp_cmd_t cmd, nvlist_t *props, 150 nvlist_t *crypto_args, dsl_crypto_params_t **dcp_out) 151 { 152 int ret; 153 uint64_t crypt = ZIO_CRYPT_INHERIT; 154 uint64_t keyformat = ZFS_KEYFORMAT_NONE; 155 uint64_t salt = 0, iters = 0; 156 dsl_crypto_params_t *dcp = NULL; 157 dsl_wrapping_key_t *wkey = NULL; 158 uint8_t *wkeydata = NULL; 159 uint_t wkeydata_len = 0; 160 char *keylocation = NULL; 161 162 dcp = kmem_zalloc(sizeof (dsl_crypto_params_t), KM_SLEEP); 163 if (!dcp) { 164 ret = SET_ERROR(ENOMEM); 165 goto error; 166 } 167 168 /* get relevant properties from the nvlist */ 169 dcp->cp_cmd = cmd; 170 171 /* get relevant arguments from the nvlists */ 172 if (props != NULL) { 173 (void) nvlist_lookup_uint64(props, 174 zfs_prop_to_name(ZFS_PROP_ENCRYPTION), &crypt); 175 (void) nvlist_lookup_uint64(props, 176 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), &keyformat); 177 (void) nvlist_lookup_string(props, 178 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), &keylocation); 179 (void) nvlist_lookup_uint64(props, 180 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), &salt); 181 (void) nvlist_lookup_uint64(props, 182 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), &iters); 183 dcp->cp_crypt = crypt; 184 } 185 186 if (crypto_args != NULL) { 187 (void) nvlist_lookup_uint8_array(crypto_args, "wkeydata", 188 &wkeydata, &wkeydata_len); 189 } 190 191 /* check for valid command */ 192 if (dcp->cp_cmd >= DCP_CMD_MAX) { 193 ret = SET_ERROR(EINVAL); 194 goto error; 195 } else { 196 dcp->cp_cmd = cmd; 197 } 198 199 /* check for valid crypt */ 200 if (dcp->cp_crypt >= ZIO_CRYPT_FUNCTIONS) { 201 ret = SET_ERROR(EINVAL); 202 goto error; 203 } else { 204 dcp->cp_crypt = crypt; 205 } 206 207 /* check for valid keyformat */ 208 if (keyformat >= ZFS_KEYFORMAT_FORMATS) { 209 ret = SET_ERROR(EINVAL); 210 goto error; 211 } 212 213 /* check for a valid keylocation (of any kind) and copy it in */ 214 if (keylocation != NULL) { 215 if (!zfs_prop_valid_keylocation(keylocation, B_FALSE)) { 216 ret = SET_ERROR(EINVAL); 217 goto error; 218 } 219 220 dcp->cp_keylocation = spa_strdup(keylocation); 221 } 222 223 /* check wrapping key length, if given */ 224 if (wkeydata != NULL && wkeydata_len != WRAPPING_KEY_LEN) { 225 ret = SET_ERROR(EINVAL); 226 goto error; 227 } 228 229 /* if the user asked for the deault crypt, determine that now */ 230 if (dcp->cp_crypt == ZIO_CRYPT_ON) 231 dcp->cp_crypt = ZIO_CRYPT_ON_VALUE; 232 233 /* create the wrapping key from the raw data */ 234 if (wkeydata != NULL) { 235 /* create the wrapping key with the verified parameters */ 236 ret = dsl_wrapping_key_create(wkeydata, keyformat, salt, 237 iters, &wkey); 238 if (ret != 0) 239 goto error; 240 241 dcp->cp_wkey = wkey; 242 } 243 244 /* 245 * Remove the encryption properties from the nvlist since they are not 246 * maintained through the DSL. 247 */ 248 (void) nvlist_remove_all(props, zfs_prop_to_name(ZFS_PROP_ENCRYPTION)); 249 (void) nvlist_remove_all(props, zfs_prop_to_name(ZFS_PROP_KEYFORMAT)); 250 (void) nvlist_remove_all(props, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT)); 251 (void) nvlist_remove_all(props, 252 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS)); 253 254 *dcp_out = dcp; 255 256 return (0); 257 258 error: 259 if (wkey != NULL) 260 dsl_wrapping_key_free(wkey); 261 if (dcp != NULL) 262 kmem_free(dcp, sizeof (dsl_crypto_params_t)); 263 264 *dcp_out = NULL; 265 return (ret); 266 } 267 268 void 269 dsl_crypto_params_free(dsl_crypto_params_t *dcp, boolean_t unload) 270 { 271 if (dcp == NULL) 272 return; 273 274 if (dcp->cp_keylocation != NULL) 275 spa_strfree(dcp->cp_keylocation); 276 if (unload && dcp->cp_wkey != NULL) 277 dsl_wrapping_key_free(dcp->cp_wkey); 278 279 kmem_free(dcp, sizeof (dsl_crypto_params_t)); 280 } 281 282 static int 283 spa_crypto_key_compare(const void *a, const void *b) 284 { 285 const dsl_crypto_key_t *dcka = a; 286 const dsl_crypto_key_t *dckb = b; 287 288 if (dcka->dck_obj < dckb->dck_obj) 289 return (-1); 290 if (dcka->dck_obj > dckb->dck_obj) 291 return (1); 292 return (0); 293 } 294 295 static int 296 spa_key_mapping_compare(const void *a, const void *b) 297 { 298 const dsl_key_mapping_t *kma = a; 299 const dsl_key_mapping_t *kmb = b; 300 301 if (kma->km_dsobj < kmb->km_dsobj) 302 return (-1); 303 if (kma->km_dsobj > kmb->km_dsobj) 304 return (1); 305 return (0); 306 } 307 308 static int 309 spa_wkey_compare(const void *a, const void *b) 310 { 311 const dsl_wrapping_key_t *wka = a; 312 const dsl_wrapping_key_t *wkb = b; 313 314 if (wka->wk_ddobj < wkb->wk_ddobj) 315 return (-1); 316 if (wka->wk_ddobj > wkb->wk_ddobj) 317 return (1); 318 return (0); 319 } 320 321 void 322 spa_keystore_init(spa_keystore_t *sk) 323 { 324 rw_init(&sk->sk_dk_lock, NULL, RW_DEFAULT, NULL); 325 rw_init(&sk->sk_km_lock, NULL, RW_DEFAULT, NULL); 326 rw_init(&sk->sk_wkeys_lock, NULL, RW_DEFAULT, NULL); 327 avl_create(&sk->sk_dsl_keys, spa_crypto_key_compare, 328 sizeof (dsl_crypto_key_t), 329 offsetof(dsl_crypto_key_t, dck_avl_link)); 330 avl_create(&sk->sk_key_mappings, spa_key_mapping_compare, 331 sizeof (dsl_key_mapping_t), 332 offsetof(dsl_key_mapping_t, km_avl_link)); 333 avl_create(&sk->sk_wkeys, spa_wkey_compare, sizeof (dsl_wrapping_key_t), 334 offsetof(dsl_wrapping_key_t, wk_avl_link)); 335 } 336 337 void 338 spa_keystore_fini(spa_keystore_t *sk) 339 { 340 dsl_wrapping_key_t *wkey; 341 void *cookie = NULL; 342 343 ASSERT(avl_is_empty(&sk->sk_dsl_keys)); 344 ASSERT(avl_is_empty(&sk->sk_key_mappings)); 345 346 while ((wkey = avl_destroy_nodes(&sk->sk_wkeys, &cookie)) != NULL) 347 dsl_wrapping_key_free(wkey); 348 349 avl_destroy(&sk->sk_wkeys); 350 avl_destroy(&sk->sk_key_mappings); 351 avl_destroy(&sk->sk_dsl_keys); 352 rw_destroy(&sk->sk_wkeys_lock); 353 rw_destroy(&sk->sk_km_lock); 354 rw_destroy(&sk->sk_dk_lock); 355 } 356 357 static int 358 dsl_dir_get_encryption_root_ddobj(dsl_dir_t *dd, uint64_t *rddobj) 359 { 360 if (dd->dd_crypto_obj == 0) 361 return (SET_ERROR(ENOENT)); 362 363 return (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 364 DSL_CRYPTO_KEY_ROOT_DDOBJ, 8, 1, rddobj)); 365 } 366 367 int 368 dsl_dir_get_encryption_version(dsl_dir_t *dd, uint64_t *version) 369 { 370 *version = 0; 371 372 if (dd->dd_crypto_obj == 0) 373 return (SET_ERROR(ENOENT)); 374 375 /* version 0 is implied by ENOENT */ 376 (void) zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 377 DSL_CRYPTO_KEY_VERSION, 8, 1, version); 378 379 return (0); 380 } 381 382 boolean_t 383 dsl_dir_incompatible_encryption_version(dsl_dir_t *dd) 384 { 385 int ret; 386 uint64_t version = 0; 387 388 ret = dsl_dir_get_encryption_version(dd, &version); 389 if (ret != 0) 390 return (B_FALSE); 391 392 return (version != ZIO_CRYPT_KEY_CURRENT_VERSION); 393 } 394 395 static int 396 spa_keystore_wkey_hold_ddobj_impl(spa_t *spa, uint64_t ddobj, 397 void *tag, dsl_wrapping_key_t **wkey_out) 398 { 399 int ret; 400 dsl_wrapping_key_t search_wkey; 401 dsl_wrapping_key_t *found_wkey; 402 403 ASSERT(RW_LOCK_HELD(&spa->spa_keystore.sk_wkeys_lock)); 404 405 /* init the search wrapping key */ 406 search_wkey.wk_ddobj = ddobj; 407 408 /* lookup the wrapping key */ 409 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, &search_wkey, NULL); 410 if (!found_wkey) { 411 ret = SET_ERROR(ENOENT); 412 goto error; 413 } 414 415 /* increment the refcount */ 416 dsl_wrapping_key_hold(found_wkey, tag); 417 418 *wkey_out = found_wkey; 419 return (0); 420 421 error: 422 *wkey_out = NULL; 423 return (ret); 424 } 425 426 static int 427 spa_keystore_wkey_hold_dd(spa_t *spa, dsl_dir_t *dd, void *tag, 428 dsl_wrapping_key_t **wkey_out) 429 { 430 int ret; 431 dsl_wrapping_key_t *wkey; 432 uint64_t rddobj; 433 boolean_t locked = B_FALSE; 434 435 if (!RW_WRITE_HELD(&spa->spa_keystore.sk_wkeys_lock)) { 436 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_READER); 437 locked = B_TRUE; 438 } 439 440 /* get the ddobj that the keylocation property was inherited from */ 441 ret = dsl_dir_get_encryption_root_ddobj(dd, &rddobj); 442 if (ret != 0) 443 goto error; 444 445 /* lookup the wkey in the avl tree */ 446 ret = spa_keystore_wkey_hold_ddobj_impl(spa, rddobj, tag, &wkey); 447 if (ret != 0) 448 goto error; 449 450 /* unlock the wkey tree if we locked it */ 451 if (locked) 452 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 453 454 *wkey_out = wkey; 455 return (0); 456 457 error: 458 if (locked) 459 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 460 461 *wkey_out = NULL; 462 return (ret); 463 } 464 465 int 466 dsl_crypto_can_set_keylocation(const char *dsname, const char *keylocation) 467 { 468 int ret = 0; 469 dsl_dir_t *dd = NULL; 470 dsl_pool_t *dp = NULL; 471 uint64_t rddobj; 472 473 /* hold the dsl dir */ 474 ret = dsl_pool_hold(dsname, FTAG, &dp); 475 if (ret != 0) 476 goto out; 477 478 ret = dsl_dir_hold(dp, dsname, FTAG, &dd, NULL); 479 if (ret != 0) 480 goto out; 481 482 /* if dd is not encrypted, the value may only be "none" */ 483 if (dd->dd_crypto_obj == 0) { 484 if (strcmp(keylocation, "none") != 0) { 485 ret = SET_ERROR(EACCES); 486 goto out; 487 } 488 489 ret = 0; 490 goto out; 491 } 492 493 /* check for a valid keylocation for encrypted datasets */ 494 if (!zfs_prop_valid_keylocation(keylocation, B_TRUE)) { 495 ret = SET_ERROR(EINVAL); 496 goto out; 497 } 498 499 /* check that this is an encryption root */ 500 ret = dsl_dir_get_encryption_root_ddobj(dd, &rddobj); 501 if (ret != 0) 502 goto out; 503 504 if (rddobj != dd->dd_object) { 505 ret = SET_ERROR(EACCES); 506 goto out; 507 } 508 509 dsl_dir_rele(dd, FTAG); 510 dsl_pool_rele(dp, FTAG); 511 512 return (0); 513 514 out: 515 if (dd != NULL) 516 dsl_dir_rele(dd, FTAG); 517 if (dp != NULL) 518 dsl_pool_rele(dp, FTAG); 519 520 return (ret); 521 } 522 523 static void 524 dsl_crypto_key_free(dsl_crypto_key_t *dck) 525 { 526 ASSERT(zfs_refcount_count(&dck->dck_holds) == 0); 527 528 /* destroy the zio_crypt_key_t */ 529 zio_crypt_key_destroy(&dck->dck_key); 530 531 /* free the refcount, wrapping key, and lock */ 532 zfs_refcount_destroy(&dck->dck_holds); 533 if (dck->dck_wkey) 534 dsl_wrapping_key_rele(dck->dck_wkey, dck); 535 536 /* free the key */ 537 kmem_free(dck, sizeof (dsl_crypto_key_t)); 538 } 539 540 static void 541 dsl_crypto_key_rele(dsl_crypto_key_t *dck, void *tag) 542 { 543 if (zfs_refcount_remove(&dck->dck_holds, tag) == 0) 544 dsl_crypto_key_free(dck); 545 } 546 547 static int 548 dsl_crypto_key_open(objset_t *mos, dsl_wrapping_key_t *wkey, 549 uint64_t dckobj, void *tag, dsl_crypto_key_t **dck_out) 550 { 551 int ret; 552 uint64_t crypt = 0, guid = 0, version = 0; 553 uint8_t raw_keydata[MASTER_KEY_MAX_LEN]; 554 uint8_t raw_hmac_keydata[SHA512_HMAC_KEYLEN]; 555 uint8_t iv[WRAPPING_IV_LEN]; 556 uint8_t mac[WRAPPING_MAC_LEN]; 557 dsl_crypto_key_t *dck; 558 559 /* allocate and initialize the key */ 560 dck = kmem_zalloc(sizeof (dsl_crypto_key_t), KM_SLEEP); 561 if (!dck) 562 return (SET_ERROR(ENOMEM)); 563 564 /* fetch all of the values we need from the ZAP */ 565 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, 566 &crypt); 567 if (ret != 0) 568 goto error; 569 570 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, &guid); 571 if (ret != 0) 572 goto error; 573 574 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MASTER_KEY, 1, 575 MASTER_KEY_MAX_LEN, raw_keydata); 576 if (ret != 0) 577 goto error; 578 579 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_HMAC_KEY, 1, 580 SHA512_HMAC_KEYLEN, raw_hmac_keydata); 581 if (ret != 0) 582 goto error; 583 584 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_IV, 1, WRAPPING_IV_LEN, 585 iv); 586 if (ret != 0) 587 goto error; 588 589 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MAC, 1, WRAPPING_MAC_LEN, 590 mac); 591 if (ret != 0) 592 goto error; 593 594 /* the initial on-disk format for encryption did not have a version */ 595 (void) zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_VERSION, 8, 1, &version); 596 597 /* 598 * Unwrap the keys. If there is an error return EACCES to indicate 599 * an authentication failure. 600 */ 601 ret = zio_crypt_key_unwrap(&wkey->wk_key, crypt, version, guid, 602 raw_keydata, raw_hmac_keydata, iv, mac, &dck->dck_key); 603 if (ret != 0) { 604 ret = SET_ERROR(EACCES); 605 goto error; 606 } 607 608 /* finish initializing the dsl_crypto_key_t */ 609 zfs_refcount_create(&dck->dck_holds); 610 dsl_wrapping_key_hold(wkey, dck); 611 dck->dck_wkey = wkey; 612 dck->dck_obj = dckobj; 613 (void) zfs_refcount_add(&dck->dck_holds, tag); 614 615 *dck_out = dck; 616 return (0); 617 618 error: 619 if (dck != NULL) { 620 bzero(dck, sizeof (dsl_crypto_key_t)); 621 kmem_free(dck, sizeof (dsl_crypto_key_t)); 622 } 623 624 *dck_out = NULL; 625 return (ret); 626 } 627 628 static int 629 spa_keystore_dsl_key_hold_impl(spa_t *spa, uint64_t dckobj, void *tag, 630 dsl_crypto_key_t **dck_out) 631 { 632 int ret; 633 dsl_crypto_key_t search_dck; 634 dsl_crypto_key_t *found_dck; 635 636 ASSERT(RW_LOCK_HELD(&spa->spa_keystore.sk_dk_lock)); 637 638 /* init the search key */ 639 search_dck.dck_obj = dckobj; 640 641 /* find the matching key in the keystore */ 642 found_dck = avl_find(&spa->spa_keystore.sk_dsl_keys, &search_dck, NULL); 643 if (!found_dck) { 644 ret = SET_ERROR(ENOENT); 645 goto error; 646 } 647 648 /* increment the refcount */ 649 (void) zfs_refcount_add(&found_dck->dck_holds, tag); 650 651 *dck_out = found_dck; 652 return (0); 653 654 error: 655 *dck_out = NULL; 656 return (ret); 657 } 658 659 static int 660 spa_keystore_dsl_key_hold_dd(spa_t *spa, dsl_dir_t *dd, void *tag, 661 dsl_crypto_key_t **dck_out) 662 { 663 int ret; 664 avl_index_t where; 665 dsl_crypto_key_t *dck_io = NULL, *dck_ks = NULL; 666 dsl_wrapping_key_t *wkey = NULL; 667 uint64_t dckobj = dd->dd_crypto_obj; 668 669 /* Lookup the key in the tree of currently loaded keys */ 670 rw_enter(&spa->spa_keystore.sk_dk_lock, RW_READER); 671 ret = spa_keystore_dsl_key_hold_impl(spa, dckobj, tag, &dck_ks); 672 rw_exit(&spa->spa_keystore.sk_dk_lock); 673 if (ret == 0) { 674 *dck_out = dck_ks; 675 return (0); 676 } 677 678 /* Lookup the wrapping key from the keystore */ 679 ret = spa_keystore_wkey_hold_dd(spa, dd, FTAG, &wkey); 680 if (ret != 0) { 681 *dck_out = NULL; 682 return (SET_ERROR(EACCES)); 683 } 684 685 /* Read the key from disk */ 686 ret = dsl_crypto_key_open(spa->spa_meta_objset, wkey, dckobj, 687 tag, &dck_io); 688 if (ret != 0) { 689 dsl_wrapping_key_rele(wkey, FTAG); 690 *dck_out = NULL; 691 return (ret); 692 } 693 694 /* 695 * Add the key to the keystore. It may already exist if it was 696 * added while performing the read from disk. In this case discard 697 * it and return the key from the keystore. 698 */ 699 rw_enter(&spa->spa_keystore.sk_dk_lock, RW_WRITER); 700 ret = spa_keystore_dsl_key_hold_impl(spa, dckobj, tag, &dck_ks); 701 if (ret != 0) { 702 (void) avl_find(&spa->spa_keystore.sk_dsl_keys, dck_io, &where); 703 avl_insert(&spa->spa_keystore.sk_dsl_keys, dck_io, where); 704 *dck_out = dck_io; 705 } else { 706 dsl_crypto_key_free(dck_io); 707 *dck_out = dck_ks; 708 } 709 710 /* Release the wrapping key (the dsl key now has a reference to it) */ 711 dsl_wrapping_key_rele(wkey, FTAG); 712 rw_exit(&spa->spa_keystore.sk_dk_lock); 713 714 return (0); 715 } 716 717 void 718 spa_keystore_dsl_key_rele(spa_t *spa, dsl_crypto_key_t *dck, void *tag) 719 { 720 rw_enter(&spa->spa_keystore.sk_dk_lock, RW_WRITER); 721 722 if (zfs_refcount_remove(&dck->dck_holds, tag) == 0) { 723 avl_remove(&spa->spa_keystore.sk_dsl_keys, dck); 724 dsl_crypto_key_free(dck); 725 } 726 727 rw_exit(&spa->spa_keystore.sk_dk_lock); 728 } 729 730 int 731 spa_keystore_load_wkey_impl(spa_t *spa, dsl_wrapping_key_t *wkey) 732 { 733 int ret; 734 avl_index_t where; 735 dsl_wrapping_key_t *found_wkey; 736 737 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 738 739 /* insert the wrapping key into the keystore */ 740 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, wkey, &where); 741 if (found_wkey != NULL) { 742 ret = SET_ERROR(EEXIST); 743 goto error_unlock; 744 } 745 avl_insert(&spa->spa_keystore.sk_wkeys, wkey, where); 746 747 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 748 749 return (0); 750 751 error_unlock: 752 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 753 return (ret); 754 } 755 756 int 757 spa_keystore_load_wkey(const char *dsname, dsl_crypto_params_t *dcp, 758 boolean_t noop) 759 { 760 int ret; 761 dsl_dir_t *dd = NULL; 762 dsl_crypto_key_t *dck = NULL; 763 dsl_wrapping_key_t *wkey = dcp->cp_wkey; 764 dsl_pool_t *dp = NULL; 765 uint64_t keyformat, salt, iters; 766 767 /* 768 * We don't validate the wrapping key's keyformat, salt, or iters 769 * since they will never be needed after the DCK has been wrapped. 770 */ 771 if (dcp->cp_wkey == NULL || 772 dcp->cp_cmd != DCP_CMD_NONE || 773 dcp->cp_crypt != ZIO_CRYPT_INHERIT || 774 dcp->cp_keylocation != NULL) 775 return (SET_ERROR(EINVAL)); 776 777 ret = dsl_pool_hold(dsname, FTAG, &dp); 778 if (ret != 0) 779 goto error; 780 781 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) { 782 ret = (SET_ERROR(ENOTSUP)); 783 goto error; 784 } 785 786 /* hold the dsl dir */ 787 ret = dsl_dir_hold(dp, dsname, FTAG, &dd, NULL); 788 if (ret != 0) 789 goto error; 790 791 /* initialize the wkey's ddobj */ 792 wkey->wk_ddobj = dd->dd_object; 793 794 /* verify that the wkey is correct by opening its dsl key */ 795 ret = dsl_crypto_key_open(dp->dp_meta_objset, wkey, 796 dd->dd_crypto_obj, FTAG, &dck); 797 if (ret != 0) 798 goto error; 799 800 /* initialize the wkey encryption parameters from the DSL Crypto Key */ 801 ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj, 802 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &keyformat); 803 if (ret != 0) 804 goto error; 805 806 ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj, 807 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &salt); 808 if (ret != 0) 809 goto error; 810 811 ret = zap_lookup(dp->dp_meta_objset, dd->dd_crypto_obj, 812 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &iters); 813 if (ret != 0) 814 goto error; 815 816 ASSERT3U(keyformat, <, ZFS_KEYFORMAT_FORMATS); 817 ASSERT3U(keyformat, !=, ZFS_KEYFORMAT_NONE); 818 IMPLY(keyformat == ZFS_KEYFORMAT_PASSPHRASE, iters != 0); 819 IMPLY(keyformat == ZFS_KEYFORMAT_PASSPHRASE, salt != 0); 820 IMPLY(keyformat != ZFS_KEYFORMAT_PASSPHRASE, iters == 0); 821 IMPLY(keyformat != ZFS_KEYFORMAT_PASSPHRASE, salt == 0); 822 823 wkey->wk_keyformat = keyformat; 824 wkey->wk_salt = salt; 825 wkey->wk_iters = iters; 826 827 /* 828 * At this point we have verified the wkey and confirmed that it can 829 * be used to decrypt a DSL Crypto Key. We can simply cleanup and 830 * return if this is all the user wanted to do. 831 */ 832 if (noop) 833 goto error; 834 835 /* insert the wrapping key into the keystore */ 836 ret = spa_keystore_load_wkey_impl(dp->dp_spa, wkey); 837 if (ret != 0) 838 goto error; 839 840 dsl_crypto_key_rele(dck, FTAG); 841 dsl_dir_rele(dd, FTAG); 842 dsl_pool_rele(dp, FTAG); 843 844 return (0); 845 846 error: 847 if (dck != NULL) 848 dsl_crypto_key_rele(dck, FTAG); 849 if (dd != NULL) 850 dsl_dir_rele(dd, FTAG); 851 if (dp != NULL) 852 dsl_pool_rele(dp, FTAG); 853 854 return (ret); 855 } 856 857 int 858 spa_keystore_unload_wkey_impl(spa_t *spa, uint64_t ddobj) 859 { 860 int ret; 861 dsl_wrapping_key_t search_wkey; 862 dsl_wrapping_key_t *found_wkey; 863 864 /* init the search wrapping key */ 865 search_wkey.wk_ddobj = ddobj; 866 867 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 868 869 /* remove the wrapping key from the keystore */ 870 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, 871 &search_wkey, NULL); 872 if (!found_wkey) { 873 ret = SET_ERROR(EACCES); 874 goto error_unlock; 875 } else if (zfs_refcount_count(&found_wkey->wk_refcnt) != 0) { 876 ret = SET_ERROR(EBUSY); 877 goto error_unlock; 878 } 879 avl_remove(&spa->spa_keystore.sk_wkeys, found_wkey); 880 881 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 882 883 /* free the wrapping key */ 884 dsl_wrapping_key_free(found_wkey); 885 886 return (0); 887 888 error_unlock: 889 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 890 return (ret); 891 } 892 893 int 894 spa_keystore_unload_wkey(const char *dsname) 895 { 896 int ret = 0; 897 dsl_dir_t *dd = NULL; 898 dsl_pool_t *dp = NULL; 899 spa_t *spa = NULL; 900 901 ret = spa_open(dsname, &spa, FTAG); 902 if (ret != 0) 903 return (ret); 904 905 /* 906 * Wait for any outstanding txg IO to complete, releasing any 907 * remaining references on the wkey. 908 */ 909 if (spa_mode(spa) != FREAD) 910 txg_wait_synced(spa->spa_dsl_pool, 0); 911 912 spa_close(spa, FTAG); 913 914 /* hold the dsl dir */ 915 ret = dsl_pool_hold(dsname, FTAG, &dp); 916 if (ret != 0) 917 goto error; 918 919 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) { 920 ret = (SET_ERROR(ENOTSUP)); 921 goto error; 922 } 923 924 ret = dsl_dir_hold(dp, dsname, FTAG, &dd, NULL); 925 if (ret != 0) 926 goto error; 927 928 /* unload the wkey */ 929 ret = spa_keystore_unload_wkey_impl(dp->dp_spa, dd->dd_object); 930 if (ret != 0) 931 goto error; 932 933 dsl_dir_rele(dd, FTAG); 934 dsl_pool_rele(dp, FTAG); 935 936 return (0); 937 938 error: 939 if (dd != NULL) 940 dsl_dir_rele(dd, FTAG); 941 if (dp != NULL) 942 dsl_pool_rele(dp, FTAG); 943 944 return (ret); 945 } 946 947 void 948 key_mapping_add_ref(dsl_key_mapping_t *km, void *tag) 949 { 950 ASSERT3U(zfs_refcount_count(&km->km_refcnt), >=, 1); 951 (void) zfs_refcount_add(&km->km_refcnt, tag); 952 } 953 954 /* 955 * The locking here is a little tricky to ensure we don't cause unnecessary 956 * performance problems. We want to release a key mapping whenever someone 957 * decrements the refcount to 0, but freeing the mapping requires removing 958 * it from the spa_keystore, which requires holding sk_km_lock as a writer. 959 * Most of the time we don't want to hold this lock as a writer, since the 960 * same lock is held as a reader for each IO that needs to encrypt / decrypt 961 * data for any dataset and in practice we will only actually free the 962 * mapping after unmounting a dataset. 963 */ 964 void 965 key_mapping_rele(spa_t *spa, dsl_key_mapping_t *km, void *tag) 966 { 967 ASSERT3U(zfs_refcount_count(&km->km_refcnt), >=, 1); 968 969 if (zfs_refcount_remove(&km->km_refcnt, tag) != 0) 970 return; 971 972 /* 973 * We think we are going to need to free the mapping. Add a 974 * reference to prevent most other releasers from thinking 975 * this might be their responsibility. This is inherently 976 * racy, so we will confirm that we are legitimately the 977 * last holder once we have the sk_km_lock as a writer. 978 */ 979 (void) zfs_refcount_add(&km->km_refcnt, FTAG); 980 981 rw_enter(&spa->spa_keystore.sk_km_lock, RW_WRITER); 982 if (zfs_refcount_remove(&km->km_refcnt, FTAG) != 0) { 983 rw_exit(&spa->spa_keystore.sk_km_lock); 984 return; 985 } 986 987 avl_remove(&spa->spa_keystore.sk_key_mappings, km); 988 rw_exit(&spa->spa_keystore.sk_km_lock); 989 990 spa_keystore_dsl_key_rele(spa, km->km_key, km); 991 kmem_free(km, sizeof (dsl_key_mapping_t)); 992 } 993 994 int 995 spa_keystore_create_mapping(spa_t *spa, dsl_dataset_t *ds, void *tag, 996 dsl_key_mapping_t **km_out) 997 { 998 int ret; 999 avl_index_t where; 1000 dsl_key_mapping_t *km, *found_km; 1001 boolean_t should_free = B_FALSE; 1002 1003 /* Allocate and initialize the mapping */ 1004 km = kmem_zalloc(sizeof (dsl_key_mapping_t), KM_SLEEP); 1005 zfs_refcount_create(&km->km_refcnt); 1006 1007 ret = spa_keystore_dsl_key_hold_dd(spa, ds->ds_dir, km, &km->km_key); 1008 if (ret != 0) { 1009 zfs_refcount_destroy(&km->km_refcnt); 1010 kmem_free(km, sizeof (dsl_key_mapping_t)); 1011 1012 if (km_out != NULL) 1013 *km_out = NULL; 1014 return (ret); 1015 } 1016 1017 km->km_dsobj = ds->ds_object; 1018 1019 rw_enter(&spa->spa_keystore.sk_km_lock, RW_WRITER); 1020 1021 /* 1022 * If a mapping already exists, simply increment its refcount and 1023 * cleanup the one we made. We want to allocate / free outside of 1024 * the lock because this lock is also used by the zio layer to lookup 1025 * key mappings. Otherwise, use the one we created. Normally, there will 1026 * only be one active reference at a time (the objset owner), but there 1027 * are times when there could be multiple async users. 1028 */ 1029 found_km = avl_find(&spa->spa_keystore.sk_key_mappings, km, &where); 1030 if (found_km != NULL) { 1031 should_free = B_TRUE; 1032 (void) zfs_refcount_add(&found_km->km_refcnt, tag); 1033 if (km_out != NULL) 1034 *km_out = found_km; 1035 } else { 1036 (void) zfs_refcount_add(&km->km_refcnt, tag); 1037 avl_insert(&spa->spa_keystore.sk_key_mappings, km, where); 1038 if (km_out != NULL) 1039 *km_out = km; 1040 } 1041 1042 rw_exit(&spa->spa_keystore.sk_km_lock); 1043 1044 if (should_free) { 1045 spa_keystore_dsl_key_rele(spa, km->km_key, km); 1046 zfs_refcount_destroy(&km->km_refcnt); 1047 kmem_free(km, sizeof (dsl_key_mapping_t)); 1048 } 1049 1050 return (0); 1051 } 1052 1053 int 1054 spa_keystore_remove_mapping(spa_t *spa, uint64_t dsobj, void *tag) 1055 { 1056 int ret; 1057 dsl_key_mapping_t search_km; 1058 dsl_key_mapping_t *found_km; 1059 1060 /* init the search key mapping */ 1061 search_km.km_dsobj = dsobj; 1062 1063 rw_enter(&spa->spa_keystore.sk_km_lock, RW_READER); 1064 1065 /* find the matching mapping */ 1066 found_km = avl_find(&spa->spa_keystore.sk_key_mappings, 1067 &search_km, NULL); 1068 if (found_km == NULL) { 1069 ret = SET_ERROR(ENOENT); 1070 goto error_unlock; 1071 } 1072 1073 rw_exit(&spa->spa_keystore.sk_km_lock); 1074 1075 key_mapping_rele(spa, found_km, tag); 1076 1077 return (0); 1078 1079 error_unlock: 1080 rw_exit(&spa->spa_keystore.sk_km_lock); 1081 return (ret); 1082 } 1083 1084 /* 1085 * This function is primarily used by the zio and arc layer to lookup 1086 * DSL Crypto Keys for encryption. Callers must release the key with 1087 * spa_keystore_dsl_key_rele(). The function may also be called with 1088 * dck_out == NULL and tag == NULL to simply check that a key exists 1089 * without getting a reference to it. 1090 */ 1091 int 1092 spa_keystore_lookup_key(spa_t *spa, uint64_t dsobj, void *tag, 1093 dsl_crypto_key_t **dck_out) 1094 { 1095 int ret; 1096 dsl_key_mapping_t search_km; 1097 dsl_key_mapping_t *found_km; 1098 1099 ASSERT((tag != NULL && dck_out != NULL) || 1100 (tag == NULL && dck_out == NULL)); 1101 1102 /* init the search key mapping */ 1103 search_km.km_dsobj = dsobj; 1104 1105 rw_enter(&spa->spa_keystore.sk_km_lock, RW_READER); 1106 1107 /* remove the mapping from the tree */ 1108 found_km = avl_find(&spa->spa_keystore.sk_key_mappings, &search_km, 1109 NULL); 1110 if (found_km == NULL) { 1111 ret = SET_ERROR(ENOENT); 1112 goto error_unlock; 1113 } 1114 1115 if (found_km && tag) 1116 (void) zfs_refcount_add(&found_km->km_key->dck_holds, tag); 1117 1118 rw_exit(&spa->spa_keystore.sk_km_lock); 1119 1120 if (dck_out != NULL) 1121 *dck_out = found_km->km_key; 1122 return (0); 1123 1124 error_unlock: 1125 rw_exit(&spa->spa_keystore.sk_km_lock); 1126 1127 if (dck_out != NULL) 1128 *dck_out = NULL; 1129 return (ret); 1130 } 1131 1132 static int 1133 dmu_objset_check_wkey_loaded(dsl_dir_t *dd) 1134 { 1135 int ret; 1136 dsl_wrapping_key_t *wkey = NULL; 1137 1138 ret = spa_keystore_wkey_hold_dd(dd->dd_pool->dp_spa, dd, FTAG, 1139 &wkey); 1140 if (ret != 0) 1141 return (SET_ERROR(EACCES)); 1142 1143 dsl_wrapping_key_rele(wkey, FTAG); 1144 1145 return (0); 1146 } 1147 1148 static zfs_keystatus_t 1149 dsl_dataset_get_keystatus(dsl_dir_t *dd) 1150 { 1151 /* check if this dd has a has a dsl key */ 1152 if (dd->dd_crypto_obj == 0) 1153 return (ZFS_KEYSTATUS_NONE); 1154 1155 return (dmu_objset_check_wkey_loaded(dd) == 0 ? 1156 ZFS_KEYSTATUS_AVAILABLE : ZFS_KEYSTATUS_UNAVAILABLE); 1157 } 1158 1159 static int 1160 dsl_dir_get_crypt(dsl_dir_t *dd, uint64_t *crypt) 1161 { 1162 if (dd->dd_crypto_obj == 0) { 1163 *crypt = ZIO_CRYPT_OFF; 1164 return (0); 1165 } 1166 1167 return (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 1168 DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, crypt)); 1169 } 1170 1171 static void 1172 dsl_crypto_key_sync_impl(objset_t *mos, uint64_t dckobj, uint64_t crypt, 1173 uint64_t root_ddobj, uint64_t guid, uint8_t *iv, uint8_t *mac, 1174 uint8_t *keydata, uint8_t *hmac_keydata, uint64_t keyformat, 1175 uint64_t salt, uint64_t iters, dmu_tx_t *tx) 1176 { 1177 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, 1178 &crypt, tx)); 1179 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_ROOT_DDOBJ, 8, 1, 1180 &root_ddobj, tx)); 1181 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, 1182 &guid, tx)); 1183 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_IV, 1, WRAPPING_IV_LEN, 1184 iv, tx)); 1185 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_MAC, 1, WRAPPING_MAC_LEN, 1186 mac, tx)); 1187 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_MASTER_KEY, 1, 1188 MASTER_KEY_MAX_LEN, keydata, tx)); 1189 VERIFY0(zap_update(mos, dckobj, DSL_CRYPTO_KEY_HMAC_KEY, 1, 1190 SHA512_HMAC_KEYLEN, hmac_keydata, tx)); 1191 VERIFY0(zap_update(mos, dckobj, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 1192 8, 1, &keyformat, tx)); 1193 VERIFY0(zap_update(mos, dckobj, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 1194 8, 1, &salt, tx)); 1195 VERIFY0(zap_update(mos, dckobj, zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 1196 8, 1, &iters, tx)); 1197 } 1198 1199 static void 1200 dsl_crypto_key_sync(dsl_crypto_key_t *dck, dmu_tx_t *tx) 1201 { 1202 zio_crypt_key_t *key = &dck->dck_key; 1203 dsl_wrapping_key_t *wkey = dck->dck_wkey; 1204 uint8_t keydata[MASTER_KEY_MAX_LEN]; 1205 uint8_t hmac_keydata[SHA512_HMAC_KEYLEN]; 1206 uint8_t iv[WRAPPING_IV_LEN]; 1207 uint8_t mac[WRAPPING_MAC_LEN]; 1208 1209 ASSERT(dmu_tx_is_syncing(tx)); 1210 ASSERT3U(key->zk_crypt, <, ZIO_CRYPT_FUNCTIONS); 1211 1212 /* encrypt and store the keys along with the IV and MAC */ 1213 VERIFY0(zio_crypt_key_wrap(&dck->dck_wkey->wk_key, key, iv, mac, 1214 keydata, hmac_keydata)); 1215 1216 /* update the ZAP with the obtained values */ 1217 dsl_crypto_key_sync_impl(tx->tx_pool->dp_meta_objset, dck->dck_obj, 1218 key->zk_crypt, wkey->wk_ddobj, key->zk_guid, iv, mac, keydata, 1219 hmac_keydata, wkey->wk_keyformat, wkey->wk_salt, wkey->wk_iters, 1220 tx); 1221 } 1222 1223 typedef struct spa_keystore_change_key_args { 1224 const char *skcka_dsname; 1225 dsl_crypto_params_t *skcka_cp; 1226 } spa_keystore_change_key_args_t; 1227 1228 static int 1229 spa_keystore_change_key_check(void *arg, dmu_tx_t *tx) 1230 { 1231 int ret; 1232 dsl_dir_t *dd = NULL; 1233 dsl_pool_t *dp = dmu_tx_pool(tx); 1234 spa_keystore_change_key_args_t *skcka = arg; 1235 dsl_crypto_params_t *dcp = skcka->skcka_cp; 1236 uint64_t rddobj; 1237 1238 /* check for the encryption feature */ 1239 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) { 1240 ret = SET_ERROR(ENOTSUP); 1241 goto error; 1242 } 1243 1244 /* check for valid key change command */ 1245 if (dcp->cp_cmd != DCP_CMD_NEW_KEY && 1246 dcp->cp_cmd != DCP_CMD_INHERIT && 1247 dcp->cp_cmd != DCP_CMD_FORCE_NEW_KEY && 1248 dcp->cp_cmd != DCP_CMD_FORCE_INHERIT) { 1249 ret = SET_ERROR(EINVAL); 1250 goto error; 1251 } 1252 1253 /* hold the dd */ 1254 ret = dsl_dir_hold(dp, skcka->skcka_dsname, FTAG, &dd, NULL); 1255 if (ret != 0) 1256 goto error; 1257 1258 /* verify that the dataset is encrypted */ 1259 if (dd->dd_crypto_obj == 0) { 1260 ret = SET_ERROR(EINVAL); 1261 goto error; 1262 } 1263 1264 /* clones must always use their origin's key */ 1265 if (dsl_dir_is_clone(dd)) { 1266 ret = SET_ERROR(EINVAL); 1267 goto error; 1268 } 1269 1270 /* lookup the ddobj we are inheriting the keylocation from */ 1271 ret = dsl_dir_get_encryption_root_ddobj(dd, &rddobj); 1272 if (ret != 0) 1273 goto error; 1274 1275 /* Handle inheritance */ 1276 if (dcp->cp_cmd == DCP_CMD_INHERIT || 1277 dcp->cp_cmd == DCP_CMD_FORCE_INHERIT) { 1278 /* no other encryption params should be given */ 1279 if (dcp->cp_crypt != ZIO_CRYPT_INHERIT || 1280 dcp->cp_keylocation != NULL || 1281 dcp->cp_wkey != NULL) { 1282 ret = SET_ERROR(EINVAL); 1283 goto error; 1284 } 1285 1286 /* check that this is an encryption root */ 1287 if (dd->dd_object != rddobj) { 1288 ret = SET_ERROR(EINVAL); 1289 goto error; 1290 } 1291 1292 /* check that the parent is encrypted */ 1293 if (dd->dd_parent->dd_crypto_obj == 0) { 1294 ret = SET_ERROR(EINVAL); 1295 goto error; 1296 } 1297 1298 /* if we are rewrapping check that both keys are loaded */ 1299 if (dcp->cp_cmd == DCP_CMD_INHERIT) { 1300 ret = dmu_objset_check_wkey_loaded(dd); 1301 if (ret != 0) 1302 goto error; 1303 1304 ret = dmu_objset_check_wkey_loaded(dd->dd_parent); 1305 if (ret != 0) 1306 goto error; 1307 } 1308 1309 dsl_dir_rele(dd, FTAG); 1310 return (0); 1311 } 1312 1313 /* handle forcing an encryption root without rewrapping */ 1314 if (dcp->cp_cmd == DCP_CMD_FORCE_NEW_KEY) { 1315 /* no other encryption params should be given */ 1316 if (dcp->cp_crypt != ZIO_CRYPT_INHERIT || 1317 dcp->cp_keylocation != NULL || 1318 dcp->cp_wkey != NULL) { 1319 ret = SET_ERROR(EINVAL); 1320 goto error; 1321 } 1322 1323 /* check that this is not an encryption root */ 1324 if (dd->dd_object == rddobj) { 1325 ret = SET_ERROR(EINVAL); 1326 goto error; 1327 } 1328 1329 dsl_dir_rele(dd, FTAG); 1330 return (0); 1331 } 1332 1333 /* crypt cannot be changed after creation */ 1334 if (dcp->cp_crypt != ZIO_CRYPT_INHERIT) { 1335 ret = SET_ERROR(EINVAL); 1336 goto error; 1337 } 1338 1339 /* we are not inheritting our parent's wkey so we need one ourselves */ 1340 if (dcp->cp_wkey == NULL) { 1341 ret = SET_ERROR(EINVAL); 1342 goto error; 1343 } 1344 1345 /* check for a valid keyformat for the new wrapping key */ 1346 if (dcp->cp_wkey->wk_keyformat >= ZFS_KEYFORMAT_FORMATS || 1347 dcp->cp_wkey->wk_keyformat == ZFS_KEYFORMAT_NONE) { 1348 ret = SET_ERROR(EINVAL); 1349 goto error; 1350 } 1351 1352 /* 1353 * If this dataset is not currently an encryption root we need a new 1354 * keylocation for this dataset's new wrapping key. Otherwise we can 1355 * just keep the one we already had. 1356 */ 1357 if (dd->dd_object != rddobj && dcp->cp_keylocation == NULL) { 1358 ret = SET_ERROR(EINVAL); 1359 goto error; 1360 } 1361 1362 /* check that the keylocation is valid if it is not NULL */ 1363 if (dcp->cp_keylocation != NULL && 1364 !zfs_prop_valid_keylocation(dcp->cp_keylocation, B_TRUE)) { 1365 ret = SET_ERROR(EINVAL); 1366 goto error; 1367 } 1368 1369 /* passphrases require pbkdf2 salt and iters */ 1370 if (dcp->cp_wkey->wk_keyformat == ZFS_KEYFORMAT_PASSPHRASE) { 1371 if (dcp->cp_wkey->wk_salt == 0 || 1372 dcp->cp_wkey->wk_iters < MIN_PBKDF2_ITERATIONS) { 1373 ret = SET_ERROR(EINVAL); 1374 goto error; 1375 } 1376 } else { 1377 if (dcp->cp_wkey->wk_salt != 0 || dcp->cp_wkey->wk_iters != 0) { 1378 ret = SET_ERROR(EINVAL); 1379 goto error; 1380 } 1381 } 1382 1383 /* make sure the dd's wkey is loaded */ 1384 ret = dmu_objset_check_wkey_loaded(dd); 1385 if (ret != 0) 1386 goto error; 1387 1388 dsl_dir_rele(dd, FTAG); 1389 1390 return (0); 1391 1392 error: 1393 if (dd != NULL) 1394 dsl_dir_rele(dd, FTAG); 1395 1396 return (ret); 1397 } 1398 1399 /* 1400 * This function deals with the intricacies of updating wrapping 1401 * key references and encryption roots recursively in the event 1402 * of a call to 'zfs change-key' or 'zfs promote'. The 'skip' 1403 * parameter should always be set to B_FALSE when called 1404 * externally. 1405 */ 1406 static void 1407 spa_keystore_change_key_sync_impl(uint64_t rddobj, uint64_t ddobj, 1408 uint64_t new_rddobj, dsl_wrapping_key_t *wkey, boolean_t skip, 1409 dmu_tx_t *tx) 1410 { 1411 int ret; 1412 zap_cursor_t *zc; 1413 zap_attribute_t *za; 1414 dsl_pool_t *dp = dmu_tx_pool(tx); 1415 dsl_dir_t *dd = NULL; 1416 dsl_crypto_key_t *dck = NULL; 1417 uint64_t curr_rddobj; 1418 1419 ASSERT(RW_WRITE_HELD(&dp->dp_spa->spa_keystore.sk_wkeys_lock)); 1420 1421 /* hold the dd */ 1422 VERIFY0(dsl_dir_hold_obj(dp, ddobj, NULL, FTAG, &dd)); 1423 1424 /* ignore special dsl dirs */ 1425 if (dd->dd_myname[0] == '$' || dd->dd_myname[0] == '%') { 1426 dsl_dir_rele(dd, FTAG); 1427 return; 1428 } 1429 1430 ret = dsl_dir_get_encryption_root_ddobj(dd, &curr_rddobj); 1431 VERIFY(ret == 0 || ret == ENOENT); 1432 1433 /* 1434 * Stop recursing if this dsl dir didn't inherit from the root 1435 * or if this dd is a clone. 1436 */ 1437 if (ret == ENOENT || 1438 (!skip && (curr_rddobj != rddobj || dsl_dir_is_clone(dd)))) { 1439 dsl_dir_rele(dd, FTAG); 1440 return; 1441 } 1442 1443 /* 1444 * If we don't have a wrapping key just update the dck to reflect the 1445 * new encryption root. Otherwise rewrap the entire dck and re-sync it 1446 * to disk. If skip is set, we don't do any of this work. 1447 */ 1448 if (!skip) { 1449 if (wkey == NULL) { 1450 VERIFY0(zap_update(dp->dp_meta_objset, 1451 dd->dd_crypto_obj, 1452 DSL_CRYPTO_KEY_ROOT_DDOBJ, 8, 1, 1453 &new_rddobj, tx)); 1454 } else { 1455 VERIFY0(spa_keystore_dsl_key_hold_dd(dp->dp_spa, dd, 1456 FTAG, &dck)); 1457 dsl_wrapping_key_hold(wkey, dck); 1458 dsl_wrapping_key_rele(dck->dck_wkey, dck); 1459 dck->dck_wkey = wkey; 1460 dsl_crypto_key_sync(dck, tx); 1461 spa_keystore_dsl_key_rele(dp->dp_spa, dck, FTAG); 1462 } 1463 } 1464 1465 zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 1466 za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); 1467 1468 /* Recurse into all child dsl dirs. */ 1469 for (zap_cursor_init(zc, dp->dp_meta_objset, 1470 dsl_dir_phys(dd)->dd_child_dir_zapobj); 1471 zap_cursor_retrieve(zc, za) == 0; 1472 zap_cursor_advance(zc)) { 1473 spa_keystore_change_key_sync_impl(rddobj, 1474 za->za_first_integer, new_rddobj, wkey, B_FALSE, tx); 1475 } 1476 zap_cursor_fini(zc); 1477 1478 /* 1479 * Recurse into all dsl dirs of clones. We utilize the skip parameter 1480 * here so that we don't attempt to process the clones directly. This 1481 * is because the clone and its origin share the same dck, which has 1482 * already been updated. 1483 */ 1484 for (zap_cursor_init(zc, dp->dp_meta_objset, 1485 dsl_dir_phys(dd)->dd_clones); 1486 zap_cursor_retrieve(zc, za) == 0; 1487 zap_cursor_advance(zc)) { 1488 dsl_dataset_t *clone; 1489 1490 VERIFY0(dsl_dataset_hold_obj(dp, za->za_first_integer, 1491 FTAG, &clone)); 1492 spa_keystore_change_key_sync_impl(rddobj, 1493 clone->ds_dir->dd_object, new_rddobj, wkey, B_TRUE, tx); 1494 dsl_dataset_rele(clone, FTAG); 1495 } 1496 zap_cursor_fini(zc); 1497 1498 kmem_free(za, sizeof (zap_attribute_t)); 1499 kmem_free(zc, sizeof (zap_cursor_t)); 1500 1501 dsl_dir_rele(dd, FTAG); 1502 } 1503 1504 static void 1505 spa_keystore_change_key_sync(void *arg, dmu_tx_t *tx) 1506 { 1507 dsl_dataset_t *ds; 1508 avl_index_t where; 1509 dsl_pool_t *dp = dmu_tx_pool(tx); 1510 spa_t *spa = dp->dp_spa; 1511 spa_keystore_change_key_args_t *skcka = arg; 1512 dsl_crypto_params_t *dcp = skcka->skcka_cp; 1513 dsl_wrapping_key_t *wkey = NULL, *found_wkey; 1514 dsl_wrapping_key_t wkey_search; 1515 char *keylocation = dcp->cp_keylocation; 1516 uint64_t rddobj, new_rddobj; 1517 1518 /* create and initialize the wrapping key */ 1519 VERIFY0(dsl_dataset_hold(dp, skcka->skcka_dsname, FTAG, &ds)); 1520 ASSERT(!ds->ds_is_snapshot); 1521 1522 if (dcp->cp_cmd == DCP_CMD_NEW_KEY || 1523 dcp->cp_cmd == DCP_CMD_FORCE_NEW_KEY) { 1524 /* 1525 * We are changing to a new wkey. Set additional properties 1526 * which can be sent along with this ioctl. Note that this 1527 * command can set keylocation even if it can't normally be 1528 * set via 'zfs set' due to a non-local keylocation. 1529 */ 1530 if (dcp->cp_cmd == DCP_CMD_NEW_KEY) { 1531 wkey = dcp->cp_wkey; 1532 wkey->wk_ddobj = ds->ds_dir->dd_object; 1533 } else { 1534 keylocation = "prompt"; 1535 } 1536 1537 if (keylocation != NULL) { 1538 dsl_prop_set_sync_impl(ds, 1539 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1540 ZPROP_SRC_LOCAL, 1, strlen(keylocation) + 1, 1541 keylocation, tx); 1542 } 1543 1544 VERIFY0(dsl_dir_get_encryption_root_ddobj(ds->ds_dir, &rddobj)); 1545 new_rddobj = ds->ds_dir->dd_object; 1546 } else { 1547 /* 1548 * We are inheriting the parent's wkey. Unset any local 1549 * keylocation and grab a reference to the wkey. 1550 */ 1551 if (dcp->cp_cmd == DCP_CMD_INHERIT) { 1552 VERIFY0(spa_keystore_wkey_hold_dd(spa, 1553 ds->ds_dir->dd_parent, FTAG, &wkey)); 1554 } 1555 1556 dsl_prop_set_sync_impl(ds, 1557 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), ZPROP_SRC_NONE, 1558 0, 0, NULL, tx); 1559 1560 rddobj = ds->ds_dir->dd_object; 1561 VERIFY0(dsl_dir_get_encryption_root_ddobj(ds->ds_dir->dd_parent, 1562 &new_rddobj)); 1563 } 1564 1565 if (wkey == NULL) { 1566 ASSERT(dcp->cp_cmd == DCP_CMD_FORCE_INHERIT || 1567 dcp->cp_cmd == DCP_CMD_FORCE_NEW_KEY); 1568 } 1569 1570 rw_enter(&spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 1571 1572 /* recurse through all children and rewrap their keys */ 1573 spa_keystore_change_key_sync_impl(rddobj, ds->ds_dir->dd_object, 1574 new_rddobj, wkey, B_FALSE, tx); 1575 1576 /* 1577 * All references to the old wkey should be released now (if it 1578 * existed). Replace the wrapping key. 1579 */ 1580 wkey_search.wk_ddobj = ds->ds_dir->dd_object; 1581 found_wkey = avl_find(&spa->spa_keystore.sk_wkeys, &wkey_search, NULL); 1582 if (found_wkey != NULL) { 1583 ASSERT0(zfs_refcount_count(&found_wkey->wk_refcnt)); 1584 avl_remove(&spa->spa_keystore.sk_wkeys, found_wkey); 1585 dsl_wrapping_key_free(found_wkey); 1586 } 1587 1588 if (dcp->cp_cmd == DCP_CMD_NEW_KEY) { 1589 (void) avl_find(&spa->spa_keystore.sk_wkeys, wkey, &where); 1590 avl_insert(&spa->spa_keystore.sk_wkeys, wkey, where); 1591 } else if (wkey != NULL) { 1592 dsl_wrapping_key_rele(wkey, FTAG); 1593 } 1594 1595 rw_exit(&spa->spa_keystore.sk_wkeys_lock); 1596 1597 dsl_dataset_rele(ds, FTAG); 1598 } 1599 1600 int 1601 spa_keystore_change_key(const char *dsname, dsl_crypto_params_t *dcp) 1602 { 1603 spa_keystore_change_key_args_t skcka; 1604 1605 /* initialize the args struct */ 1606 skcka.skcka_dsname = dsname; 1607 skcka.skcka_cp = dcp; 1608 1609 /* 1610 * Perform the actual work in syncing context. The blocks modified 1611 * here could be calculated but it would require holding the pool 1612 * lock and traversing all of the datasets that will have their keys 1613 * changed. 1614 */ 1615 return (dsl_sync_task(dsname, spa_keystore_change_key_check, 1616 spa_keystore_change_key_sync, &skcka, 15, 1617 ZFS_SPACE_CHECK_RESERVED)); 1618 } 1619 1620 int 1621 dsl_dir_rename_crypt_check(dsl_dir_t *dd, dsl_dir_t *newparent) 1622 { 1623 int ret; 1624 uint64_t curr_rddobj, parent_rddobj; 1625 1626 if (dd->dd_crypto_obj == 0) 1627 return (0); 1628 1629 ret = dsl_dir_get_encryption_root_ddobj(dd, &curr_rddobj); 1630 if (ret != 0) 1631 goto error; 1632 1633 /* 1634 * if this is not an encryption root, we must make sure we are not 1635 * moving dd to a new encryption root 1636 */ 1637 if (dd->dd_object != curr_rddobj) { 1638 ret = dsl_dir_get_encryption_root_ddobj(newparent, 1639 &parent_rddobj); 1640 if (ret != 0) 1641 goto error; 1642 1643 if (parent_rddobj != curr_rddobj) { 1644 ret = SET_ERROR(EACCES); 1645 goto error; 1646 } 1647 } 1648 1649 return (0); 1650 1651 error: 1652 return (ret); 1653 } 1654 1655 /* 1656 * Check to make sure that a promote from targetdd to origindd will not require 1657 * any key rewraps. 1658 */ 1659 int 1660 dsl_dataset_promote_crypt_check(dsl_dir_t *target, dsl_dir_t *origin) 1661 { 1662 int ret; 1663 uint64_t rddobj, op_rddobj, tp_rddobj; 1664 1665 /* If the dataset is not encrypted we don't need to check anything */ 1666 if (origin->dd_crypto_obj == 0) 1667 return (0); 1668 1669 /* 1670 * If we are not changing the first origin snapshot in a chain 1671 * the encryption root won't change either. 1672 */ 1673 if (dsl_dir_is_clone(origin)) 1674 return (0); 1675 1676 /* 1677 * If the origin is the encryption root we will update 1678 * the DSL Crypto Key to point to the target instead. 1679 */ 1680 ret = dsl_dir_get_encryption_root_ddobj(origin, &rddobj); 1681 if (ret != 0) 1682 return (ret); 1683 1684 if (rddobj == origin->dd_object) 1685 return (0); 1686 1687 /* 1688 * The origin is inheriting its encryption root from its parent. 1689 * Check that the parent of the target has the same encryption root. 1690 */ 1691 ret = dsl_dir_get_encryption_root_ddobj(origin->dd_parent, &op_rddobj); 1692 if (ret != 0) 1693 return (ret); 1694 1695 ret = dsl_dir_get_encryption_root_ddobj(target->dd_parent, &tp_rddobj); 1696 if (ret != 0) 1697 return (ret); 1698 1699 if (op_rddobj != tp_rddobj) 1700 return (SET_ERROR(EACCES)); 1701 1702 return (0); 1703 } 1704 1705 void 1706 dsl_dataset_promote_crypt_sync(dsl_dir_t *target, dsl_dir_t *origin, 1707 dmu_tx_t *tx) 1708 { 1709 uint64_t rddobj; 1710 dsl_pool_t *dp = target->dd_pool; 1711 dsl_dataset_t *targetds; 1712 dsl_dataset_t *originds; 1713 char *keylocation; 1714 1715 if (origin->dd_crypto_obj == 0) 1716 return; 1717 if (dsl_dir_is_clone(origin)) 1718 return; 1719 1720 VERIFY0(dsl_dir_get_encryption_root_ddobj(origin, &rddobj)); 1721 1722 if (rddobj != origin->dd_object) 1723 return; 1724 1725 /* 1726 * If the target is being promoted to the encryption root update the 1727 * DSL Crypto Key and keylocation to reflect that. We also need to 1728 * update the DSL Crypto Keys of all children inheriting their 1729 * encryption root to point to the new target. Otherwise, the check 1730 * function ensured that the encryption root will not change. 1731 */ 1732 keylocation = kmem_alloc(ZAP_MAXVALUELEN, KM_SLEEP); 1733 1734 VERIFY0(dsl_dataset_hold_obj(dp, 1735 dsl_dir_phys(target)->dd_head_dataset_obj, FTAG, &targetds)); 1736 VERIFY0(dsl_dataset_hold_obj(dp, 1737 dsl_dir_phys(origin)->dd_head_dataset_obj, FTAG, &originds)); 1738 1739 VERIFY0(dsl_prop_get_dd(origin, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1740 1, ZAP_MAXVALUELEN, keylocation, NULL, B_FALSE)); 1741 dsl_prop_set_sync_impl(targetds, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1742 ZPROP_SRC_LOCAL, 1, strlen(keylocation) + 1, keylocation, tx); 1743 dsl_prop_set_sync_impl(originds, zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 1744 ZPROP_SRC_NONE, 0, 0, NULL, tx); 1745 1746 rw_enter(&dp->dp_spa->spa_keystore.sk_wkeys_lock, RW_WRITER); 1747 spa_keystore_change_key_sync_impl(rddobj, origin->dd_object, 1748 target->dd_object, NULL, B_FALSE, tx); 1749 rw_exit(&dp->dp_spa->spa_keystore.sk_wkeys_lock); 1750 1751 dsl_dataset_rele(targetds, FTAG); 1752 dsl_dataset_rele(originds, FTAG); 1753 kmem_free(keylocation, ZAP_MAXVALUELEN); 1754 } 1755 1756 int 1757 dmu_objset_create_crypt_check(dsl_dir_t *parentdd, dsl_crypto_params_t *dcp, 1758 boolean_t *will_encrypt) 1759 { 1760 int ret; 1761 uint64_t pcrypt, crypt; 1762 dsl_crypto_params_t dummy_dcp = { 0 }; 1763 1764 if (will_encrypt != NULL) 1765 *will_encrypt = B_FALSE; 1766 1767 if (dcp == NULL) 1768 dcp = &dummy_dcp; 1769 1770 if (dcp->cp_cmd != DCP_CMD_NONE) 1771 return (SET_ERROR(EINVAL)); 1772 1773 if (parentdd != NULL) { 1774 ret = dsl_dir_get_crypt(parentdd, &pcrypt); 1775 if (ret != 0) 1776 return (ret); 1777 } else { 1778 pcrypt = ZIO_CRYPT_OFF; 1779 } 1780 1781 crypt = (dcp->cp_crypt == ZIO_CRYPT_INHERIT) ? pcrypt : dcp->cp_crypt; 1782 1783 ASSERT3U(pcrypt, !=, ZIO_CRYPT_INHERIT); 1784 ASSERT3U(crypt, !=, ZIO_CRYPT_INHERIT); 1785 1786 /* check for valid dcp with no encryption (inherited or local) */ 1787 if (crypt == ZIO_CRYPT_OFF) { 1788 /* Must not specify encryption params */ 1789 if (dcp->cp_wkey != NULL || 1790 (dcp->cp_keylocation != NULL && 1791 strcmp(dcp->cp_keylocation, "none") != 0)) 1792 return (SET_ERROR(EINVAL)); 1793 1794 return (0); 1795 } 1796 1797 if (will_encrypt != NULL) 1798 *will_encrypt = B_TRUE; 1799 1800 /* 1801 * We will now definitely be encrypting. Check the feature flag. When 1802 * creating the pool the caller will check this for us since we won't 1803 * technically have the feature activated yet. 1804 */ 1805 if (parentdd != NULL && 1806 !spa_feature_is_enabled(parentdd->dd_pool->dp_spa, 1807 SPA_FEATURE_ENCRYPTION)) { 1808 return (SET_ERROR(EOPNOTSUPP)); 1809 } 1810 1811 /* check for errata #4 (encryption enabled, bookmark_v2 disabled) */ 1812 if (parentdd != NULL && 1813 !spa_feature_is_enabled(parentdd->dd_pool->dp_spa, 1814 SPA_FEATURE_BOOKMARK_V2)) { 1815 return (SET_ERROR(EOPNOTSUPP)); 1816 } 1817 1818 /* handle inheritance */ 1819 if (dcp->cp_wkey == NULL) { 1820 ASSERT3P(parentdd, !=, NULL); 1821 1822 /* key must be fully unspecified */ 1823 if (dcp->cp_keylocation != NULL) 1824 return (SET_ERROR(EINVAL)); 1825 1826 /* parent must have a key to inherit */ 1827 if (pcrypt == ZIO_CRYPT_OFF) 1828 return (SET_ERROR(EINVAL)); 1829 1830 /* check for parent key */ 1831 ret = dmu_objset_check_wkey_loaded(parentdd); 1832 if (ret != 0) 1833 return (ret); 1834 1835 return (0); 1836 } 1837 1838 /* At this point we should have a fully specified key. Check location */ 1839 if (dcp->cp_keylocation == NULL || 1840 !zfs_prop_valid_keylocation(dcp->cp_keylocation, B_TRUE)) 1841 return (SET_ERROR(EINVAL)); 1842 1843 /* Must have fully specified keyformat */ 1844 switch (dcp->cp_wkey->wk_keyformat) { 1845 case ZFS_KEYFORMAT_HEX: 1846 case ZFS_KEYFORMAT_RAW: 1847 /* requires no pbkdf2 iters and salt */ 1848 if (dcp->cp_wkey->wk_salt != 0 || 1849 dcp->cp_wkey->wk_iters != 0) 1850 return (SET_ERROR(EINVAL)); 1851 break; 1852 case ZFS_KEYFORMAT_PASSPHRASE: 1853 /* requires pbkdf2 iters and salt */ 1854 if (dcp->cp_wkey->wk_salt == 0 || 1855 dcp->cp_wkey->wk_iters < MIN_PBKDF2_ITERATIONS) 1856 return (SET_ERROR(EINVAL)); 1857 break; 1858 case ZFS_KEYFORMAT_NONE: 1859 default: 1860 /* keyformat must be specified and valid */ 1861 return (SET_ERROR(EINVAL)); 1862 } 1863 1864 return (0); 1865 } 1866 1867 void 1868 dsl_dataset_create_crypt_sync(uint64_t dsobj, dsl_dir_t *dd, 1869 dsl_dataset_t *origin, dsl_crypto_params_t *dcp, dmu_tx_t *tx) 1870 { 1871 dsl_pool_t *dp = dd->dd_pool; 1872 uint64_t crypt; 1873 dsl_wrapping_key_t *wkey; 1874 1875 /* clones always use their origin's wrapping key */ 1876 if (dsl_dir_is_clone(dd)) { 1877 ASSERT3P(dcp, ==, NULL); 1878 1879 /* 1880 * If this is an encrypted clone we just need to clone the 1881 * dck into dd. Zapify the dd so we can do that. 1882 */ 1883 if (origin->ds_dir->dd_crypto_obj != 0) { 1884 dmu_buf_will_dirty(dd->dd_dbuf, tx); 1885 dsl_dir_zapify(dd, tx); 1886 1887 dd->dd_crypto_obj = 1888 dsl_crypto_key_clone_sync(origin->ds_dir, tx); 1889 VERIFY0(zap_add(dp->dp_meta_objset, dd->dd_object, 1890 DD_FIELD_CRYPTO_KEY_OBJ, sizeof (uint64_t), 1, 1891 &dd->dd_crypto_obj, tx)); 1892 } 1893 1894 return; 1895 } 1896 1897 /* 1898 * A NULL dcp at this point indicates this is the origin dataset 1899 * which does not have an objset to encrypt. Raw receives will handle 1900 * encryption separately later. In both cases we can simply return. 1901 */ 1902 if (dcp == NULL || dcp->cp_cmd == DCP_CMD_RAW_RECV) 1903 return; 1904 1905 crypt = dcp->cp_crypt; 1906 wkey = dcp->cp_wkey; 1907 1908 /* figure out the effective crypt */ 1909 if (crypt == ZIO_CRYPT_INHERIT && dd->dd_parent != NULL) 1910 VERIFY0(dsl_dir_get_crypt(dd->dd_parent, &crypt)); 1911 1912 /* if we aren't doing encryption just return */ 1913 if (crypt == ZIO_CRYPT_OFF || crypt == ZIO_CRYPT_INHERIT) 1914 return; 1915 1916 /* zapify the dd so that we can add the crypto key obj to it */ 1917 dmu_buf_will_dirty(dd->dd_dbuf, tx); 1918 dsl_dir_zapify(dd, tx); 1919 1920 /* use the new key if given or inherit from the parent */ 1921 if (wkey == NULL) { 1922 VERIFY0(spa_keystore_wkey_hold_dd(dp->dp_spa, 1923 dd->dd_parent, FTAG, &wkey)); 1924 } else { 1925 wkey->wk_ddobj = dd->dd_object; 1926 } 1927 1928 ASSERT3P(wkey, !=, NULL); 1929 1930 /* Create or clone the DSL crypto key and activate the feature */ 1931 dd->dd_crypto_obj = dsl_crypto_key_create_sync(crypt, wkey, tx); 1932 VERIFY0(zap_add(dp->dp_meta_objset, dd->dd_object, 1933 DD_FIELD_CRYPTO_KEY_OBJ, sizeof (uint64_t), 1, &dd->dd_crypto_obj, 1934 tx)); 1935 dsl_dataset_activate_feature(dsobj, SPA_FEATURE_ENCRYPTION, tx); 1936 1937 /* 1938 * If we inherited the wrapping key we release our reference now. 1939 * Otherwise, this is a new key and we need to load it into the 1940 * keystore. 1941 */ 1942 if (dcp->cp_wkey == NULL) { 1943 dsl_wrapping_key_rele(wkey, FTAG); 1944 } else { 1945 VERIFY0(spa_keystore_load_wkey_impl(dp->dp_spa, wkey)); 1946 } 1947 } 1948 1949 typedef struct dsl_crypto_recv_key_arg { 1950 uint64_t dcrka_dsobj; 1951 uint64_t dcrka_fromobj; 1952 dmu_objset_type_t dcrka_ostype; 1953 nvlist_t *dcrka_nvl; 1954 boolean_t dcrka_do_key; 1955 } dsl_crypto_recv_key_arg_t; 1956 1957 static int 1958 dsl_crypto_recv_raw_objset_check(dsl_dataset_t *ds, dsl_dataset_t *fromds, 1959 dmu_objset_type_t ostype, nvlist_t *nvl, dmu_tx_t *tx) 1960 { 1961 int ret; 1962 objset_t *os; 1963 dnode_t *mdn; 1964 uint8_t *buf = NULL; 1965 uint_t len; 1966 uint64_t intval, nlevels, blksz, ibs; 1967 uint64_t nblkptr, maxblkid; 1968 1969 if (ostype != DMU_OST_ZFS && ostype != DMU_OST_ZVOL) 1970 return (SET_ERROR(EINVAL)); 1971 1972 /* raw receives also need info about the structure of the metadnode */ 1973 ret = nvlist_lookup_uint64(nvl, "mdn_compress", &intval); 1974 if (ret != 0 || intval >= ZIO_COMPRESS_LEGACY_FUNCTIONS) 1975 return (SET_ERROR(EINVAL)); 1976 1977 ret = nvlist_lookup_uint64(nvl, "mdn_checksum", &intval); 1978 if (ret != 0 || intval >= ZIO_CHECKSUM_LEGACY_FUNCTIONS) 1979 return (SET_ERROR(EINVAL)); 1980 1981 ret = nvlist_lookup_uint64(nvl, "mdn_nlevels", &nlevels); 1982 if (ret != 0 || nlevels > DN_MAX_LEVELS) 1983 return (SET_ERROR(EINVAL)); 1984 1985 ret = nvlist_lookup_uint64(nvl, "mdn_blksz", &blksz); 1986 if (ret != 0 || blksz < SPA_MINBLOCKSIZE) 1987 return (SET_ERROR(EINVAL)); 1988 else if (blksz > spa_maxblocksize(tx->tx_pool->dp_spa)) 1989 return (SET_ERROR(ENOTSUP)); 1990 1991 ret = nvlist_lookup_uint64(nvl, "mdn_indblkshift", &ibs); 1992 if (ret != 0 || ibs < DN_MIN_INDBLKSHIFT || ibs > DN_MAX_INDBLKSHIFT) 1993 return (SET_ERROR(ENOTSUP)); 1994 1995 ret = nvlist_lookup_uint64(nvl, "mdn_nblkptr", &nblkptr); 1996 if (ret != 0 || nblkptr != DN_MAX_NBLKPTR) 1997 return (SET_ERROR(ENOTSUP)); 1998 1999 ret = nvlist_lookup_uint64(nvl, "mdn_maxblkid", &maxblkid); 2000 if (ret != 0) 2001 return (SET_ERROR(EINVAL)); 2002 2003 ret = nvlist_lookup_uint8_array(nvl, "portable_mac", &buf, &len); 2004 if (ret != 0 || len != ZIO_OBJSET_MAC_LEN) 2005 return (SET_ERROR(EINVAL)); 2006 2007 ret = dmu_objset_from_ds(ds, &os); 2008 if (ret != 0) 2009 return (ret); 2010 2011 /* 2012 * Useraccounting is not portable and must be done with the keys loaded. 2013 * Therefore, whenever we do any kind of receive the useraccounting 2014 * must not be present. 2015 */ 2016 ASSERT0(os->os_flags & OBJSET_FLAG_USERACCOUNTING_COMPLETE); 2017 2018 mdn = DMU_META_DNODE(os); 2019 2020 /* 2021 * If we already created the objset, make sure its unchangeable 2022 * properties match the ones received in the nvlist. 2023 */ 2024 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2025 if (!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)) && 2026 (mdn->dn_nlevels != nlevels || mdn->dn_datablksz != blksz || 2027 mdn->dn_indblkshift != ibs || mdn->dn_nblkptr != nblkptr)) { 2028 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2029 return (SET_ERROR(EINVAL)); 2030 } 2031 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2032 2033 /* 2034 * Check that the ivset guid of the fromds matches the one from the 2035 * send stream. Older versions of the encryption code did not have 2036 * an ivset guid on the from dataset and did not send one in the 2037 * stream. For these streams we provide the 2038 * zfs_disable_ivset_guid_check tunable to allow these datasets to 2039 * be received with a generated ivset guid. 2040 */ 2041 if (fromds != NULL && !zfs_disable_ivset_guid_check) { 2042 uint64_t from_ivset_guid = 0; 2043 intval = 0; 2044 2045 (void) nvlist_lookup_uint64(nvl, "from_ivset_guid", &intval); 2046 (void) zap_lookup(tx->tx_pool->dp_meta_objset, 2047 fromds->ds_object, DS_FIELD_IVSET_GUID, 2048 sizeof (from_ivset_guid), 1, &from_ivset_guid); 2049 2050 if (intval == 0 || from_ivset_guid == 0) 2051 return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISSING)); 2052 2053 if (intval != from_ivset_guid) 2054 return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISMATCH)); 2055 } 2056 2057 /* 2058 * Check that the ivset guid of the fromds matches the one from the 2059 * send stream. Older versions of the encryption code did not have 2060 * an ivset guid on the from dataset and did not send one in the 2061 * stream. For these streams we provide the 2062 * zfs_disable_ivset_guid_check tunable to allow these datasets to 2063 * be received with a generated ivset guid. 2064 */ 2065 if (fromds != NULL && !zfs_disable_ivset_guid_check) { 2066 uint64_t from_ivset_guid = 0; 2067 intval = 0; 2068 2069 (void) nvlist_lookup_uint64(nvl, "from_ivset_guid", &intval); 2070 (void) zap_lookup(tx->tx_pool->dp_meta_objset, 2071 fromds->ds_object, DS_FIELD_IVSET_GUID, 2072 sizeof (from_ivset_guid), 1, &from_ivset_guid); 2073 2074 if (intval == 0 || from_ivset_guid == 0) 2075 return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISSING)); 2076 2077 if (intval != from_ivset_guid) 2078 return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISMATCH)); 2079 } 2080 2081 return (0); 2082 } 2083 2084 static void 2085 dsl_crypto_recv_raw_objset_sync(dsl_dataset_t *ds, dmu_objset_type_t ostype, 2086 nvlist_t *nvl, dmu_tx_t *tx) 2087 { 2088 dsl_pool_t *dp = tx->tx_pool; 2089 objset_t *os; 2090 dnode_t *mdn; 2091 zio_t *zio; 2092 uint8_t *portable_mac; 2093 uint_t len; 2094 uint64_t compress, checksum, nlevels, blksz, ibs, maxblkid; 2095 boolean_t newds = B_FALSE; 2096 2097 VERIFY0(dmu_objset_from_ds(ds, &os)); 2098 mdn = DMU_META_DNODE(os); 2099 2100 /* 2101 * Fetch the values we need from the nvlist. "to_ivset_guid" must 2102 * be set on the snapshot, which doesn't exist yet. The receive 2103 * code will take care of this for us later. 2104 */ 2105 compress = fnvlist_lookup_uint64(nvl, "mdn_compress"); 2106 checksum = fnvlist_lookup_uint64(nvl, "mdn_checksum"); 2107 nlevels = fnvlist_lookup_uint64(nvl, "mdn_nlevels"); 2108 blksz = fnvlist_lookup_uint64(nvl, "mdn_blksz"); 2109 ibs = fnvlist_lookup_uint64(nvl, "mdn_indblkshift"); 2110 maxblkid = fnvlist_lookup_uint64(nvl, "mdn_maxblkid"); 2111 VERIFY0(nvlist_lookup_uint8_array(nvl, "portable_mac", &portable_mac, 2112 &len)); 2113 2114 /* if we haven't created an objset for the ds yet, do that now */ 2115 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2116 if (BP_IS_HOLE(dsl_dataset_get_blkptr(ds))) { 2117 (void) dmu_objset_create_impl_dnstats(dp->dp_spa, ds, 2118 dsl_dataset_get_blkptr(ds), ostype, nlevels, blksz, 2119 ibs, tx); 2120 newds = B_TRUE; 2121 } 2122 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2123 2124 /* 2125 * Set the portable MAC. The local MAC will always be zero since the 2126 * incoming data will all be portable and user accounting will be 2127 * deferred until the next mount. Afterwards, flag the os to be 2128 * written out raw next time. 2129 */ 2130 arc_release(os->os_phys_buf, &os->os_phys_buf); 2131 bcopy(portable_mac, os->os_phys->os_portable_mac, ZIO_OBJSET_MAC_LEN); 2132 bzero(os->os_phys->os_local_mac, ZIO_OBJSET_MAC_LEN); 2133 os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE; 2134 2135 /* set metadnode compression and checksum */ 2136 mdn->dn_compress = compress; 2137 mdn->dn_checksum = checksum; 2138 2139 rw_enter(&mdn->dn_struct_rwlock, RW_WRITER); 2140 dnode_new_blkid(mdn, maxblkid, tx, B_FALSE, B_TRUE); 2141 rw_exit(&mdn->dn_struct_rwlock); 2142 2143 /* 2144 * We can't normally dirty the dataset in syncing context unless 2145 * we are creating a new dataset. In this case, we perform a 2146 * pseudo txg sync here instead. 2147 */ 2148 if (newds) { 2149 dsl_dataset_dirty(ds, tx); 2150 } else { 2151 zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED); 2152 dsl_dataset_sync(ds, zio, tx); 2153 VERIFY0(zio_wait(zio)); 2154 2155 /* dsl_dataset_sync_done will drop this reference. */ 2156 dmu_buf_add_ref(ds->ds_dbuf, ds); 2157 dsl_dataset_sync_done(ds, tx); 2158 } 2159 } 2160 2161 int 2162 dsl_crypto_recv_raw_key_check(dsl_dataset_t *ds, nvlist_t *nvl, dmu_tx_t *tx) 2163 { 2164 int ret; 2165 objset_t *mos = tx->tx_pool->dp_meta_objset; 2166 uint8_t *buf = NULL; 2167 uint_t len; 2168 uint64_t intval, key_guid, version; 2169 boolean_t is_passphrase = B_FALSE; 2170 2171 ASSERT(dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT); 2172 2173 /* 2174 * Read and check all the encryption values from the nvlist. We need 2175 * all of the fields of a DSL Crypto Key, as well as a fully specified 2176 * wrapping key. 2177 */ 2178 ret = nvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE, &intval); 2179 if (ret != 0 || intval >= ZIO_CRYPT_FUNCTIONS || 2180 intval <= ZIO_CRYPT_OFF) 2181 return (SET_ERROR(EINVAL)); 2182 2183 ret = nvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_GUID, &intval); 2184 if (ret != 0) 2185 return (SET_ERROR(EINVAL)); 2186 2187 /* 2188 * If this is an incremental receive make sure the given key guid 2189 * matches the one we already have. 2190 */ 2191 if (ds->ds_dir->dd_crypto_obj != 0) { 2192 ret = zap_lookup(mos, ds->ds_dir->dd_crypto_obj, 2193 DSL_CRYPTO_KEY_GUID, 8, 1, &key_guid); 2194 if (ret != 0) 2195 return (ret); 2196 if (intval != key_guid) 2197 return (SET_ERROR(EACCES)); 2198 } 2199 2200 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY, 2201 &buf, &len); 2202 if (ret != 0 || len != MASTER_KEY_MAX_LEN) 2203 return (SET_ERROR(EINVAL)); 2204 2205 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_HMAC_KEY, 2206 &buf, &len); 2207 if (ret != 0 || len != SHA512_HMAC_KEYLEN) 2208 return (SET_ERROR(EINVAL)); 2209 2210 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_IV, &buf, &len); 2211 if (ret != 0 || len != WRAPPING_IV_LEN) 2212 return (SET_ERROR(EINVAL)); 2213 2214 ret = nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MAC, &buf, &len); 2215 if (ret != 0 || len != WRAPPING_MAC_LEN) 2216 return (SET_ERROR(EINVAL)); 2217 2218 /* 2219 * We don't support receiving old on-disk formats. The version 0 2220 * implementation protected several fields in an objset that were 2221 * not always portable during a raw receive. As a result, we call 2222 * the old version an on-disk errata #3. 2223 */ 2224 ret = nvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_VERSION, &version); 2225 if (ret != 0 || version != ZIO_CRYPT_KEY_CURRENT_VERSION) 2226 return (SET_ERROR(ENOTSUP)); 2227 2228 ret = nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 2229 &intval); 2230 if (ret != 0 || intval >= ZFS_KEYFORMAT_FORMATS || 2231 intval == ZFS_KEYFORMAT_NONE) 2232 return (SET_ERROR(EINVAL)); 2233 2234 is_passphrase = (intval == ZFS_KEYFORMAT_PASSPHRASE); 2235 2236 /* 2237 * for raw receives we allow any number of pbkdf2iters since there 2238 * won't be a chance for the user to change it. 2239 */ 2240 ret = nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 2241 &intval); 2242 if (ret != 0 || (is_passphrase == (intval == 0))) 2243 return (SET_ERROR(EINVAL)); 2244 2245 ret = nvlist_lookup_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 2246 &intval); 2247 if (ret != 0 || (is_passphrase == (intval == 0))) 2248 return (SET_ERROR(EINVAL)); 2249 2250 return (0); 2251 } 2252 2253 void 2254 dsl_crypto_recv_raw_key_sync(dsl_dataset_t *ds, nvlist_t *nvl, dmu_tx_t *tx) 2255 { 2256 dsl_pool_t *dp = tx->tx_pool; 2257 objset_t *mos = dp->dp_meta_objset; 2258 dsl_dir_t *dd = ds->ds_dir; 2259 uint_t len; 2260 uint64_t rddobj, one = 1; 2261 uint8_t *keydata, *hmac_keydata, *iv, *mac; 2262 uint64_t crypt, key_guid, keyformat, iters, salt; 2263 uint64_t version = ZIO_CRYPT_KEY_CURRENT_VERSION; 2264 char *keylocation = "prompt"; 2265 2266 /* lookup the values we need to create the DSL Crypto Key */ 2267 crypt = fnvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE); 2268 key_guid = fnvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_GUID); 2269 keyformat = fnvlist_lookup_uint64(nvl, 2270 zfs_prop_to_name(ZFS_PROP_KEYFORMAT)); 2271 iters = fnvlist_lookup_uint64(nvl, 2272 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS)); 2273 salt = fnvlist_lookup_uint64(nvl, 2274 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT)); 2275 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY, 2276 &keydata, &len)); 2277 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_HMAC_KEY, 2278 &hmac_keydata, &len)); 2279 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_IV, &iv, &len)); 2280 VERIFY0(nvlist_lookup_uint8_array(nvl, DSL_CRYPTO_KEY_MAC, &mac, &len)); 2281 2282 /* if this is a new dataset setup the DSL Crypto Key. */ 2283 if (dd->dd_crypto_obj == 0) { 2284 /* zapify the dsl dir so we can add the key object to it */ 2285 dmu_buf_will_dirty(dd->dd_dbuf, tx); 2286 dsl_dir_zapify(dd, tx); 2287 2288 /* create the DSL Crypto Key on disk and activate the feature */ 2289 dd->dd_crypto_obj = zap_create(mos, 2290 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 2291 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, 2292 dd->dd_crypto_obj, DSL_CRYPTO_KEY_REFCOUNT, 2293 sizeof (uint64_t), 1, &one, tx)); 2294 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, 2295 dd->dd_crypto_obj, DSL_CRYPTO_KEY_VERSION, 2296 sizeof (uint64_t), 1, &version, tx)); 2297 2298 dsl_dataset_activate_feature(ds->ds_object, 2299 SPA_FEATURE_ENCRYPTION, tx); 2300 ds->ds_feature_inuse[SPA_FEATURE_ENCRYPTION] = B_TRUE; 2301 2302 /* save the dd_crypto_obj on disk */ 2303 VERIFY0(zap_add(mos, dd->dd_object, DD_FIELD_CRYPTO_KEY_OBJ, 2304 sizeof (uint64_t), 1, &dd->dd_crypto_obj, tx)); 2305 2306 /* 2307 * Set the keylocation to prompt by default. If keylocation 2308 * has been provided via the properties, this will be overridden 2309 * later. 2310 */ 2311 dsl_prop_set_sync_impl(ds, 2312 zfs_prop_to_name(ZFS_PROP_KEYLOCATION), 2313 ZPROP_SRC_LOCAL, 1, strlen(keylocation) + 1, 2314 keylocation, tx); 2315 2316 rddobj = dd->dd_object; 2317 } else { 2318 VERIFY0(dsl_dir_get_encryption_root_ddobj(dd, &rddobj)); 2319 } 2320 2321 /* sync the key data to the ZAP object on disk */ 2322 dsl_crypto_key_sync_impl(mos, dd->dd_crypto_obj, crypt, 2323 rddobj, key_guid, iv, mac, keydata, hmac_keydata, keyformat, salt, 2324 iters, tx); 2325 } 2326 2327 int 2328 dsl_crypto_recv_key_check(void *arg, dmu_tx_t *tx) 2329 { 2330 int ret; 2331 dsl_crypto_recv_key_arg_t *dcrka = arg; 2332 dsl_dataset_t *ds = NULL, *fromds = NULL; 2333 2334 ret = dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_dsobj, 2335 FTAG, &ds); 2336 if (ret != 0) 2337 goto out; 2338 2339 if (dcrka->dcrka_fromobj != 0) { 2340 ret = dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_fromobj, 2341 FTAG, &fromds); 2342 if (ret != 0) 2343 goto out; 2344 } 2345 2346 ret = dsl_crypto_recv_raw_objset_check(ds, fromds, 2347 dcrka->dcrka_ostype, dcrka->dcrka_nvl, tx); 2348 if (ret != 0) 2349 goto out; 2350 2351 /* 2352 * We run this check even if we won't be doing this part of 2353 * the receive now so that we don't make the user wait until 2354 * the receive finishes to fail. 2355 */ 2356 ret = dsl_crypto_recv_raw_key_check(ds, dcrka->dcrka_nvl, tx); 2357 if (ret != 0) 2358 goto out; 2359 2360 out: 2361 if (ds != NULL) 2362 dsl_dataset_rele(ds, FTAG); 2363 if (fromds != NULL) 2364 dsl_dataset_rele(fromds, FTAG); 2365 return (ret); 2366 } 2367 2368 void 2369 dsl_crypto_recv_key_sync(void *arg, dmu_tx_t *tx) 2370 { 2371 dsl_crypto_recv_key_arg_t *dcrka = arg; 2372 dsl_dataset_t *ds; 2373 2374 VERIFY0(dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_dsobj, 2375 FTAG, &ds)); 2376 dsl_crypto_recv_raw_objset_sync(ds, dcrka->dcrka_ostype, 2377 dcrka->dcrka_nvl, tx); 2378 if (dcrka->dcrka_do_key) 2379 dsl_crypto_recv_raw_key_sync(ds, dcrka->dcrka_nvl, tx); 2380 dsl_dataset_rele(ds, FTAG); 2381 } 2382 2383 /* 2384 * This function is used to sync an nvlist representing a DSL Crypto Key and 2385 * the associated encryption parameters. The key will be written exactly as is 2386 * without wrapping it. 2387 */ 2388 int 2389 dsl_crypto_recv_raw(const char *poolname, uint64_t dsobj, uint64_t fromobj, 2390 dmu_objset_type_t ostype, nvlist_t *nvl, boolean_t do_key) 2391 { 2392 dsl_crypto_recv_key_arg_t dcrka; 2393 2394 dcrka.dcrka_dsobj = dsobj; 2395 dcrka.dcrka_fromobj = fromobj; 2396 dcrka.dcrka_ostype = ostype; 2397 dcrka.dcrka_nvl = nvl; 2398 dcrka.dcrka_do_key = do_key; 2399 2400 return (dsl_sync_task(poolname, dsl_crypto_recv_key_check, 2401 dsl_crypto_recv_key_sync, &dcrka, 1, ZFS_SPACE_CHECK_NORMAL)); 2402 } 2403 2404 int 2405 dsl_crypto_populate_key_nvlist(dsl_dataset_t *ds, uint64_t from_ivset_guid, 2406 nvlist_t **nvl_out) 2407 { 2408 int ret; 2409 objset_t *os; 2410 dnode_t *mdn; 2411 uint64_t rddobj; 2412 nvlist_t *nvl = NULL; 2413 uint64_t dckobj = ds->ds_dir->dd_crypto_obj; 2414 dsl_dir_t *rdd = NULL; 2415 dsl_pool_t *dp = ds->ds_dir->dd_pool; 2416 objset_t *mos = dp->dp_meta_objset; 2417 uint64_t crypt = 0, key_guid = 0, format = 0; 2418 uint64_t iters = 0, salt = 0, version = 0; 2419 uint64_t to_ivset_guid = 0; 2420 uint8_t raw_keydata[MASTER_KEY_MAX_LEN]; 2421 uint8_t raw_hmac_keydata[SHA512_HMAC_KEYLEN]; 2422 uint8_t iv[WRAPPING_IV_LEN]; 2423 uint8_t mac[WRAPPING_MAC_LEN]; 2424 2425 ASSERT(dckobj != 0); 2426 2427 VERIFY0(dmu_objset_from_ds(ds, &os)); 2428 mdn = DMU_META_DNODE(os); 2429 2430 ret = nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP); 2431 if (ret != 0) 2432 goto error; 2433 2434 /* lookup values from the DSL Crypto Key */ 2435 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_CRYPTO_SUITE, 8, 1, 2436 &crypt); 2437 if (ret != 0) 2438 goto error; 2439 2440 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, &key_guid); 2441 if (ret != 0) 2442 goto error; 2443 2444 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MASTER_KEY, 1, 2445 MASTER_KEY_MAX_LEN, raw_keydata); 2446 if (ret != 0) 2447 goto error; 2448 2449 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_HMAC_KEY, 1, 2450 SHA512_HMAC_KEYLEN, raw_hmac_keydata); 2451 if (ret != 0) 2452 goto error; 2453 2454 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_IV, 1, WRAPPING_IV_LEN, 2455 iv); 2456 if (ret != 0) 2457 goto error; 2458 2459 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_MAC, 1, WRAPPING_MAC_LEN, 2460 mac); 2461 if (ret != 0) 2462 goto error; 2463 2464 /* see zfs_disable_ivset_guid_check tunable for errata info */ 2465 ret = zap_lookup(mos, ds->ds_object, DS_FIELD_IVSET_GUID, 8, 1, 2466 &to_ivset_guid); 2467 if (ret != 0) 2468 ASSERT3U(dp->dp_spa->spa_errata, !=, 0); 2469 2470 /* 2471 * We don't support raw sends of legacy on-disk formats. See the 2472 * comment in dsl_crypto_recv_key_check() for details. 2473 */ 2474 ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_VERSION, 8, 1, &version); 2475 if (ret != 0 || version != ZIO_CRYPT_KEY_CURRENT_VERSION) { 2476 dp->dp_spa->spa_errata = ZPOOL_ERRATA_ZOL_6845_ENCRYPTION; 2477 ret = SET_ERROR(ENOTSUP); 2478 goto error; 2479 } 2480 2481 /* 2482 * Lookup wrapping key properties. An early version of the code did 2483 * not correctly add these values to the wrapping key or the DSL 2484 * Crypto Key on disk for non encryption roots, so to be safe we 2485 * always take the slightly circuitous route of looking it up from 2486 * the encryption root's key. 2487 */ 2488 ret = dsl_dir_get_encryption_root_ddobj(ds->ds_dir, &rddobj); 2489 if (ret != 0) 2490 goto error; 2491 2492 dsl_pool_config_enter(dp, FTAG); 2493 2494 ret = dsl_dir_hold_obj(dp, rddobj, NULL, FTAG, &rdd); 2495 if (ret != 0) 2496 goto error_unlock; 2497 2498 ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj, 2499 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &format); 2500 if (ret != 0) 2501 goto error_unlock; 2502 2503 if (format == ZFS_KEYFORMAT_PASSPHRASE) { 2504 ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj, 2505 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &iters); 2506 if (ret != 0) 2507 goto error_unlock; 2508 2509 ret = zap_lookup(dp->dp_meta_objset, rdd->dd_crypto_obj, 2510 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &salt); 2511 if (ret != 0) 2512 goto error_unlock; 2513 } 2514 2515 dsl_dir_rele(rdd, FTAG); 2516 dsl_pool_config_exit(dp, FTAG); 2517 2518 fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE, crypt); 2519 fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_GUID, key_guid); 2520 fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_VERSION, version); 2521 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY, 2522 raw_keydata, MASTER_KEY_MAX_LEN)); 2523 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_HMAC_KEY, 2524 raw_hmac_keydata, SHA512_HMAC_KEYLEN)); 2525 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_IV, iv, 2526 WRAPPING_IV_LEN)); 2527 VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_MAC, mac, 2528 WRAPPING_MAC_LEN)); 2529 VERIFY0(nvlist_add_uint8_array(nvl, "portable_mac", 2530 os->os_phys->os_portable_mac, ZIO_OBJSET_MAC_LEN)); 2531 fnvlist_add_uint64(nvl, zfs_prop_to_name(ZFS_PROP_KEYFORMAT), format); 2532 fnvlist_add_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), iters); 2533 fnvlist_add_uint64(nvl, zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), salt); 2534 fnvlist_add_uint64(nvl, "mdn_checksum", mdn->dn_checksum); 2535 fnvlist_add_uint64(nvl, "mdn_compress", mdn->dn_compress); 2536 fnvlist_add_uint64(nvl, "mdn_nlevels", mdn->dn_nlevels); 2537 fnvlist_add_uint64(nvl, "mdn_blksz", mdn->dn_datablksz); 2538 fnvlist_add_uint64(nvl, "mdn_indblkshift", mdn->dn_indblkshift); 2539 fnvlist_add_uint64(nvl, "mdn_nblkptr", mdn->dn_nblkptr); 2540 fnvlist_add_uint64(nvl, "mdn_maxblkid", mdn->dn_maxblkid); 2541 fnvlist_add_uint64(nvl, "to_ivset_guid", to_ivset_guid); 2542 fnvlist_add_uint64(nvl, "from_ivset_guid", from_ivset_guid); 2543 2544 *nvl_out = nvl; 2545 return (0); 2546 2547 error_unlock: 2548 dsl_pool_config_exit(dp, FTAG); 2549 error: 2550 if (rdd != NULL) 2551 dsl_dir_rele(rdd, FTAG); 2552 nvlist_free(nvl); 2553 2554 *nvl_out = NULL; 2555 return (ret); 2556 } 2557 2558 uint64_t 2559 dsl_crypto_key_create_sync(uint64_t crypt, dsl_wrapping_key_t *wkey, 2560 dmu_tx_t *tx) 2561 { 2562 dsl_crypto_key_t dck; 2563 uint64_t version = ZIO_CRYPT_KEY_CURRENT_VERSION; 2564 uint64_t one = 1ULL; 2565 2566 ASSERT(dmu_tx_is_syncing(tx)); 2567 ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS); 2568 ASSERT3U(crypt, >, ZIO_CRYPT_OFF); 2569 2570 /* create the DSL Crypto Key ZAP object */ 2571 dck.dck_obj = zap_create(tx->tx_pool->dp_meta_objset, 2572 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 2573 2574 /* fill in the key (on the stack) and sync it to disk */ 2575 dck.dck_wkey = wkey; 2576 VERIFY0(zio_crypt_key_init(crypt, &dck.dck_key)); 2577 2578 dsl_crypto_key_sync(&dck, tx); 2579 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, dck.dck_obj, 2580 DSL_CRYPTO_KEY_REFCOUNT, sizeof (uint64_t), 1, &one, tx)); 2581 VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, dck.dck_obj, 2582 DSL_CRYPTO_KEY_VERSION, sizeof (uint64_t), 1, &version, tx)); 2583 2584 zio_crypt_key_destroy(&dck.dck_key); 2585 bzero(&dck.dck_key, sizeof (zio_crypt_key_t)); 2586 2587 return (dck.dck_obj); 2588 } 2589 2590 uint64_t 2591 dsl_crypto_key_clone_sync(dsl_dir_t *origindd, dmu_tx_t *tx) 2592 { 2593 objset_t *mos = tx->tx_pool->dp_meta_objset; 2594 2595 ASSERT(dmu_tx_is_syncing(tx)); 2596 2597 VERIFY0(zap_increment(mos, origindd->dd_crypto_obj, 2598 DSL_CRYPTO_KEY_REFCOUNT, 1, tx)); 2599 2600 return (origindd->dd_crypto_obj); 2601 } 2602 2603 void 2604 dsl_crypto_key_destroy_sync(uint64_t dckobj, dmu_tx_t *tx) 2605 { 2606 objset_t *mos = tx->tx_pool->dp_meta_objset; 2607 uint64_t refcnt; 2608 2609 /* Decrement the refcount, destroy if this is the last reference */ 2610 VERIFY0(zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_REFCOUNT, 2611 sizeof (uint64_t), 1, &refcnt)); 2612 2613 if (refcnt != 1) { 2614 VERIFY0(zap_increment(mos, dckobj, DSL_CRYPTO_KEY_REFCOUNT, 2615 -1, tx)); 2616 } else { 2617 VERIFY0(zap_destroy(mos, dckobj, tx)); 2618 } 2619 } 2620 2621 void 2622 dsl_dataset_crypt_stats(dsl_dataset_t *ds, nvlist_t *nv) 2623 { 2624 uint64_t intval; 2625 dsl_dir_t *dd = ds->ds_dir; 2626 dsl_dir_t *enc_root; 2627 char buf[ZFS_MAX_DATASET_NAME_LEN]; 2628 2629 if (dd->dd_crypto_obj == 0) 2630 return; 2631 2632 intval = dsl_dataset_get_keystatus(dd); 2633 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_KEYSTATUS, intval); 2634 2635 if (dsl_dir_get_crypt(dd, &intval) == 0) 2636 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_ENCRYPTION, intval); 2637 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2638 DSL_CRYPTO_KEY_GUID, 8, 1, &intval) == 0) { 2639 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_KEY_GUID, intval); 2640 } 2641 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2642 zfs_prop_to_name(ZFS_PROP_KEYFORMAT), 8, 1, &intval) == 0) { 2643 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_KEYFORMAT, intval); 2644 } 2645 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2646 zfs_prop_to_name(ZFS_PROP_PBKDF2_SALT), 8, 1, &intval) == 0) { 2647 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_PBKDF2_SALT, intval); 2648 } 2649 if (zap_lookup(dd->dd_pool->dp_meta_objset, dd->dd_crypto_obj, 2650 zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &intval) == 0) { 2651 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_PBKDF2_ITERS, intval); 2652 } 2653 if (zap_lookup(dd->dd_pool->dp_meta_objset, ds->ds_object, 2654 DS_FIELD_IVSET_GUID, 8, 1, &intval) == 0) { 2655 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_IVSET_GUID, intval); 2656 } 2657 2658 if (dsl_dir_get_encryption_root_ddobj(dd, &intval) == 0) { 2659 VERIFY0(dsl_dir_hold_obj(dd->dd_pool, intval, NULL, FTAG, 2660 &enc_root)); 2661 dsl_dir_name(enc_root, buf); 2662 dsl_dir_rele(enc_root, FTAG); 2663 dsl_prop_nvlist_add_string(nv, ZFS_PROP_ENCRYPTION_ROOT, buf); 2664 } 2665 } 2666 2667 int 2668 spa_crypt_get_salt(spa_t *spa, uint64_t dsobj, uint8_t *salt) 2669 { 2670 int ret; 2671 dsl_crypto_key_t *dck = NULL; 2672 2673 /* look up the key from the spa's keystore */ 2674 ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck); 2675 if (ret != 0) 2676 goto error; 2677 2678 ret = zio_crypt_key_get_salt(&dck->dck_key, salt); 2679 if (ret != 0) 2680 goto error; 2681 2682 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2683 return (0); 2684 2685 error: 2686 if (dck != NULL) 2687 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2688 return (ret); 2689 } 2690 2691 /* 2692 * Objset blocks are a special case for MAC generation. These blocks have 2 2693 * 256-bit MACs which are embedded within the block itself, rather than a 2694 * single 128 bit MAC. As a result, this function handles encoding and decoding 2695 * the MACs on its own, unlike other functions in this file. 2696 */ 2697 int 2698 spa_do_crypt_objset_mac_abd(boolean_t generate, spa_t *spa, uint64_t dsobj, 2699 abd_t *abd, uint_t datalen, boolean_t byteswap) 2700 { 2701 int ret; 2702 dsl_crypto_key_t *dck = NULL; 2703 void *buf = abd_borrow_buf_copy(abd, datalen); 2704 objset_phys_t *osp = buf; 2705 uint8_t portable_mac[ZIO_OBJSET_MAC_LEN]; 2706 uint8_t local_mac[ZIO_OBJSET_MAC_LEN]; 2707 2708 /* look up the key from the spa's keystore */ 2709 ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck); 2710 if (ret != 0) 2711 goto error; 2712 2713 /* calculate both HMACs */ 2714 ret = zio_crypt_do_objset_hmacs(&dck->dck_key, buf, datalen, 2715 byteswap, portable_mac, local_mac); 2716 if (ret != 0) 2717 goto error; 2718 2719 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2720 2721 /* if we are generating encode the HMACs in the objset_phys_t */ 2722 if (generate) { 2723 bcopy(portable_mac, osp->os_portable_mac, ZIO_OBJSET_MAC_LEN); 2724 bcopy(local_mac, osp->os_local_mac, ZIO_OBJSET_MAC_LEN); 2725 abd_return_buf_copy(abd, buf, datalen); 2726 return (0); 2727 } 2728 2729 if (bcmp(portable_mac, osp->os_portable_mac, ZIO_OBJSET_MAC_LEN) != 0 || 2730 bcmp(local_mac, osp->os_local_mac, ZIO_OBJSET_MAC_LEN) != 0) { 2731 abd_return_buf(abd, buf, datalen); 2732 return (SET_ERROR(ECKSUM)); 2733 } 2734 2735 abd_return_buf(abd, buf, datalen); 2736 2737 return (0); 2738 2739 error: 2740 if (dck != NULL) 2741 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2742 abd_return_buf(abd, buf, datalen); 2743 return (ret); 2744 } 2745 2746 int 2747 spa_do_crypt_mac_abd(boolean_t generate, spa_t *spa, uint64_t dsobj, abd_t *abd, 2748 uint_t datalen, uint8_t *mac) 2749 { 2750 int ret; 2751 dsl_crypto_key_t *dck = NULL; 2752 uint8_t *buf = abd_borrow_buf_copy(abd, datalen); 2753 uint8_t digestbuf[ZIO_DATA_MAC_LEN]; 2754 2755 /* look up the key from the spa's keystore */ 2756 ret = spa_keystore_lookup_key(spa, dsobj, FTAG, &dck); 2757 if (ret != 0) 2758 goto error; 2759 2760 /* perform the hmac */ 2761 ret = zio_crypt_do_hmac(&dck->dck_key, buf, datalen, 2762 digestbuf, ZIO_DATA_MAC_LEN); 2763 if (ret != 0) 2764 goto error; 2765 2766 abd_return_buf(abd, buf, datalen); 2767 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2768 2769 /* 2770 * Truncate and fill in mac buffer if we were asked to generate a MAC. 2771 * Otherwise verify that the MAC matched what we expected. 2772 */ 2773 if (generate) { 2774 bcopy(digestbuf, mac, ZIO_DATA_MAC_LEN); 2775 return (0); 2776 } 2777 2778 if (bcmp(digestbuf, mac, ZIO_DATA_MAC_LEN) != 0) 2779 return (SET_ERROR(ECKSUM)); 2780 2781 return (0); 2782 2783 error: 2784 if (dck != NULL) 2785 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2786 abd_return_buf(abd, buf, datalen); 2787 return (ret); 2788 } 2789 2790 /* 2791 * This function serves as a multiplexer for encryption and decryption of 2792 * all blocks (except the L2ARC). For encryption, it will populate the IV, 2793 * salt, MAC, and cabd (the ciphertext). On decryption it will simply use 2794 * these fields to populate pabd (the plaintext). 2795 */ 2796 /* ARGSUSED */ 2797 int 2798 spa_do_crypt_abd(boolean_t encrypt, spa_t *spa, const zbookmark_phys_t *zb, 2799 dmu_object_type_t ot, boolean_t dedup, boolean_t bswap, uint8_t *salt, 2800 uint8_t *iv, uint8_t *mac, uint_t datalen, abd_t *pabd, abd_t *cabd, 2801 boolean_t *no_crypt) 2802 { 2803 int ret; 2804 dsl_crypto_key_t *dck = NULL; 2805 uint8_t *plainbuf = NULL, *cipherbuf = NULL; 2806 2807 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION)); 2808 2809 /* look up the key from the spa's keystore */ 2810 ret = spa_keystore_lookup_key(spa, zb->zb_objset, FTAG, &dck); 2811 if (ret != 0) { 2812 ret = SET_ERROR(EACCES); 2813 return (ret); 2814 } 2815 2816 if (encrypt) { 2817 plainbuf = abd_borrow_buf_copy(pabd, datalen); 2818 cipherbuf = abd_borrow_buf(cabd, datalen); 2819 } else { 2820 plainbuf = abd_borrow_buf(pabd, datalen); 2821 cipherbuf = abd_borrow_buf_copy(cabd, datalen); 2822 } 2823 2824 /* 2825 * Both encryption and decryption functions need a salt for key 2826 * generation and an IV. When encrypting a non-dedup block, we 2827 * generate the salt and IV randomly to be stored by the caller. Dedup 2828 * blocks perform a (more expensive) HMAC of the plaintext to obtain 2829 * the salt and the IV. ZIL blocks have their salt and IV generated 2830 * at allocation time in zio_alloc_zil(). On decryption, we simply use 2831 * the provided values. 2832 */ 2833 if (encrypt && ot != DMU_OT_INTENT_LOG && !dedup) { 2834 ret = zio_crypt_key_get_salt(&dck->dck_key, salt); 2835 if (ret != 0) 2836 goto error; 2837 2838 ret = zio_crypt_generate_iv(iv); 2839 if (ret != 0) 2840 goto error; 2841 } else if (encrypt && dedup) { 2842 ret = zio_crypt_generate_iv_salt_dedup(&dck->dck_key, 2843 plainbuf, datalen, iv, salt); 2844 if (ret != 0) 2845 goto error; 2846 } 2847 2848 /* call lower level function to perform encryption / decryption */ 2849 ret = zio_do_crypt_data(encrypt, &dck->dck_key, ot, bswap, salt, iv, 2850 mac, datalen, plainbuf, cipherbuf, no_crypt); 2851 2852 /* 2853 * Handle injected decryption faults. Unfortunately, we cannot inject 2854 * faults for dnode blocks because we might trigger the panic in 2855 * dbuf_prepare_encrypted_dnode_leaf(), which exists because syncing 2856 * context is not prepared to handle malicious decryption failures. 2857 */ 2858 if (zio_injection_enabled && !encrypt && ot != DMU_OT_DNODE && ret == 0) 2859 ret = zio_handle_decrypt_injection(spa, zb, ot, ECKSUM); 2860 if (ret != 0) 2861 goto error; 2862 2863 if (encrypt) { 2864 abd_return_buf(pabd, plainbuf, datalen); 2865 abd_return_buf_copy(cabd, cipherbuf, datalen); 2866 } else { 2867 abd_return_buf_copy(pabd, plainbuf, datalen); 2868 abd_return_buf(cabd, cipherbuf, datalen); 2869 } 2870 2871 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2872 2873 return (0); 2874 2875 error: 2876 if (encrypt) { 2877 /* zero out any state we might have changed while encrypting */ 2878 bzero(salt, ZIO_DATA_SALT_LEN); 2879 bzero(iv, ZIO_DATA_IV_LEN); 2880 bzero(mac, ZIO_DATA_MAC_LEN); 2881 abd_return_buf(pabd, plainbuf, datalen); 2882 abd_return_buf_copy(cabd, cipherbuf, datalen); 2883 } else { 2884 abd_return_buf_copy(pabd, plainbuf, datalen); 2885 abd_return_buf(cabd, cipherbuf, datalen); 2886 } 2887 2888 spa_keystore_dsl_key_rele(spa, dck, FTAG); 2889 2890 return (ret); 2891 } 2892