1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2014, Joyent, Inc. All rights reserved. 26 * Copyright 2014 HybridCluster. All rights reserved. 27 * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved. 28 * Copyright (c) 2019, Klara Inc. 29 * Copyright (c) 2019, Allan Jude 30 * Copyright (c) 2019 Datto Inc. 31 * Copyright (c) 2022 Axcient. 32 */ 33 34 #include <sys/arc.h> 35 #include <sys/spa_impl.h> 36 #include <sys/dmu.h> 37 #include <sys/dmu_impl.h> 38 #include <sys/dmu_send.h> 39 #include <sys/dmu_recv.h> 40 #include <sys/dmu_tx.h> 41 #include <sys/dbuf.h> 42 #include <sys/dnode.h> 43 #include <sys/zfs_context.h> 44 #include <sys/dmu_objset.h> 45 #include <sys/dmu_traverse.h> 46 #include <sys/dsl_dataset.h> 47 #include <sys/dsl_dir.h> 48 #include <sys/dsl_prop.h> 49 #include <sys/dsl_pool.h> 50 #include <sys/dsl_synctask.h> 51 #include <sys/zfs_ioctl.h> 52 #include <sys/zap.h> 53 #include <sys/zvol.h> 54 #include <sys/zio_checksum.h> 55 #include <sys/zfs_znode.h> 56 #include <zfs_fletcher.h> 57 #include <sys/avl.h> 58 #include <sys/ddt.h> 59 #include <sys/zfs_onexit.h> 60 #include <sys/dsl_destroy.h> 61 #include <sys/blkptr.h> 62 #include <sys/dsl_bookmark.h> 63 #include <sys/zfeature.h> 64 #include <sys/bqueue.h> 65 #include <sys/objlist.h> 66 #ifdef _KERNEL 67 #include <sys/zfs_vfsops.h> 68 #endif 69 #include <sys/zfs_file.h> 70 71 static uint_t zfs_recv_queue_length = SPA_MAXBLOCKSIZE; 72 static uint_t zfs_recv_queue_ff = 20; 73 static uint_t zfs_recv_write_batch_size = 1024 * 1024; 74 static int zfs_recv_best_effort_corrective = 0; 75 76 static const void *const dmu_recv_tag = "dmu_recv_tag"; 77 const char *const recv_clone_name = "%recv"; 78 79 typedef enum { 80 ORNS_NO, 81 ORNS_YES, 82 ORNS_MAYBE 83 } or_need_sync_t; 84 85 static int receive_read_payload_and_next_header(dmu_recv_cookie_t *ra, int len, 86 void *buf); 87 88 struct receive_record_arg { 89 dmu_replay_record_t header; 90 void *payload; /* Pointer to a buffer containing the payload */ 91 /* 92 * If the record is a WRITE or SPILL, pointer to the abd containing the 93 * payload. 94 */ 95 abd_t *abd; 96 int payload_size; 97 uint64_t bytes_read; /* bytes read from stream when record created */ 98 boolean_t eos_marker; /* Marks the end of the stream */ 99 bqueue_node_t node; 100 }; 101 102 struct receive_writer_arg { 103 objset_t *os; 104 boolean_t byteswap; 105 bqueue_t q; 106 107 /* 108 * These three members are used to signal to the main thread when 109 * we're done. 110 */ 111 kmutex_t mutex; 112 kcondvar_t cv; 113 boolean_t done; 114 115 int err; 116 const char *tofs; 117 boolean_t heal; 118 boolean_t resumable; 119 boolean_t raw; /* DMU_BACKUP_FEATURE_RAW set */ 120 boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */ 121 boolean_t full; /* this is a full send stream */ 122 uint64_t last_object; 123 uint64_t last_offset; 124 uint64_t max_object; /* highest object ID referenced in stream */ 125 uint64_t bytes_read; /* bytes read when current record created */ 126 127 list_t write_batch; 128 129 /* Encryption parameters for the last received DRR_OBJECT_RANGE */ 130 boolean_t or_crypt_params_present; 131 uint64_t or_firstobj; 132 uint64_t or_numslots; 133 uint8_t or_salt[ZIO_DATA_SALT_LEN]; 134 uint8_t or_iv[ZIO_DATA_IV_LEN]; 135 uint8_t or_mac[ZIO_DATA_MAC_LEN]; 136 boolean_t or_byteorder; 137 zio_t *heal_pio; 138 139 /* Keep track of DRR_FREEOBJECTS right after DRR_OBJECT_RANGE */ 140 or_need_sync_t or_need_sync; 141 }; 142 143 typedef struct dmu_recv_begin_arg { 144 const char *drba_origin; 145 dmu_recv_cookie_t *drba_cookie; 146 cred_t *drba_cred; 147 proc_t *drba_proc; 148 dsl_crypto_params_t *drba_dcp; 149 } dmu_recv_begin_arg_t; 150 151 static void 152 byteswap_record(dmu_replay_record_t *drr) 153 { 154 #define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X)) 155 #define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X)) 156 drr->drr_type = BSWAP_32(drr->drr_type); 157 drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen); 158 159 switch (drr->drr_type) { 160 case DRR_BEGIN: 161 DO64(drr_begin.drr_magic); 162 DO64(drr_begin.drr_versioninfo); 163 DO64(drr_begin.drr_creation_time); 164 DO32(drr_begin.drr_type); 165 DO32(drr_begin.drr_flags); 166 DO64(drr_begin.drr_toguid); 167 DO64(drr_begin.drr_fromguid); 168 break; 169 case DRR_OBJECT: 170 DO64(drr_object.drr_object); 171 DO32(drr_object.drr_type); 172 DO32(drr_object.drr_bonustype); 173 DO32(drr_object.drr_blksz); 174 DO32(drr_object.drr_bonuslen); 175 DO32(drr_object.drr_raw_bonuslen); 176 DO64(drr_object.drr_toguid); 177 DO64(drr_object.drr_maxblkid); 178 break; 179 case DRR_FREEOBJECTS: 180 DO64(drr_freeobjects.drr_firstobj); 181 DO64(drr_freeobjects.drr_numobjs); 182 DO64(drr_freeobjects.drr_toguid); 183 break; 184 case DRR_WRITE: 185 DO64(drr_write.drr_object); 186 DO32(drr_write.drr_type); 187 DO64(drr_write.drr_offset); 188 DO64(drr_write.drr_logical_size); 189 DO64(drr_write.drr_toguid); 190 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum); 191 DO64(drr_write.drr_key.ddk_prop); 192 DO64(drr_write.drr_compressed_size); 193 break; 194 case DRR_WRITE_EMBEDDED: 195 DO64(drr_write_embedded.drr_object); 196 DO64(drr_write_embedded.drr_offset); 197 DO64(drr_write_embedded.drr_length); 198 DO64(drr_write_embedded.drr_toguid); 199 DO32(drr_write_embedded.drr_lsize); 200 DO32(drr_write_embedded.drr_psize); 201 break; 202 case DRR_FREE: 203 DO64(drr_free.drr_object); 204 DO64(drr_free.drr_offset); 205 DO64(drr_free.drr_length); 206 DO64(drr_free.drr_toguid); 207 break; 208 case DRR_SPILL: 209 DO64(drr_spill.drr_object); 210 DO64(drr_spill.drr_length); 211 DO64(drr_spill.drr_toguid); 212 DO64(drr_spill.drr_compressed_size); 213 DO32(drr_spill.drr_type); 214 break; 215 case DRR_OBJECT_RANGE: 216 DO64(drr_object_range.drr_firstobj); 217 DO64(drr_object_range.drr_numslots); 218 DO64(drr_object_range.drr_toguid); 219 break; 220 case DRR_REDACT: 221 DO64(drr_redact.drr_object); 222 DO64(drr_redact.drr_offset); 223 DO64(drr_redact.drr_length); 224 DO64(drr_redact.drr_toguid); 225 break; 226 case DRR_END: 227 DO64(drr_end.drr_toguid); 228 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum); 229 break; 230 default: 231 break; 232 } 233 234 if (drr->drr_type != DRR_BEGIN) { 235 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum); 236 } 237 238 #undef DO64 239 #undef DO32 240 } 241 242 static boolean_t 243 redact_snaps_contains(uint64_t *snaps, uint64_t num_snaps, uint64_t guid) 244 { 245 for (int i = 0; i < num_snaps; i++) { 246 if (snaps[i] == guid) 247 return (B_TRUE); 248 } 249 return (B_FALSE); 250 } 251 252 /* 253 * Check that the new stream we're trying to receive is redacted with respect to 254 * a subset of the snapshots that the origin was redacted with respect to. For 255 * the reasons behind this, see the man page on redacted zfs sends and receives. 256 */ 257 static boolean_t 258 compatible_redact_snaps(uint64_t *origin_snaps, uint64_t origin_num_snaps, 259 uint64_t *redact_snaps, uint64_t num_redact_snaps) 260 { 261 /* 262 * Short circuit the comparison; if we are redacted with respect to 263 * more snapshots than the origin, we can't be redacted with respect 264 * to a subset. 265 */ 266 if (num_redact_snaps > origin_num_snaps) { 267 return (B_FALSE); 268 } 269 270 for (int i = 0; i < num_redact_snaps; i++) { 271 if (!redact_snaps_contains(origin_snaps, origin_num_snaps, 272 redact_snaps[i])) { 273 return (B_FALSE); 274 } 275 } 276 return (B_TRUE); 277 } 278 279 static boolean_t 280 redact_check(dmu_recv_begin_arg_t *drba, dsl_dataset_t *origin) 281 { 282 uint64_t *origin_snaps; 283 uint64_t origin_num_snaps; 284 dmu_recv_cookie_t *drc = drba->drba_cookie; 285 struct drr_begin *drrb = drc->drc_drrb; 286 int featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); 287 int err = 0; 288 boolean_t ret = B_TRUE; 289 uint64_t *redact_snaps; 290 uint_t numredactsnaps; 291 292 /* 293 * If this is a full send stream, we're safe no matter what. 294 */ 295 if (drrb->drr_fromguid == 0) 296 return (ret); 297 298 VERIFY(dsl_dataset_get_uint64_array_feature(origin, 299 SPA_FEATURE_REDACTED_DATASETS, &origin_num_snaps, &origin_snaps)); 300 301 if (nvlist_lookup_uint64_array(drc->drc_begin_nvl, 302 BEGINNV_REDACT_FROM_SNAPS, &redact_snaps, &numredactsnaps) == 303 0) { 304 /* 305 * If the send stream was sent from the redaction bookmark or 306 * the redacted version of the dataset, then we're safe. Verify 307 * that this is from the a compatible redaction bookmark or 308 * redacted dataset. 309 */ 310 if (!compatible_redact_snaps(origin_snaps, origin_num_snaps, 311 redact_snaps, numredactsnaps)) { 312 err = EINVAL; 313 } 314 } else if (featureflags & DMU_BACKUP_FEATURE_REDACTED) { 315 /* 316 * If the stream is redacted, it must be redacted with respect 317 * to a subset of what the origin is redacted with respect to. 318 * See case number 2 in the zfs man page section on redacted zfs 319 * send. 320 */ 321 err = nvlist_lookup_uint64_array(drc->drc_begin_nvl, 322 BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps); 323 324 if (err != 0 || !compatible_redact_snaps(origin_snaps, 325 origin_num_snaps, redact_snaps, numredactsnaps)) { 326 err = EINVAL; 327 } 328 } else if (!redact_snaps_contains(origin_snaps, origin_num_snaps, 329 drrb->drr_toguid)) { 330 /* 331 * If the stream isn't redacted but the origin is, this must be 332 * one of the snapshots the origin is redacted with respect to. 333 * See case number 1 in the zfs man page section on redacted zfs 334 * send. 335 */ 336 err = EINVAL; 337 } 338 339 if (err != 0) 340 ret = B_FALSE; 341 return (ret); 342 } 343 344 /* 345 * If we previously received a stream with --large-block, we don't support 346 * receiving an incremental on top of it without --large-block. This avoids 347 * forcing a read-modify-write or trying to re-aggregate a string of WRITE 348 * records. 349 */ 350 static int 351 recv_check_large_blocks(dsl_dataset_t *ds, uint64_t featureflags) 352 { 353 if (dsl_dataset_feature_is_active(ds, SPA_FEATURE_LARGE_BLOCKS) && 354 !(featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS)) 355 return (SET_ERROR(ZFS_ERR_STREAM_LARGE_BLOCK_MISMATCH)); 356 return (0); 357 } 358 359 static int 360 recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds, 361 uint64_t fromguid, uint64_t featureflags) 362 { 363 uint64_t obj; 364 uint64_t children; 365 int error; 366 dsl_dataset_t *snap; 367 dsl_pool_t *dp = ds->ds_dir->dd_pool; 368 boolean_t encrypted = ds->ds_dir->dd_crypto_obj != 0; 369 boolean_t raw = (featureflags & DMU_BACKUP_FEATURE_RAW) != 0; 370 boolean_t embed = (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) != 0; 371 372 /* Temporary clone name must not exist. */ 373 error = zap_lookup(dp->dp_meta_objset, 374 dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, recv_clone_name, 375 8, 1, &obj); 376 if (error != ENOENT) 377 return (error == 0 ? SET_ERROR(EBUSY) : error); 378 379 /* Resume state must not be set. */ 380 if (dsl_dataset_has_resume_receive_state(ds)) 381 return (SET_ERROR(EBUSY)); 382 383 /* New snapshot name must not exist if we're not healing it. */ 384 error = zap_lookup(dp->dp_meta_objset, 385 dsl_dataset_phys(ds)->ds_snapnames_zapobj, 386 drba->drba_cookie->drc_tosnap, 8, 1, &obj); 387 if (drba->drba_cookie->drc_heal) { 388 if (error != 0) 389 return (error); 390 } else if (error != ENOENT) { 391 return (error == 0 ? SET_ERROR(EEXIST) : error); 392 } 393 394 /* Must not have children if receiving a ZVOL. */ 395 error = zap_count(dp->dp_meta_objset, 396 dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, &children); 397 if (error != 0) 398 return (error); 399 if (drba->drba_cookie->drc_drrb->drr_type != DMU_OST_ZFS && 400 children > 0) 401 return (SET_ERROR(ZFS_ERR_WRONG_PARENT)); 402 403 /* 404 * Check snapshot limit before receiving. We'll recheck again at the 405 * end, but might as well abort before receiving if we're already over 406 * the limit. 407 * 408 * Note that we do not check the file system limit with 409 * dsl_dir_fscount_check because the temporary %clones don't count 410 * against that limit. 411 */ 412 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, ZFS_PROP_SNAPSHOT_LIMIT, 413 NULL, drba->drba_cred, drba->drba_proc); 414 if (error != 0) 415 return (error); 416 417 if (drba->drba_cookie->drc_heal) { 418 /* Encryption is incompatible with embedded data. */ 419 if (encrypted && embed) 420 return (SET_ERROR(EINVAL)); 421 422 /* Healing is not supported when in 'force' mode. */ 423 if (drba->drba_cookie->drc_force) 424 return (SET_ERROR(EINVAL)); 425 426 /* Must have keys loaded if doing encrypted non-raw recv. */ 427 if (encrypted && !raw) { 428 if (spa_keystore_lookup_key(dp->dp_spa, ds->ds_object, 429 NULL, NULL) != 0) 430 return (SET_ERROR(EACCES)); 431 } 432 433 error = dsl_dataset_hold_obj(dp, obj, FTAG, &snap); 434 if (error != 0) 435 return (error); 436 437 /* 438 * When not doing best effort corrective recv healing can only 439 * be done if the send stream is for the same snapshot as the 440 * one we are trying to heal. 441 */ 442 if (zfs_recv_best_effort_corrective == 0 && 443 drba->drba_cookie->drc_drrb->drr_toguid != 444 dsl_dataset_phys(snap)->ds_guid) { 445 dsl_dataset_rele(snap, FTAG); 446 return (SET_ERROR(ENOTSUP)); 447 } 448 dsl_dataset_rele(snap, FTAG); 449 } else if (fromguid != 0) { 450 /* Sanity check the incremental recv */ 451 uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj; 452 453 /* Can't perform a raw receive on top of a non-raw receive */ 454 if (!encrypted && raw) 455 return (SET_ERROR(EINVAL)); 456 457 /* Encryption is incompatible with embedded data */ 458 if (encrypted && embed) 459 return (SET_ERROR(EINVAL)); 460 461 /* Find snapshot in this dir that matches fromguid. */ 462 while (obj != 0) { 463 error = dsl_dataset_hold_obj(dp, obj, FTAG, 464 &snap); 465 if (error != 0) 466 return (SET_ERROR(ENODEV)); 467 if (snap->ds_dir != ds->ds_dir) { 468 dsl_dataset_rele(snap, FTAG); 469 return (SET_ERROR(ENODEV)); 470 } 471 if (dsl_dataset_phys(snap)->ds_guid == fromguid) 472 break; 473 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj; 474 dsl_dataset_rele(snap, FTAG); 475 } 476 if (obj == 0) 477 return (SET_ERROR(ENODEV)); 478 479 if (drba->drba_cookie->drc_force) { 480 drba->drba_cookie->drc_fromsnapobj = obj; 481 } else { 482 /* 483 * If we are not forcing, there must be no 484 * changes since fromsnap. Raw sends have an 485 * additional constraint that requires that 486 * no "noop" snapshots exist between fromsnap 487 * and tosnap for the IVset checking code to 488 * work properly. 489 */ 490 if (dsl_dataset_modified_since_snap(ds, snap) || 491 (raw && 492 dsl_dataset_phys(ds)->ds_prev_snap_obj != 493 snap->ds_object)) { 494 dsl_dataset_rele(snap, FTAG); 495 return (SET_ERROR(ETXTBSY)); 496 } 497 drba->drba_cookie->drc_fromsnapobj = 498 ds->ds_prev->ds_object; 499 } 500 501 if (dsl_dataset_feature_is_active(snap, 502 SPA_FEATURE_REDACTED_DATASETS) && !redact_check(drba, 503 snap)) { 504 dsl_dataset_rele(snap, FTAG); 505 return (SET_ERROR(EINVAL)); 506 } 507 508 error = recv_check_large_blocks(snap, featureflags); 509 if (error != 0) { 510 dsl_dataset_rele(snap, FTAG); 511 return (error); 512 } 513 514 dsl_dataset_rele(snap, FTAG); 515 } else { 516 /* If full and not healing then must be forced. */ 517 if (!drba->drba_cookie->drc_force) 518 return (SET_ERROR(EEXIST)); 519 520 /* 521 * We don't support using zfs recv -F to blow away 522 * encrypted filesystems. This would require the 523 * dsl dir to point to the old encryption key and 524 * the new one at the same time during the receive. 525 */ 526 if ((!encrypted && raw) || encrypted) 527 return (SET_ERROR(EINVAL)); 528 529 /* 530 * Perform the same encryption checks we would if 531 * we were creating a new dataset from scratch. 532 */ 533 if (!raw) { 534 boolean_t will_encrypt; 535 536 error = dmu_objset_create_crypt_check( 537 ds->ds_dir->dd_parent, drba->drba_dcp, 538 &will_encrypt); 539 if (error != 0) 540 return (error); 541 542 if (will_encrypt && embed) 543 return (SET_ERROR(EINVAL)); 544 } 545 } 546 547 return (0); 548 } 549 550 /* 551 * Check that any feature flags used in the data stream we're receiving are 552 * supported by the pool we are receiving into. 553 * 554 * Note that some of the features we explicitly check here have additional 555 * (implicit) features they depend on, but those dependencies are enforced 556 * through the zfeature_register() calls declaring the features that we 557 * explicitly check. 558 */ 559 static int 560 recv_begin_check_feature_flags_impl(uint64_t featureflags, spa_t *spa) 561 { 562 /* 563 * Check if there are any unsupported feature flags. 564 */ 565 if (!DMU_STREAM_SUPPORTED(featureflags)) { 566 return (SET_ERROR(ZFS_ERR_UNKNOWN_SEND_STREAM_FEATURE)); 567 } 568 569 /* Verify pool version supports SA if SA_SPILL feature set */ 570 if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) && 571 spa_version(spa) < SPA_VERSION_SA) 572 return (SET_ERROR(ENOTSUP)); 573 574 /* 575 * LZ4 compressed, ZSTD compressed, embedded, mooched, large blocks, 576 * and large_dnodes in the stream can only be used if those pool 577 * features are enabled because we don't attempt to decompress / 578 * un-embed / un-mooch / split up the blocks / dnodes during the 579 * receive process. 580 */ 581 if ((featureflags & DMU_BACKUP_FEATURE_LZ4) && 582 !spa_feature_is_enabled(spa, SPA_FEATURE_LZ4_COMPRESS)) 583 return (SET_ERROR(ENOTSUP)); 584 if ((featureflags & DMU_BACKUP_FEATURE_ZSTD) && 585 !spa_feature_is_enabled(spa, SPA_FEATURE_ZSTD_COMPRESS)) 586 return (SET_ERROR(ENOTSUP)); 587 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) && 588 !spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) 589 return (SET_ERROR(ENOTSUP)); 590 if ((featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) && 591 !spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) 592 return (SET_ERROR(ENOTSUP)); 593 if ((featureflags & DMU_BACKUP_FEATURE_LARGE_DNODE) && 594 !spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) 595 return (SET_ERROR(ENOTSUP)); 596 597 /* 598 * Receiving redacted streams requires that redacted datasets are 599 * enabled. 600 */ 601 if ((featureflags & DMU_BACKUP_FEATURE_REDACTED) && 602 !spa_feature_is_enabled(spa, SPA_FEATURE_REDACTED_DATASETS)) 603 return (SET_ERROR(ENOTSUP)); 604 605 return (0); 606 } 607 608 static int 609 dmu_recv_begin_check(void *arg, dmu_tx_t *tx) 610 { 611 dmu_recv_begin_arg_t *drba = arg; 612 dsl_pool_t *dp = dmu_tx_pool(tx); 613 struct drr_begin *drrb = drba->drba_cookie->drc_drrb; 614 uint64_t fromguid = drrb->drr_fromguid; 615 int flags = drrb->drr_flags; 616 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 617 int error; 618 uint64_t featureflags = drba->drba_cookie->drc_featureflags; 619 dsl_dataset_t *ds; 620 const char *tofs = drba->drba_cookie->drc_tofs; 621 622 /* already checked */ 623 ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); 624 ASSERT(!(featureflags & DMU_BACKUP_FEATURE_RESUMING)); 625 626 if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == 627 DMU_COMPOUNDSTREAM || 628 drrb->drr_type >= DMU_OST_NUMTYPES || 629 ((flags & DRR_FLAG_CLONE) && drba->drba_origin == NULL)) 630 return (SET_ERROR(EINVAL)); 631 632 error = recv_begin_check_feature_flags_impl(featureflags, dp->dp_spa); 633 if (error != 0) 634 return (error); 635 636 /* Resumable receives require extensible datasets */ 637 if (drba->drba_cookie->drc_resumable && 638 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EXTENSIBLE_DATASET)) 639 return (SET_ERROR(ENOTSUP)); 640 641 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 642 /* raw receives require the encryption feature */ 643 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) 644 return (SET_ERROR(ENOTSUP)); 645 646 /* embedded data is incompatible with encryption and raw recv */ 647 if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) 648 return (SET_ERROR(EINVAL)); 649 650 /* raw receives require spill block allocation flag */ 651 if (!(flags & DRR_FLAG_SPILL_BLOCK)) 652 return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING)); 653 } else { 654 /* 655 * We support unencrypted datasets below encrypted ones now, 656 * so add the DS_HOLD_FLAG_DECRYPT flag only if we are dealing 657 * with a dataset we may encrypt. 658 */ 659 if (drba->drba_dcp == NULL || 660 drba->drba_dcp->cp_crypt != ZIO_CRYPT_OFF) { 661 dsflags |= DS_HOLD_FLAG_DECRYPT; 662 } 663 } 664 665 error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds); 666 if (error == 0) { 667 /* target fs already exists; recv into temp clone */ 668 669 /* Can't recv a clone into an existing fs */ 670 if (flags & DRR_FLAG_CLONE || drba->drba_origin) { 671 dsl_dataset_rele_flags(ds, dsflags, FTAG); 672 return (SET_ERROR(EINVAL)); 673 } 674 675 error = recv_begin_check_existing_impl(drba, ds, fromguid, 676 featureflags); 677 dsl_dataset_rele_flags(ds, dsflags, FTAG); 678 } else if (error == ENOENT) { 679 /* target fs does not exist; must be a full backup or clone */ 680 char buf[ZFS_MAX_DATASET_NAME_LEN]; 681 objset_t *os; 682 683 /* healing recv must be done "into" an existing snapshot */ 684 if (drba->drba_cookie->drc_heal == B_TRUE) 685 return (SET_ERROR(ENOTSUP)); 686 687 /* 688 * If it's a non-clone incremental, we are missing the 689 * target fs, so fail the recv. 690 */ 691 if (fromguid != 0 && !((flags & DRR_FLAG_CLONE) || 692 drba->drba_origin)) 693 return (SET_ERROR(ENOENT)); 694 695 /* 696 * If we're receiving a full send as a clone, and it doesn't 697 * contain all the necessary free records and freeobject 698 * records, reject it. 699 */ 700 if (fromguid == 0 && drba->drba_origin != NULL && 701 !(flags & DRR_FLAG_FREERECORDS)) 702 return (SET_ERROR(EINVAL)); 703 704 /* Open the parent of tofs */ 705 ASSERT3U(strlen(tofs), <, sizeof (buf)); 706 (void) strlcpy(buf, tofs, strrchr(tofs, '/') - tofs + 1); 707 error = dsl_dataset_hold(dp, buf, FTAG, &ds); 708 if (error != 0) 709 return (error); 710 711 if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0 && 712 drba->drba_origin == NULL) { 713 boolean_t will_encrypt; 714 715 /* 716 * Check that we aren't breaking any encryption rules 717 * and that we have all the parameters we need to 718 * create an encrypted dataset if necessary. If we are 719 * making an encrypted dataset the stream can't have 720 * embedded data. 721 */ 722 error = dmu_objset_create_crypt_check(ds->ds_dir, 723 drba->drba_dcp, &will_encrypt); 724 if (error != 0) { 725 dsl_dataset_rele(ds, FTAG); 726 return (error); 727 } 728 729 if (will_encrypt && 730 (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) { 731 dsl_dataset_rele(ds, FTAG); 732 return (SET_ERROR(EINVAL)); 733 } 734 } 735 736 /* 737 * Check filesystem and snapshot limits before receiving. We'll 738 * recheck snapshot limits again at the end (we create the 739 * filesystems and increment those counts during begin_sync). 740 */ 741 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, 742 ZFS_PROP_FILESYSTEM_LIMIT, NULL, 743 drba->drba_cred, drba->drba_proc); 744 if (error != 0) { 745 dsl_dataset_rele(ds, FTAG); 746 return (error); 747 } 748 749 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, 750 ZFS_PROP_SNAPSHOT_LIMIT, NULL, 751 drba->drba_cred, drba->drba_proc); 752 if (error != 0) { 753 dsl_dataset_rele(ds, FTAG); 754 return (error); 755 } 756 757 /* can't recv below anything but filesystems (eg. no ZVOLs) */ 758 error = dmu_objset_from_ds(ds, &os); 759 if (error != 0) { 760 dsl_dataset_rele(ds, FTAG); 761 return (error); 762 } 763 if (dmu_objset_type(os) != DMU_OST_ZFS) { 764 dsl_dataset_rele(ds, FTAG); 765 return (SET_ERROR(ZFS_ERR_WRONG_PARENT)); 766 } 767 768 if (drba->drba_origin != NULL) { 769 dsl_dataset_t *origin; 770 error = dsl_dataset_hold_flags(dp, drba->drba_origin, 771 dsflags, FTAG, &origin); 772 if (error != 0) { 773 dsl_dataset_rele(ds, FTAG); 774 return (error); 775 } 776 if (!origin->ds_is_snapshot) { 777 dsl_dataset_rele_flags(origin, dsflags, FTAG); 778 dsl_dataset_rele(ds, FTAG); 779 return (SET_ERROR(EINVAL)); 780 } 781 if (dsl_dataset_phys(origin)->ds_guid != fromguid && 782 fromguid != 0) { 783 dsl_dataset_rele_flags(origin, dsflags, FTAG); 784 dsl_dataset_rele(ds, FTAG); 785 return (SET_ERROR(ENODEV)); 786 } 787 788 if (origin->ds_dir->dd_crypto_obj != 0 && 789 (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) { 790 dsl_dataset_rele_flags(origin, dsflags, FTAG); 791 dsl_dataset_rele(ds, FTAG); 792 return (SET_ERROR(EINVAL)); 793 } 794 795 /* 796 * If the origin is redacted we need to verify that this 797 * send stream can safely be received on top of the 798 * origin. 799 */ 800 if (dsl_dataset_feature_is_active(origin, 801 SPA_FEATURE_REDACTED_DATASETS)) { 802 if (!redact_check(drba, origin)) { 803 dsl_dataset_rele_flags(origin, dsflags, 804 FTAG); 805 dsl_dataset_rele_flags(ds, dsflags, 806 FTAG); 807 return (SET_ERROR(EINVAL)); 808 } 809 } 810 811 error = recv_check_large_blocks(ds, featureflags); 812 if (error != 0) { 813 dsl_dataset_rele_flags(origin, dsflags, FTAG); 814 dsl_dataset_rele_flags(ds, dsflags, FTAG); 815 return (error); 816 } 817 818 dsl_dataset_rele_flags(origin, dsflags, FTAG); 819 } 820 821 dsl_dataset_rele(ds, FTAG); 822 error = 0; 823 } 824 return (error); 825 } 826 827 static void 828 dmu_recv_begin_sync(void *arg, dmu_tx_t *tx) 829 { 830 dmu_recv_begin_arg_t *drba = arg; 831 dsl_pool_t *dp = dmu_tx_pool(tx); 832 objset_t *mos = dp->dp_meta_objset; 833 dmu_recv_cookie_t *drc = drba->drba_cookie; 834 struct drr_begin *drrb = drc->drc_drrb; 835 const char *tofs = drc->drc_tofs; 836 uint64_t featureflags = drc->drc_featureflags; 837 dsl_dataset_t *ds, *newds; 838 objset_t *os; 839 uint64_t dsobj; 840 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 841 int error; 842 uint64_t crflags = 0; 843 dsl_crypto_params_t dummy_dcp = { 0 }; 844 dsl_crypto_params_t *dcp = drba->drba_dcp; 845 846 if (drrb->drr_flags & DRR_FLAG_CI_DATA) 847 crflags |= DS_FLAG_CI_DATASET; 848 849 if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0) 850 dsflags |= DS_HOLD_FLAG_DECRYPT; 851 852 /* 853 * Raw, non-incremental recvs always use a dummy dcp with 854 * the raw cmd set. Raw incremental recvs do not use a dcp 855 * since the encryption parameters are already set in stone. 856 */ 857 if (dcp == NULL && drrb->drr_fromguid == 0 && 858 drba->drba_origin == NULL) { 859 ASSERT3P(dcp, ==, NULL); 860 dcp = &dummy_dcp; 861 862 if (featureflags & DMU_BACKUP_FEATURE_RAW) 863 dcp->cp_cmd = DCP_CMD_RAW_RECV; 864 } 865 866 error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds); 867 if (error == 0) { 868 /* Create temporary clone unless we're doing corrective recv */ 869 dsl_dataset_t *snap = NULL; 870 871 if (drba->drba_cookie->drc_fromsnapobj != 0) { 872 VERIFY0(dsl_dataset_hold_obj(dp, 873 drba->drba_cookie->drc_fromsnapobj, FTAG, &snap)); 874 ASSERT3P(dcp, ==, NULL); 875 } 876 if (drc->drc_heal) { 877 /* When healing we want to use the provided snapshot */ 878 VERIFY0(dsl_dataset_snap_lookup(ds, drc->drc_tosnap, 879 &dsobj)); 880 } else { 881 dsobj = dsl_dataset_create_sync(ds->ds_dir, 882 recv_clone_name, snap, crflags, drba->drba_cred, 883 dcp, tx); 884 } 885 if (drba->drba_cookie->drc_fromsnapobj != 0) 886 dsl_dataset_rele(snap, FTAG); 887 dsl_dataset_rele_flags(ds, dsflags, FTAG); 888 } else { 889 dsl_dir_t *dd; 890 const char *tail; 891 dsl_dataset_t *origin = NULL; 892 893 VERIFY0(dsl_dir_hold(dp, tofs, FTAG, &dd, &tail)); 894 895 if (drba->drba_origin != NULL) { 896 VERIFY0(dsl_dataset_hold(dp, drba->drba_origin, 897 FTAG, &origin)); 898 ASSERT3P(dcp, ==, NULL); 899 } 900 901 /* Create new dataset. */ 902 dsobj = dsl_dataset_create_sync(dd, strrchr(tofs, '/') + 1, 903 origin, crflags, drba->drba_cred, dcp, tx); 904 if (origin != NULL) 905 dsl_dataset_rele(origin, FTAG); 906 dsl_dir_rele(dd, FTAG); 907 drc->drc_newfs = B_TRUE; 908 } 909 VERIFY0(dsl_dataset_own_obj_force(dp, dsobj, dsflags, dmu_recv_tag, 910 &newds)); 911 if (dsl_dataset_feature_is_active(newds, 912 SPA_FEATURE_REDACTED_DATASETS)) { 913 /* 914 * If the origin dataset is redacted, the child will be redacted 915 * when we create it. We clear the new dataset's 916 * redaction info; if it should be redacted, we'll fill 917 * in its information later. 918 */ 919 dsl_dataset_deactivate_feature(newds, 920 SPA_FEATURE_REDACTED_DATASETS, tx); 921 } 922 VERIFY0(dmu_objset_from_ds(newds, &os)); 923 924 if (drc->drc_resumable) { 925 dsl_dataset_zapify(newds, tx); 926 if (drrb->drr_fromguid != 0) { 927 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_FROMGUID, 928 8, 1, &drrb->drr_fromguid, tx)); 929 } 930 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TOGUID, 931 8, 1, &drrb->drr_toguid, tx)); 932 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TONAME, 933 1, strlen(drrb->drr_toname) + 1, drrb->drr_toname, tx)); 934 uint64_t one = 1; 935 uint64_t zero = 0; 936 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OBJECT, 937 8, 1, &one, tx)); 938 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OFFSET, 939 8, 1, &zero, tx)); 940 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_BYTES, 941 8, 1, &zero, tx)); 942 if (featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) { 943 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_LARGEBLOCK, 944 8, 1, &one, tx)); 945 } 946 if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) { 947 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_EMBEDOK, 948 8, 1, &one, tx)); 949 } 950 if (featureflags & DMU_BACKUP_FEATURE_COMPRESSED) { 951 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_COMPRESSOK, 952 8, 1, &one, tx)); 953 } 954 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 955 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_RAWOK, 956 8, 1, &one, tx)); 957 } 958 959 uint64_t *redact_snaps; 960 uint_t numredactsnaps; 961 if (nvlist_lookup_uint64_array(drc->drc_begin_nvl, 962 BEGINNV_REDACT_FROM_SNAPS, &redact_snaps, 963 &numredactsnaps) == 0) { 964 VERIFY0(zap_add(mos, dsobj, 965 DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, 966 sizeof (*redact_snaps), numredactsnaps, 967 redact_snaps, tx)); 968 } 969 } 970 971 /* 972 * Usually the os->os_encrypted value is tied to the presence of a 973 * DSL Crypto Key object in the dd. However, that will not be received 974 * until dmu_recv_stream(), so we set the value manually for now. 975 */ 976 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 977 os->os_encrypted = B_TRUE; 978 drba->drba_cookie->drc_raw = B_TRUE; 979 } 980 981 if (featureflags & DMU_BACKUP_FEATURE_REDACTED) { 982 uint64_t *redact_snaps; 983 uint_t numredactsnaps; 984 VERIFY0(nvlist_lookup_uint64_array(drc->drc_begin_nvl, 985 BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps)); 986 dsl_dataset_activate_redaction(newds, redact_snaps, 987 numredactsnaps, tx); 988 } 989 990 dmu_buf_will_dirty(newds->ds_dbuf, tx); 991 dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT; 992 993 /* 994 * If we actually created a non-clone, we need to create the objset 995 * in our new dataset. If this is a raw send we postpone this until 996 * dmu_recv_stream() so that we can allocate the metadnode with the 997 * properties from the DRR_BEGIN payload. 998 */ 999 rrw_enter(&newds->ds_bp_rwlock, RW_READER, FTAG); 1000 if (BP_IS_HOLE(dsl_dataset_get_blkptr(newds)) && 1001 (featureflags & DMU_BACKUP_FEATURE_RAW) == 0 && 1002 !drc->drc_heal) { 1003 (void) dmu_objset_create_impl(dp->dp_spa, 1004 newds, dsl_dataset_get_blkptr(newds), drrb->drr_type, tx); 1005 } 1006 rrw_exit(&newds->ds_bp_rwlock, FTAG); 1007 1008 drba->drba_cookie->drc_ds = newds; 1009 drba->drba_cookie->drc_os = os; 1010 1011 spa_history_log_internal_ds(newds, "receive", tx, " "); 1012 } 1013 1014 static int 1015 dmu_recv_resume_begin_check(void *arg, dmu_tx_t *tx) 1016 { 1017 dmu_recv_begin_arg_t *drba = arg; 1018 dmu_recv_cookie_t *drc = drba->drba_cookie; 1019 dsl_pool_t *dp = dmu_tx_pool(tx); 1020 struct drr_begin *drrb = drc->drc_drrb; 1021 int error; 1022 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 1023 dsl_dataset_t *ds; 1024 const char *tofs = drc->drc_tofs; 1025 1026 /* already checked */ 1027 ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); 1028 ASSERT(drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING); 1029 1030 if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == 1031 DMU_COMPOUNDSTREAM || 1032 drrb->drr_type >= DMU_OST_NUMTYPES) 1033 return (SET_ERROR(EINVAL)); 1034 1035 /* 1036 * This is mostly a sanity check since we should have already done these 1037 * checks during a previous attempt to receive the data. 1038 */ 1039 error = recv_begin_check_feature_flags_impl(drc->drc_featureflags, 1040 dp->dp_spa); 1041 if (error != 0) 1042 return (error); 1043 1044 /* 6 extra bytes for /%recv */ 1045 char recvname[ZFS_MAX_DATASET_NAME_LEN + 6]; 1046 1047 (void) snprintf(recvname, sizeof (recvname), "%s/%s", 1048 tofs, recv_clone_name); 1049 1050 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) { 1051 /* raw receives require spill block allocation flag */ 1052 if (!(drrb->drr_flags & DRR_FLAG_SPILL_BLOCK)) 1053 return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING)); 1054 } else { 1055 dsflags |= DS_HOLD_FLAG_DECRYPT; 1056 } 1057 1058 boolean_t recvexist = B_TRUE; 1059 if (dsl_dataset_hold_flags(dp, recvname, dsflags, FTAG, &ds) != 0) { 1060 /* %recv does not exist; continue in tofs */ 1061 recvexist = B_FALSE; 1062 error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds); 1063 if (error != 0) 1064 return (error); 1065 } 1066 1067 /* 1068 * Resume of full/newfs recv on existing dataset should be done with 1069 * force flag 1070 */ 1071 if (recvexist && drrb->drr_fromguid == 0 && !drc->drc_force) { 1072 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1073 return (SET_ERROR(ZFS_ERR_RESUME_EXISTS)); 1074 } 1075 1076 /* check that ds is marked inconsistent */ 1077 if (!DS_IS_INCONSISTENT(ds)) { 1078 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1079 return (SET_ERROR(EINVAL)); 1080 } 1081 1082 /* check that there is resuming data, and that the toguid matches */ 1083 if (!dsl_dataset_is_zapified(ds)) { 1084 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1085 return (SET_ERROR(EINVAL)); 1086 } 1087 uint64_t val; 1088 error = zap_lookup(dp->dp_meta_objset, ds->ds_object, 1089 DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val); 1090 if (error != 0 || drrb->drr_toguid != val) { 1091 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1092 return (SET_ERROR(EINVAL)); 1093 } 1094 1095 /* 1096 * Check if the receive is still running. If so, it will be owned. 1097 * Note that nothing else can own the dataset (e.g. after the receive 1098 * fails) because it will be marked inconsistent. 1099 */ 1100 if (dsl_dataset_has_owner(ds)) { 1101 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1102 return (SET_ERROR(EBUSY)); 1103 } 1104 1105 /* There should not be any snapshots of this fs yet. */ 1106 if (ds->ds_prev != NULL && ds->ds_prev->ds_dir == ds->ds_dir) { 1107 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1108 return (SET_ERROR(EINVAL)); 1109 } 1110 1111 /* 1112 * Note: resume point will be checked when we process the first WRITE 1113 * record. 1114 */ 1115 1116 /* check that the origin matches */ 1117 val = 0; 1118 (void) zap_lookup(dp->dp_meta_objset, ds->ds_object, 1119 DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val); 1120 if (drrb->drr_fromguid != val) { 1121 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1122 return (SET_ERROR(EINVAL)); 1123 } 1124 1125 if (ds->ds_prev != NULL && drrb->drr_fromguid != 0) 1126 drc->drc_fromsnapobj = ds->ds_prev->ds_object; 1127 1128 /* 1129 * If we're resuming, and the send is redacted, then the original send 1130 * must have been redacted, and must have been redacted with respect to 1131 * the same snapshots. 1132 */ 1133 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_REDACTED) { 1134 uint64_t num_ds_redact_snaps; 1135 uint64_t *ds_redact_snaps; 1136 1137 uint_t num_stream_redact_snaps; 1138 uint64_t *stream_redact_snaps; 1139 1140 if (nvlist_lookup_uint64_array(drc->drc_begin_nvl, 1141 BEGINNV_REDACT_SNAPS, &stream_redact_snaps, 1142 &num_stream_redact_snaps) != 0) { 1143 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1144 return (SET_ERROR(EINVAL)); 1145 } 1146 1147 if (!dsl_dataset_get_uint64_array_feature(ds, 1148 SPA_FEATURE_REDACTED_DATASETS, &num_ds_redact_snaps, 1149 &ds_redact_snaps)) { 1150 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1151 return (SET_ERROR(EINVAL)); 1152 } 1153 1154 for (int i = 0; i < num_ds_redact_snaps; i++) { 1155 if (!redact_snaps_contains(ds_redact_snaps, 1156 num_ds_redact_snaps, stream_redact_snaps[i])) { 1157 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1158 return (SET_ERROR(EINVAL)); 1159 } 1160 } 1161 } 1162 1163 error = recv_check_large_blocks(ds, drc->drc_featureflags); 1164 if (error != 0) { 1165 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1166 return (error); 1167 } 1168 1169 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1170 return (0); 1171 } 1172 1173 static void 1174 dmu_recv_resume_begin_sync(void *arg, dmu_tx_t *tx) 1175 { 1176 dmu_recv_begin_arg_t *drba = arg; 1177 dsl_pool_t *dp = dmu_tx_pool(tx); 1178 const char *tofs = drba->drba_cookie->drc_tofs; 1179 uint64_t featureflags = drba->drba_cookie->drc_featureflags; 1180 dsl_dataset_t *ds; 1181 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 1182 /* 6 extra bytes for /%recv */ 1183 char recvname[ZFS_MAX_DATASET_NAME_LEN + 6]; 1184 1185 (void) snprintf(recvname, sizeof (recvname), "%s/%s", tofs, 1186 recv_clone_name); 1187 1188 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 1189 drba->drba_cookie->drc_raw = B_TRUE; 1190 } else { 1191 dsflags |= DS_HOLD_FLAG_DECRYPT; 1192 } 1193 1194 if (dsl_dataset_own_force(dp, recvname, dsflags, dmu_recv_tag, &ds) 1195 != 0) { 1196 /* %recv does not exist; continue in tofs */ 1197 VERIFY0(dsl_dataset_own_force(dp, tofs, dsflags, dmu_recv_tag, 1198 &ds)); 1199 drba->drba_cookie->drc_newfs = B_TRUE; 1200 } 1201 1202 ASSERT(DS_IS_INCONSISTENT(ds)); 1203 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 1204 ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)) || 1205 drba->drba_cookie->drc_raw); 1206 rrw_exit(&ds->ds_bp_rwlock, FTAG); 1207 1208 drba->drba_cookie->drc_ds = ds; 1209 VERIFY0(dmu_objset_from_ds(ds, &drba->drba_cookie->drc_os)); 1210 drba->drba_cookie->drc_should_save = B_TRUE; 1211 1212 spa_history_log_internal_ds(ds, "resume receive", tx, " "); 1213 } 1214 1215 /* 1216 * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin() 1217 * succeeds; otherwise we will leak the holds on the datasets. 1218 */ 1219 int 1220 dmu_recv_begin(const char *tofs, const char *tosnap, 1221 dmu_replay_record_t *drr_begin, boolean_t force, boolean_t heal, 1222 boolean_t resumable, nvlist_t *localprops, nvlist_t *hidden_args, 1223 const char *origin, dmu_recv_cookie_t *drc, zfs_file_t *fp, 1224 offset_t *voffp) 1225 { 1226 dmu_recv_begin_arg_t drba = { 0 }; 1227 int err = 0; 1228 1229 memset(drc, 0, sizeof (dmu_recv_cookie_t)); 1230 drc->drc_drr_begin = drr_begin; 1231 drc->drc_drrb = &drr_begin->drr_u.drr_begin; 1232 drc->drc_tosnap = tosnap; 1233 drc->drc_tofs = tofs; 1234 drc->drc_force = force; 1235 drc->drc_heal = heal; 1236 drc->drc_resumable = resumable; 1237 drc->drc_cred = CRED(); 1238 drc->drc_proc = curproc; 1239 drc->drc_clone = (origin != NULL); 1240 1241 if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) { 1242 drc->drc_byteswap = B_TRUE; 1243 (void) fletcher_4_incremental_byteswap(drr_begin, 1244 sizeof (dmu_replay_record_t), &drc->drc_cksum); 1245 byteswap_record(drr_begin); 1246 } else if (drc->drc_drrb->drr_magic == DMU_BACKUP_MAGIC) { 1247 (void) fletcher_4_incremental_native(drr_begin, 1248 sizeof (dmu_replay_record_t), &drc->drc_cksum); 1249 } else { 1250 return (SET_ERROR(EINVAL)); 1251 } 1252 1253 drc->drc_fp = fp; 1254 drc->drc_voff = *voffp; 1255 drc->drc_featureflags = 1256 DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo); 1257 1258 uint32_t payloadlen = drc->drc_drr_begin->drr_payloadlen; 1259 1260 /* 1261 * Since OpenZFS 2.0.0, we have enforced a 64MB limit in userspace 1262 * configurable via ZFS_SENDRECV_MAX_NVLIST. We enforce 256MB as a hard 1263 * upper limit. Systems with less than 1GB of RAM will see a lower 1264 * limit from `arc_all_memory() / 4`. 1265 */ 1266 if (payloadlen > (MIN((1U << 28), arc_all_memory() / 4))) 1267 return (E2BIG); 1268 1269 1270 if (payloadlen != 0) { 1271 void *payload = vmem_alloc(payloadlen, KM_SLEEP); 1272 /* 1273 * For compatibility with recursive send streams, we don't do 1274 * this here if the stream could be part of a package. Instead, 1275 * we'll do it in dmu_recv_stream. If we pull the next header 1276 * too early, and it's the END record, we break the `recv_skip` 1277 * logic. 1278 */ 1279 1280 err = receive_read_payload_and_next_header(drc, payloadlen, 1281 payload); 1282 if (err != 0) { 1283 vmem_free(payload, payloadlen); 1284 return (err); 1285 } 1286 err = nvlist_unpack(payload, payloadlen, &drc->drc_begin_nvl, 1287 KM_SLEEP); 1288 vmem_free(payload, payloadlen); 1289 if (err != 0) { 1290 kmem_free(drc->drc_next_rrd, 1291 sizeof (*drc->drc_next_rrd)); 1292 return (err); 1293 } 1294 } 1295 1296 if (drc->drc_drrb->drr_flags & DRR_FLAG_SPILL_BLOCK) 1297 drc->drc_spill = B_TRUE; 1298 1299 drba.drba_origin = origin; 1300 drba.drba_cookie = drc; 1301 drba.drba_cred = CRED(); 1302 drba.drba_proc = curproc; 1303 1304 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) { 1305 err = dsl_sync_task(tofs, 1306 dmu_recv_resume_begin_check, dmu_recv_resume_begin_sync, 1307 &drba, 5, ZFS_SPACE_CHECK_NORMAL); 1308 } else { 1309 /* 1310 * For non-raw, non-incremental, non-resuming receives the 1311 * user can specify encryption parameters on the command line 1312 * with "zfs recv -o". For these receives we create a dcp and 1313 * pass it to the sync task. Creating the dcp will implicitly 1314 * remove the encryption params from the localprops nvlist, 1315 * which avoids errors when trying to set these normally 1316 * read-only properties. Any other kind of receive that 1317 * attempts to set these properties will fail as a result. 1318 */ 1319 if ((DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo) & 1320 DMU_BACKUP_FEATURE_RAW) == 0 && 1321 origin == NULL && drc->drc_drrb->drr_fromguid == 0) { 1322 err = dsl_crypto_params_create_nvlist(DCP_CMD_NONE, 1323 localprops, hidden_args, &drba.drba_dcp); 1324 } 1325 1326 if (err == 0) { 1327 err = dsl_sync_task(tofs, 1328 dmu_recv_begin_check, dmu_recv_begin_sync, 1329 &drba, 5, ZFS_SPACE_CHECK_NORMAL); 1330 dsl_crypto_params_free(drba.drba_dcp, !!err); 1331 } 1332 } 1333 1334 if (err != 0) { 1335 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 1336 nvlist_free(drc->drc_begin_nvl); 1337 } 1338 return (err); 1339 } 1340 1341 /* 1342 * Holds data need for corrective recv callback 1343 */ 1344 typedef struct cr_cb_data { 1345 uint64_t size; 1346 zbookmark_phys_t zb; 1347 spa_t *spa; 1348 } cr_cb_data_t; 1349 1350 static void 1351 corrective_read_done(zio_t *zio) 1352 { 1353 cr_cb_data_t *data = zio->io_private; 1354 /* Corruption corrected; update error log if needed */ 1355 if (zio->io_error == 0) 1356 spa_remove_error(data->spa, &data->zb, &zio->io_bp->blk_birth); 1357 kmem_free(data, sizeof (cr_cb_data_t)); 1358 abd_free(zio->io_abd); 1359 } 1360 1361 /* 1362 * zio_rewrite the data pointed to by bp with the data from the rrd's abd. 1363 */ 1364 static int 1365 do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw, 1366 struct receive_record_arg *rrd, blkptr_t *bp) 1367 { 1368 int err; 1369 zio_t *io; 1370 zbookmark_phys_t zb; 1371 dnode_t *dn; 1372 abd_t *abd = rrd->abd; 1373 zio_cksum_t bp_cksum = bp->blk_cksum; 1374 zio_flag_t flags = ZIO_FLAG_SPECULATIVE | 1375 ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_CANFAIL; 1376 1377 if (rwa->raw) 1378 flags |= ZIO_FLAG_RAW; 1379 1380 err = dnode_hold(rwa->os, drrw->drr_object, FTAG, &dn); 1381 if (err != 0) 1382 return (err); 1383 SET_BOOKMARK(&zb, dmu_objset_id(rwa->os), drrw->drr_object, 0, 1384 dbuf_whichblock(dn, 0, drrw->drr_offset)); 1385 dnode_rele(dn, FTAG); 1386 1387 if (!rwa->raw && DRR_WRITE_COMPRESSED(drrw)) { 1388 /* Decompress the stream data */ 1389 abd_t *dabd = abd_alloc_linear( 1390 drrw->drr_logical_size, B_FALSE); 1391 err = zio_decompress_data(drrw->drr_compressiontype, 1392 abd, abd_to_buf(dabd), abd_get_size(abd), 1393 abd_get_size(dabd), NULL); 1394 1395 if (err != 0) { 1396 abd_free(dabd); 1397 return (err); 1398 } 1399 /* Swap in the newly decompressed data into the abd */ 1400 abd_free(abd); 1401 abd = dabd; 1402 } 1403 1404 if (!rwa->raw && BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 1405 /* Recompress the data */ 1406 abd_t *cabd = abd_alloc_linear(BP_GET_PSIZE(bp), 1407 B_FALSE); 1408 void *buf = abd_to_buf(cabd); 1409 uint64_t csize = zio_compress_data(BP_GET_COMPRESS(bp), 1410 abd, &buf, abd_get_size(abd), 1411 rwa->os->os_complevel); 1412 abd_zero_off(cabd, csize, BP_GET_PSIZE(bp) - csize); 1413 /* Swap in newly compressed data into the abd */ 1414 abd_free(abd); 1415 abd = cabd; 1416 flags |= ZIO_FLAG_RAW_COMPRESS; 1417 } 1418 1419 /* 1420 * The stream is not encrypted but the data on-disk is. 1421 * We need to re-encrypt the buf using the same 1422 * encryption type, salt, iv, and mac that was used to encrypt 1423 * the block previosly. 1424 */ 1425 if (!rwa->raw && BP_USES_CRYPT(bp)) { 1426 dsl_dataset_t *ds; 1427 dsl_crypto_key_t *dck = NULL; 1428 uint8_t salt[ZIO_DATA_SALT_LEN]; 1429 uint8_t iv[ZIO_DATA_IV_LEN]; 1430 uint8_t mac[ZIO_DATA_MAC_LEN]; 1431 boolean_t no_crypt = B_FALSE; 1432 dsl_pool_t *dp = dmu_objset_pool(rwa->os); 1433 abd_t *eabd = abd_alloc_linear(BP_GET_PSIZE(bp), B_FALSE); 1434 1435 zio_crypt_decode_params_bp(bp, salt, iv); 1436 zio_crypt_decode_mac_bp(bp, mac); 1437 1438 dsl_pool_config_enter(dp, FTAG); 1439 err = dsl_dataset_hold_flags(dp, rwa->tofs, 1440 DS_HOLD_FLAG_DECRYPT, FTAG, &ds); 1441 if (err != 0) { 1442 dsl_pool_config_exit(dp, FTAG); 1443 abd_free(eabd); 1444 return (SET_ERROR(EACCES)); 1445 } 1446 1447 /* Look up the key from the spa's keystore */ 1448 err = spa_keystore_lookup_key(rwa->os->os_spa, 1449 zb.zb_objset, FTAG, &dck); 1450 if (err != 0) { 1451 dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, 1452 FTAG); 1453 dsl_pool_config_exit(dp, FTAG); 1454 abd_free(eabd); 1455 return (SET_ERROR(EACCES)); 1456 } 1457 1458 err = zio_do_crypt_abd(B_TRUE, &dck->dck_key, 1459 BP_GET_TYPE(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, 1460 mac, abd_get_size(abd), abd, eabd, &no_crypt); 1461 1462 spa_keystore_dsl_key_rele(rwa->os->os_spa, dck, FTAG); 1463 dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, FTAG); 1464 dsl_pool_config_exit(dp, FTAG); 1465 1466 ASSERT0(no_crypt); 1467 if (err != 0) { 1468 abd_free(eabd); 1469 return (err); 1470 } 1471 /* Swap in the newly encrypted data into the abd */ 1472 abd_free(abd); 1473 abd = eabd; 1474 1475 /* 1476 * We want to prevent zio_rewrite() from trying to 1477 * encrypt the data again 1478 */ 1479 flags |= ZIO_FLAG_RAW_ENCRYPT; 1480 } 1481 rrd->abd = abd; 1482 1483 io = zio_rewrite(NULL, rwa->os->os_spa, bp->blk_birth, bp, abd, 1484 BP_GET_PSIZE(bp), NULL, NULL, ZIO_PRIORITY_SYNC_WRITE, flags, &zb); 1485 1486 ASSERT(abd_get_size(abd) == BP_GET_LSIZE(bp) || 1487 abd_get_size(abd) == BP_GET_PSIZE(bp)); 1488 1489 /* compute new bp checksum value and make sure it matches the old one */ 1490 zio_checksum_compute(io, BP_GET_CHECKSUM(bp), abd, abd_get_size(abd)); 1491 if (!ZIO_CHECKSUM_EQUAL(bp_cksum, io->io_bp->blk_cksum)) { 1492 zio_destroy(io); 1493 if (zfs_recv_best_effort_corrective != 0) 1494 return (0); 1495 return (SET_ERROR(ECKSUM)); 1496 } 1497 1498 /* Correct the corruption in place */ 1499 err = zio_wait(io); 1500 if (err == 0) { 1501 cr_cb_data_t *cb_data = 1502 kmem_alloc(sizeof (cr_cb_data_t), KM_SLEEP); 1503 cb_data->spa = rwa->os->os_spa; 1504 cb_data->size = drrw->drr_logical_size; 1505 cb_data->zb = zb; 1506 /* Test if healing worked by re-reading the bp */ 1507 err = zio_wait(zio_read(rwa->heal_pio, rwa->os->os_spa, bp, 1508 abd_alloc_for_io(drrw->drr_logical_size, B_FALSE), 1509 drrw->drr_logical_size, corrective_read_done, 1510 cb_data, ZIO_PRIORITY_ASYNC_READ, flags, NULL)); 1511 } 1512 if (err != 0 && zfs_recv_best_effort_corrective != 0) 1513 err = 0; 1514 1515 return (err); 1516 } 1517 1518 static int 1519 receive_read(dmu_recv_cookie_t *drc, int len, void *buf) 1520 { 1521 int done = 0; 1522 1523 /* 1524 * The code doesn't rely on this (lengths being multiples of 8). See 1525 * comment in dump_bytes. 1526 */ 1527 ASSERT(len % 8 == 0 || 1528 (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) != 0); 1529 1530 while (done < len) { 1531 ssize_t resid = len - done; 1532 zfs_file_t *fp = drc->drc_fp; 1533 int err = zfs_file_read(fp, (char *)buf + done, 1534 len - done, &resid); 1535 if (err == 0 && resid == len - done) { 1536 /* 1537 * Note: ECKSUM or ZFS_ERR_STREAM_TRUNCATED indicates 1538 * that the receive was interrupted and can 1539 * potentially be resumed. 1540 */ 1541 err = SET_ERROR(ZFS_ERR_STREAM_TRUNCATED); 1542 } 1543 drc->drc_voff += len - done - resid; 1544 done = len - resid; 1545 if (err != 0) 1546 return (err); 1547 } 1548 1549 drc->drc_bytes_read += len; 1550 1551 ASSERT3U(done, ==, len); 1552 return (0); 1553 } 1554 1555 static inline uint8_t 1556 deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size) 1557 { 1558 if (bonus_type == DMU_OT_SA) { 1559 return (1); 1560 } else { 1561 return (1 + 1562 ((DN_OLD_MAX_BONUSLEN - 1563 MIN(DN_OLD_MAX_BONUSLEN, bonus_size)) >> SPA_BLKPTRSHIFT)); 1564 } 1565 } 1566 1567 static void 1568 save_resume_state(struct receive_writer_arg *rwa, 1569 uint64_t object, uint64_t offset, dmu_tx_t *tx) 1570 { 1571 int txgoff = dmu_tx_get_txg(tx) & TXG_MASK; 1572 1573 if (!rwa->resumable) 1574 return; 1575 1576 /* 1577 * We use ds_resume_bytes[] != 0 to indicate that we need to 1578 * update this on disk, so it must not be 0. 1579 */ 1580 ASSERT(rwa->bytes_read != 0); 1581 1582 /* 1583 * We only resume from write records, which have a valid 1584 * (non-meta-dnode) object number. 1585 */ 1586 ASSERT(object != 0); 1587 1588 /* 1589 * For resuming to work correctly, we must receive records in order, 1590 * sorted by object,offset. This is checked by the callers, but 1591 * assert it here for good measure. 1592 */ 1593 ASSERT3U(object, >=, rwa->os->os_dsl_dataset->ds_resume_object[txgoff]); 1594 ASSERT(object != rwa->os->os_dsl_dataset->ds_resume_object[txgoff] || 1595 offset >= rwa->os->os_dsl_dataset->ds_resume_offset[txgoff]); 1596 ASSERT3U(rwa->bytes_read, >=, 1597 rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff]); 1598 1599 rwa->os->os_dsl_dataset->ds_resume_object[txgoff] = object; 1600 rwa->os->os_dsl_dataset->ds_resume_offset[txgoff] = offset; 1601 rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff] = rwa->bytes_read; 1602 } 1603 1604 static int 1605 receive_object_is_same_generation(objset_t *os, uint64_t object, 1606 dmu_object_type_t old_bonus_type, dmu_object_type_t new_bonus_type, 1607 const void *new_bonus, boolean_t *samegenp) 1608 { 1609 zfs_file_info_t zoi; 1610 int err; 1611 1612 dmu_buf_t *old_bonus_dbuf; 1613 err = dmu_bonus_hold(os, object, FTAG, &old_bonus_dbuf); 1614 if (err != 0) 1615 return (err); 1616 err = dmu_get_file_info(os, old_bonus_type, old_bonus_dbuf->db_data, 1617 &zoi); 1618 dmu_buf_rele(old_bonus_dbuf, FTAG); 1619 if (err != 0) 1620 return (err); 1621 uint64_t old_gen = zoi.zfi_generation; 1622 1623 err = dmu_get_file_info(os, new_bonus_type, new_bonus, &zoi); 1624 if (err != 0) 1625 return (err); 1626 uint64_t new_gen = zoi.zfi_generation; 1627 1628 *samegenp = (old_gen == new_gen); 1629 return (0); 1630 } 1631 1632 static int 1633 receive_handle_existing_object(const struct receive_writer_arg *rwa, 1634 const struct drr_object *drro, const dmu_object_info_t *doi, 1635 const void *bonus_data, 1636 uint64_t *object_to_hold, uint32_t *new_blksz) 1637 { 1638 uint32_t indblksz = drro->drr_indblkshift ? 1639 1ULL << drro->drr_indblkshift : 0; 1640 int nblkptr = deduce_nblkptr(drro->drr_bonustype, 1641 drro->drr_bonuslen); 1642 uint8_t dn_slots = drro->drr_dn_slots != 0 ? 1643 drro->drr_dn_slots : DNODE_MIN_SLOTS; 1644 boolean_t do_free_range = B_FALSE; 1645 int err; 1646 1647 *object_to_hold = drro->drr_object; 1648 1649 /* nblkptr should be bounded by the bonus size and type */ 1650 if (rwa->raw && nblkptr != drro->drr_nblkptr) 1651 return (SET_ERROR(EINVAL)); 1652 1653 /* 1654 * After the previous send stream, the sending system may 1655 * have freed this object, and then happened to re-allocate 1656 * this object number in a later txg. In this case, we are 1657 * receiving a different logical file, and the block size may 1658 * appear to be different. i.e. we may have a different 1659 * block size for this object than what the send stream says. 1660 * In this case we need to remove the object's contents, 1661 * so that its structure can be changed and then its contents 1662 * entirely replaced by subsequent WRITE records. 1663 * 1664 * If this is a -L (--large-block) incremental stream, and 1665 * the previous stream was not -L, the block size may appear 1666 * to increase. i.e. we may have a smaller block size for 1667 * this object than what the send stream says. In this case 1668 * we need to keep the object's contents and block size 1669 * intact, so that we don't lose parts of the object's 1670 * contents that are not changed by this incremental send 1671 * stream. 1672 * 1673 * We can distinguish between the two above cases by using 1674 * the ZPL's generation number (see 1675 * receive_object_is_same_generation()). However, we only 1676 * want to rely on the generation number when absolutely 1677 * necessary, because with raw receives, the generation is 1678 * encrypted. We also want to minimize dependence on the 1679 * ZPL, so that other types of datasets can also be received 1680 * (e.g. ZVOLs, although note that ZVOLS currently do not 1681 * reallocate their objects or change their structure). 1682 * Therefore, we check a number of different cases where we 1683 * know it is safe to discard the object's contents, before 1684 * using the ZPL's generation number to make the above 1685 * distinction. 1686 */ 1687 if (drro->drr_blksz != doi->doi_data_block_size) { 1688 if (rwa->raw) { 1689 /* 1690 * RAW streams always have large blocks, so 1691 * we are sure that the data is not needed 1692 * due to changing --large-block to be on. 1693 * Which is fortunate since the bonus buffer 1694 * (which contains the ZPL generation) is 1695 * encrypted, and the key might not be 1696 * loaded. 1697 */ 1698 do_free_range = B_TRUE; 1699 } else if (rwa->full) { 1700 /* 1701 * This is a full send stream, so it always 1702 * replaces what we have. Even if the 1703 * generation numbers happen to match, this 1704 * can not actually be the same logical file. 1705 * This is relevant when receiving a full 1706 * send as a clone. 1707 */ 1708 do_free_range = B_TRUE; 1709 } else if (drro->drr_type != 1710 DMU_OT_PLAIN_FILE_CONTENTS || 1711 doi->doi_type != DMU_OT_PLAIN_FILE_CONTENTS) { 1712 /* 1713 * PLAIN_FILE_CONTENTS are the only type of 1714 * objects that have ever been stored with 1715 * large blocks, so we don't need the special 1716 * logic below. ZAP blocks can shrink (when 1717 * there's only one block), so we don't want 1718 * to hit the error below about block size 1719 * only increasing. 1720 */ 1721 do_free_range = B_TRUE; 1722 } else if (doi->doi_max_offset <= 1723 doi->doi_data_block_size) { 1724 /* 1725 * There is only one block. We can free it, 1726 * because its contents will be replaced by a 1727 * WRITE record. This can not be the no-L -> 1728 * -L case, because the no-L case would have 1729 * resulted in multiple blocks. If we 1730 * supported -L -> no-L, it would not be safe 1731 * to free the file's contents. Fortunately, 1732 * that is not allowed (see 1733 * recv_check_large_blocks()). 1734 */ 1735 do_free_range = B_TRUE; 1736 } else { 1737 boolean_t is_same_gen; 1738 err = receive_object_is_same_generation(rwa->os, 1739 drro->drr_object, doi->doi_bonus_type, 1740 drro->drr_bonustype, bonus_data, &is_same_gen); 1741 if (err != 0) 1742 return (SET_ERROR(EINVAL)); 1743 1744 if (is_same_gen) { 1745 /* 1746 * This is the same logical file, and 1747 * the block size must be increasing. 1748 * It could only decrease if 1749 * --large-block was changed to be 1750 * off, which is checked in 1751 * recv_check_large_blocks(). 1752 */ 1753 if (drro->drr_blksz <= 1754 doi->doi_data_block_size) 1755 return (SET_ERROR(EINVAL)); 1756 /* 1757 * We keep the existing blocksize and 1758 * contents. 1759 */ 1760 *new_blksz = 1761 doi->doi_data_block_size; 1762 } else { 1763 do_free_range = B_TRUE; 1764 } 1765 } 1766 } 1767 1768 /* nblkptr can only decrease if the object was reallocated */ 1769 if (nblkptr < doi->doi_nblkptr) 1770 do_free_range = B_TRUE; 1771 1772 /* number of slots can only change on reallocation */ 1773 if (dn_slots != doi->doi_dnodesize >> DNODE_SHIFT) 1774 do_free_range = B_TRUE; 1775 1776 /* 1777 * For raw sends we also check a few other fields to 1778 * ensure we are preserving the objset structure exactly 1779 * as it was on the receive side: 1780 * - A changed indirect block size 1781 * - A smaller nlevels 1782 */ 1783 if (rwa->raw) { 1784 if (indblksz != doi->doi_metadata_block_size) 1785 do_free_range = B_TRUE; 1786 if (drro->drr_nlevels < doi->doi_indirection) 1787 do_free_range = B_TRUE; 1788 } 1789 1790 if (do_free_range) { 1791 err = dmu_free_long_range(rwa->os, drro->drr_object, 1792 0, DMU_OBJECT_END); 1793 if (err != 0) 1794 return (SET_ERROR(EINVAL)); 1795 } 1796 1797 /* 1798 * The dmu does not currently support decreasing nlevels 1799 * or changing the number of dnode slots on an object. For 1800 * non-raw sends, this does not matter and the new object 1801 * can just use the previous one's nlevels. For raw sends, 1802 * however, the structure of the received dnode (including 1803 * nlevels and dnode slots) must match that of the send 1804 * side. Therefore, instead of using dmu_object_reclaim(), 1805 * we must free the object completely and call 1806 * dmu_object_claim_dnsize() instead. 1807 */ 1808 if ((rwa->raw && drro->drr_nlevels < doi->doi_indirection) || 1809 dn_slots != doi->doi_dnodesize >> DNODE_SHIFT) { 1810 err = dmu_free_long_object(rwa->os, drro->drr_object); 1811 if (err != 0) 1812 return (SET_ERROR(EINVAL)); 1813 1814 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1815 *object_to_hold = DMU_NEW_OBJECT; 1816 } 1817 1818 /* 1819 * For raw receives, free everything beyond the new incoming 1820 * maxblkid. Normally this would be done with a DRR_FREE 1821 * record that would come after this DRR_OBJECT record is 1822 * processed. However, for raw receives we manually set the 1823 * maxblkid from the drr_maxblkid and so we must first free 1824 * everything above that blkid to ensure the DMU is always 1825 * consistent with itself. We will never free the first block 1826 * of the object here because a maxblkid of 0 could indicate 1827 * an object with a single block or one with no blocks. This 1828 * free may be skipped when dmu_free_long_range() was called 1829 * above since it covers the entire object's contents. 1830 */ 1831 if (rwa->raw && *object_to_hold != DMU_NEW_OBJECT && !do_free_range) { 1832 err = dmu_free_long_range(rwa->os, drro->drr_object, 1833 (drro->drr_maxblkid + 1) * doi->doi_data_block_size, 1834 DMU_OBJECT_END); 1835 if (err != 0) 1836 return (SET_ERROR(EINVAL)); 1837 } 1838 return (0); 1839 } 1840 1841 noinline static int 1842 receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, 1843 void *data) 1844 { 1845 dmu_object_info_t doi; 1846 dmu_tx_t *tx; 1847 int err; 1848 uint32_t new_blksz = drro->drr_blksz; 1849 uint8_t dn_slots = drro->drr_dn_slots != 0 ? 1850 drro->drr_dn_slots : DNODE_MIN_SLOTS; 1851 1852 if (drro->drr_type == DMU_OT_NONE || 1853 !DMU_OT_IS_VALID(drro->drr_type) || 1854 !DMU_OT_IS_VALID(drro->drr_bonustype) || 1855 drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS || 1856 drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS || 1857 P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) || 1858 drro->drr_blksz < SPA_MINBLOCKSIZE || 1859 drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(rwa->os)) || 1860 drro->drr_bonuslen > 1861 DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(rwa->os))) || 1862 dn_slots > 1863 (spa_maxdnodesize(dmu_objset_spa(rwa->os)) >> DNODE_SHIFT)) { 1864 return (SET_ERROR(EINVAL)); 1865 } 1866 1867 if (rwa->raw) { 1868 /* 1869 * We should have received a DRR_OBJECT_RANGE record 1870 * containing this block and stored it in rwa. 1871 */ 1872 if (drro->drr_object < rwa->or_firstobj || 1873 drro->drr_object >= rwa->or_firstobj + rwa->or_numslots || 1874 drro->drr_raw_bonuslen < drro->drr_bonuslen || 1875 drro->drr_indblkshift > SPA_MAXBLOCKSHIFT || 1876 drro->drr_nlevels > DN_MAX_LEVELS || 1877 drro->drr_nblkptr > DN_MAX_NBLKPTR || 1878 DN_SLOTS_TO_BONUSLEN(dn_slots) < 1879 drro->drr_raw_bonuslen) 1880 return (SET_ERROR(EINVAL)); 1881 } else { 1882 /* 1883 * The DRR_OBJECT_SPILL flag is valid when the DRR_BEGIN 1884 * record indicates this by setting DRR_FLAG_SPILL_BLOCK. 1885 */ 1886 if (((drro->drr_flags & ~(DRR_OBJECT_SPILL))) || 1887 (!rwa->spill && DRR_OBJECT_HAS_SPILL(drro->drr_flags))) { 1888 return (SET_ERROR(EINVAL)); 1889 } 1890 1891 if (drro->drr_raw_bonuslen != 0 || drro->drr_nblkptr != 0 || 1892 drro->drr_indblkshift != 0 || drro->drr_nlevels != 0) { 1893 return (SET_ERROR(EINVAL)); 1894 } 1895 } 1896 1897 err = dmu_object_info(rwa->os, drro->drr_object, &doi); 1898 1899 if (err != 0 && err != ENOENT && err != EEXIST) 1900 return (SET_ERROR(EINVAL)); 1901 1902 if (drro->drr_object > rwa->max_object) 1903 rwa->max_object = drro->drr_object; 1904 1905 /* 1906 * If we are losing blkptrs or changing the block size this must 1907 * be a new file instance. We must clear out the previous file 1908 * contents before we can change this type of metadata in the dnode. 1909 * Raw receives will also check that the indirect structure of the 1910 * dnode hasn't changed. 1911 */ 1912 uint64_t object_to_hold; 1913 if (err == 0) { 1914 err = receive_handle_existing_object(rwa, drro, &doi, data, 1915 &object_to_hold, &new_blksz); 1916 if (err != 0) 1917 return (err); 1918 } else if (err == EEXIST) { 1919 /* 1920 * The object requested is currently an interior slot of a 1921 * multi-slot dnode. This will be resolved when the next txg 1922 * is synced out, since the send stream will have told us 1923 * to free this slot when we freed the associated dnode 1924 * earlier in the stream. 1925 */ 1926 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1927 1928 if (dmu_object_info(rwa->os, drro->drr_object, NULL) != ENOENT) 1929 return (SET_ERROR(EINVAL)); 1930 1931 /* object was freed and we are about to allocate a new one */ 1932 object_to_hold = DMU_NEW_OBJECT; 1933 } else { 1934 /* 1935 * If the only record in this range so far was DRR_FREEOBJECTS 1936 * with at least one actually freed object, it's possible that 1937 * the block will now be converted to a hole. We need to wait 1938 * for the txg to sync to prevent races. 1939 */ 1940 if (rwa->or_need_sync == ORNS_YES) 1941 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1942 1943 /* object is free and we are about to allocate a new one */ 1944 object_to_hold = DMU_NEW_OBJECT; 1945 } 1946 1947 /* Only relevant for the first object in the range */ 1948 rwa->or_need_sync = ORNS_NO; 1949 1950 /* 1951 * If this is a multi-slot dnode there is a chance that this 1952 * object will expand into a slot that is already used by 1953 * another object from the previous snapshot. We must free 1954 * these objects before we attempt to allocate the new dnode. 1955 */ 1956 if (dn_slots > 1) { 1957 boolean_t need_sync = B_FALSE; 1958 1959 for (uint64_t slot = drro->drr_object + 1; 1960 slot < drro->drr_object + dn_slots; 1961 slot++) { 1962 dmu_object_info_t slot_doi; 1963 1964 err = dmu_object_info(rwa->os, slot, &slot_doi); 1965 if (err == ENOENT || err == EEXIST) 1966 continue; 1967 else if (err != 0) 1968 return (err); 1969 1970 err = dmu_free_long_object(rwa->os, slot); 1971 if (err != 0) 1972 return (err); 1973 1974 need_sync = B_TRUE; 1975 } 1976 1977 if (need_sync) 1978 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1979 } 1980 1981 tx = dmu_tx_create(rwa->os); 1982 dmu_tx_hold_bonus(tx, object_to_hold); 1983 dmu_tx_hold_write(tx, object_to_hold, 0, 0); 1984 err = dmu_tx_assign(tx, TXG_WAIT); 1985 if (err != 0) { 1986 dmu_tx_abort(tx); 1987 return (err); 1988 } 1989 1990 if (object_to_hold == DMU_NEW_OBJECT) { 1991 /* Currently free, wants to be allocated */ 1992 err = dmu_object_claim_dnsize(rwa->os, drro->drr_object, 1993 drro->drr_type, new_blksz, 1994 drro->drr_bonustype, drro->drr_bonuslen, 1995 dn_slots << DNODE_SHIFT, tx); 1996 } else if (drro->drr_type != doi.doi_type || 1997 new_blksz != doi.doi_data_block_size || 1998 drro->drr_bonustype != doi.doi_bonus_type || 1999 drro->drr_bonuslen != doi.doi_bonus_size) { 2000 /* Currently allocated, but with different properties */ 2001 err = dmu_object_reclaim_dnsize(rwa->os, drro->drr_object, 2002 drro->drr_type, new_blksz, 2003 drro->drr_bonustype, drro->drr_bonuslen, 2004 dn_slots << DNODE_SHIFT, rwa->spill ? 2005 DRR_OBJECT_HAS_SPILL(drro->drr_flags) : B_FALSE, tx); 2006 } else if (rwa->spill && !DRR_OBJECT_HAS_SPILL(drro->drr_flags)) { 2007 /* 2008 * Currently allocated, the existing version of this object 2009 * may reference a spill block that is no longer allocated 2010 * at the source and needs to be freed. 2011 */ 2012 err = dmu_object_rm_spill(rwa->os, drro->drr_object, tx); 2013 } 2014 2015 if (err != 0) { 2016 dmu_tx_commit(tx); 2017 return (SET_ERROR(EINVAL)); 2018 } 2019 2020 if (rwa->or_crypt_params_present) { 2021 /* 2022 * Set the crypt params for the buffer associated with this 2023 * range of dnodes. This causes the blkptr_t to have the 2024 * same crypt params (byteorder, salt, iv, mac) as on the 2025 * sending side. 2026 * 2027 * Since we are committing this tx now, it is possible for 2028 * the dnode block to end up on-disk with the incorrect MAC, 2029 * if subsequent objects in this block are received in a 2030 * different txg. However, since the dataset is marked as 2031 * inconsistent, no code paths will do a non-raw read (or 2032 * decrypt the block / verify the MAC). The receive code and 2033 * scrub code can safely do raw reads and verify the 2034 * checksum. They don't need to verify the MAC. 2035 */ 2036 dmu_buf_t *db = NULL; 2037 uint64_t offset = rwa->or_firstobj * DNODE_MIN_SIZE; 2038 2039 err = dmu_buf_hold_by_dnode(DMU_META_DNODE(rwa->os), 2040 offset, FTAG, &db, DMU_READ_PREFETCH | DMU_READ_NO_DECRYPT); 2041 if (err != 0) { 2042 dmu_tx_commit(tx); 2043 return (SET_ERROR(EINVAL)); 2044 } 2045 2046 dmu_buf_set_crypt_params(db, rwa->or_byteorder, 2047 rwa->or_salt, rwa->or_iv, rwa->or_mac, tx); 2048 2049 dmu_buf_rele(db, FTAG); 2050 2051 rwa->or_crypt_params_present = B_FALSE; 2052 } 2053 2054 dmu_object_set_checksum(rwa->os, drro->drr_object, 2055 drro->drr_checksumtype, tx); 2056 dmu_object_set_compress(rwa->os, drro->drr_object, 2057 drro->drr_compress, tx); 2058 2059 /* handle more restrictive dnode structuring for raw recvs */ 2060 if (rwa->raw) { 2061 /* 2062 * Set the indirect block size, block shift, nlevels. 2063 * This will not fail because we ensured all of the 2064 * blocks were freed earlier if this is a new object. 2065 * For non-new objects block size and indirect block 2066 * shift cannot change and nlevels can only increase. 2067 */ 2068 ASSERT3U(new_blksz, ==, drro->drr_blksz); 2069 VERIFY0(dmu_object_set_blocksize(rwa->os, drro->drr_object, 2070 drro->drr_blksz, drro->drr_indblkshift, tx)); 2071 VERIFY0(dmu_object_set_nlevels(rwa->os, drro->drr_object, 2072 drro->drr_nlevels, tx)); 2073 2074 /* 2075 * Set the maxblkid. This will always succeed because 2076 * we freed all blocks beyond the new maxblkid above. 2077 */ 2078 VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object, 2079 drro->drr_maxblkid, tx)); 2080 } 2081 2082 if (data != NULL) { 2083 dmu_buf_t *db; 2084 dnode_t *dn; 2085 uint32_t flags = DMU_READ_NO_PREFETCH; 2086 2087 if (rwa->raw) 2088 flags |= DMU_READ_NO_DECRYPT; 2089 2090 VERIFY0(dnode_hold(rwa->os, drro->drr_object, FTAG, &dn)); 2091 VERIFY0(dmu_bonus_hold_by_dnode(dn, FTAG, &db, flags)); 2092 2093 dmu_buf_will_dirty(db, tx); 2094 2095 ASSERT3U(db->db_size, >=, drro->drr_bonuslen); 2096 memcpy(db->db_data, data, DRR_OBJECT_PAYLOAD_SIZE(drro)); 2097 2098 /* 2099 * Raw bonus buffers have their byteorder determined by the 2100 * DRR_OBJECT_RANGE record. 2101 */ 2102 if (rwa->byteswap && !rwa->raw) { 2103 dmu_object_byteswap_t byteswap = 2104 DMU_OT_BYTESWAP(drro->drr_bonustype); 2105 dmu_ot_byteswap[byteswap].ob_func(db->db_data, 2106 DRR_OBJECT_PAYLOAD_SIZE(drro)); 2107 } 2108 dmu_buf_rele(db, FTAG); 2109 dnode_rele(dn, FTAG); 2110 } 2111 dmu_tx_commit(tx); 2112 2113 return (0); 2114 } 2115 2116 noinline static int 2117 receive_freeobjects(struct receive_writer_arg *rwa, 2118 struct drr_freeobjects *drrfo) 2119 { 2120 uint64_t obj; 2121 int next_err = 0; 2122 2123 if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj) 2124 return (SET_ERROR(EINVAL)); 2125 2126 for (obj = drrfo->drr_firstobj == 0 ? 1 : drrfo->drr_firstobj; 2127 obj < drrfo->drr_firstobj + drrfo->drr_numobjs && 2128 obj < DN_MAX_OBJECT && next_err == 0; 2129 next_err = dmu_object_next(rwa->os, &obj, FALSE, 0)) { 2130 dmu_object_info_t doi; 2131 int err; 2132 2133 err = dmu_object_info(rwa->os, obj, &doi); 2134 if (err == ENOENT) 2135 continue; 2136 else if (err != 0) 2137 return (err); 2138 2139 err = dmu_free_long_object(rwa->os, obj); 2140 2141 if (err != 0) 2142 return (err); 2143 2144 if (rwa->or_need_sync == ORNS_MAYBE) 2145 rwa->or_need_sync = ORNS_YES; 2146 } 2147 if (next_err != ESRCH) 2148 return (next_err); 2149 return (0); 2150 } 2151 2152 /* 2153 * Note: if this fails, the caller will clean up any records left on the 2154 * rwa->write_batch list. 2155 */ 2156 static int 2157 flush_write_batch_impl(struct receive_writer_arg *rwa) 2158 { 2159 dnode_t *dn; 2160 int err; 2161 2162 if (dnode_hold(rwa->os, rwa->last_object, FTAG, &dn) != 0) 2163 return (SET_ERROR(EINVAL)); 2164 2165 struct receive_record_arg *last_rrd = list_tail(&rwa->write_batch); 2166 struct drr_write *last_drrw = &last_rrd->header.drr_u.drr_write; 2167 2168 struct receive_record_arg *first_rrd = list_head(&rwa->write_batch); 2169 struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write; 2170 2171 ASSERT3U(rwa->last_object, ==, last_drrw->drr_object); 2172 ASSERT3U(rwa->last_offset, ==, last_drrw->drr_offset); 2173 2174 dmu_tx_t *tx = dmu_tx_create(rwa->os); 2175 dmu_tx_hold_write_by_dnode(tx, dn, first_drrw->drr_offset, 2176 last_drrw->drr_offset - first_drrw->drr_offset + 2177 last_drrw->drr_logical_size); 2178 err = dmu_tx_assign(tx, TXG_WAIT); 2179 if (err != 0) { 2180 dmu_tx_abort(tx); 2181 dnode_rele(dn, FTAG); 2182 return (err); 2183 } 2184 2185 struct receive_record_arg *rrd; 2186 while ((rrd = list_head(&rwa->write_batch)) != NULL) { 2187 struct drr_write *drrw = &rrd->header.drr_u.drr_write; 2188 abd_t *abd = rrd->abd; 2189 2190 ASSERT3U(drrw->drr_object, ==, rwa->last_object); 2191 2192 if (drrw->drr_logical_size != dn->dn_datablksz) { 2193 /* 2194 * The WRITE record is larger than the object's block 2195 * size. We must be receiving an incremental 2196 * large-block stream into a dataset that previously did 2197 * a non-large-block receive. Lightweight writes must 2198 * be exactly one block, so we need to decompress the 2199 * data (if compressed) and do a normal dmu_write(). 2200 */ 2201 ASSERT3U(drrw->drr_logical_size, >, dn->dn_datablksz); 2202 if (DRR_WRITE_COMPRESSED(drrw)) { 2203 abd_t *decomp_abd = 2204 abd_alloc_linear(drrw->drr_logical_size, 2205 B_FALSE); 2206 2207 err = zio_decompress_data( 2208 drrw->drr_compressiontype, 2209 abd, abd_to_buf(decomp_abd), 2210 abd_get_size(abd), 2211 abd_get_size(decomp_abd), NULL); 2212 2213 if (err == 0) { 2214 dmu_write_by_dnode(dn, 2215 drrw->drr_offset, 2216 drrw->drr_logical_size, 2217 abd_to_buf(decomp_abd), tx); 2218 } 2219 abd_free(decomp_abd); 2220 } else { 2221 dmu_write_by_dnode(dn, 2222 drrw->drr_offset, 2223 drrw->drr_logical_size, 2224 abd_to_buf(abd), tx); 2225 } 2226 if (err == 0) 2227 abd_free(abd); 2228 } else { 2229 zio_prop_t zp = {0}; 2230 dmu_write_policy(rwa->os, dn, 0, 0, &zp); 2231 2232 zio_flag_t zio_flags = 0; 2233 2234 if (rwa->raw) { 2235 zp.zp_encrypt = B_TRUE; 2236 zp.zp_compress = drrw->drr_compressiontype; 2237 zp.zp_byteorder = ZFS_HOST_BYTEORDER ^ 2238 !!DRR_IS_RAW_BYTESWAPPED(drrw->drr_flags) ^ 2239 rwa->byteswap; 2240 memcpy(zp.zp_salt, drrw->drr_salt, 2241 ZIO_DATA_SALT_LEN); 2242 memcpy(zp.zp_iv, drrw->drr_iv, 2243 ZIO_DATA_IV_LEN); 2244 memcpy(zp.zp_mac, drrw->drr_mac, 2245 ZIO_DATA_MAC_LEN); 2246 if (DMU_OT_IS_ENCRYPTED(zp.zp_type)) { 2247 zp.zp_nopwrite = B_FALSE; 2248 zp.zp_copies = MIN(zp.zp_copies, 2249 SPA_DVAS_PER_BP - 1); 2250 } 2251 zio_flags |= ZIO_FLAG_RAW; 2252 } else if (DRR_WRITE_COMPRESSED(drrw)) { 2253 ASSERT3U(drrw->drr_compressed_size, >, 0); 2254 ASSERT3U(drrw->drr_logical_size, >=, 2255 drrw->drr_compressed_size); 2256 zp.zp_compress = drrw->drr_compressiontype; 2257 zio_flags |= ZIO_FLAG_RAW_COMPRESS; 2258 } else if (rwa->byteswap) { 2259 /* 2260 * Note: compressed blocks never need to be 2261 * byteswapped, because WRITE records for 2262 * metadata blocks are never compressed. The 2263 * exception is raw streams, which are written 2264 * in the original byteorder, and the byteorder 2265 * bit is preserved in the BP by setting 2266 * zp_byteorder above. 2267 */ 2268 dmu_object_byteswap_t byteswap = 2269 DMU_OT_BYTESWAP(drrw->drr_type); 2270 dmu_ot_byteswap[byteswap].ob_func( 2271 abd_to_buf(abd), 2272 DRR_WRITE_PAYLOAD_SIZE(drrw)); 2273 } 2274 2275 /* 2276 * Since this data can't be read until the receive 2277 * completes, we can do a "lightweight" write for 2278 * improved performance. 2279 */ 2280 err = dmu_lightweight_write_by_dnode(dn, 2281 drrw->drr_offset, abd, &zp, zio_flags, tx); 2282 } 2283 2284 if (err != 0) { 2285 /* 2286 * This rrd is left on the list, so the caller will 2287 * free it (and the abd). 2288 */ 2289 break; 2290 } 2291 2292 /* 2293 * Note: If the receive fails, we want the resume stream to 2294 * start with the same record that we last successfully 2295 * received (as opposed to the next record), so that we can 2296 * verify that we are resuming from the correct location. 2297 */ 2298 save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx); 2299 2300 list_remove(&rwa->write_batch, rrd); 2301 kmem_free(rrd, sizeof (*rrd)); 2302 } 2303 2304 dmu_tx_commit(tx); 2305 dnode_rele(dn, FTAG); 2306 return (err); 2307 } 2308 2309 noinline static int 2310 flush_write_batch(struct receive_writer_arg *rwa) 2311 { 2312 if (list_is_empty(&rwa->write_batch)) 2313 return (0); 2314 int err = rwa->err; 2315 if (err == 0) 2316 err = flush_write_batch_impl(rwa); 2317 if (err != 0) { 2318 struct receive_record_arg *rrd; 2319 while ((rrd = list_remove_head(&rwa->write_batch)) != NULL) { 2320 abd_free(rrd->abd); 2321 kmem_free(rrd, sizeof (*rrd)); 2322 } 2323 } 2324 ASSERT(list_is_empty(&rwa->write_batch)); 2325 return (err); 2326 } 2327 2328 noinline static int 2329 receive_process_write_record(struct receive_writer_arg *rwa, 2330 struct receive_record_arg *rrd) 2331 { 2332 int err = 0; 2333 2334 ASSERT3U(rrd->header.drr_type, ==, DRR_WRITE); 2335 struct drr_write *drrw = &rrd->header.drr_u.drr_write; 2336 2337 if (drrw->drr_offset + drrw->drr_logical_size < drrw->drr_offset || 2338 !DMU_OT_IS_VALID(drrw->drr_type)) 2339 return (SET_ERROR(EINVAL)); 2340 2341 if (rwa->heal) { 2342 blkptr_t *bp; 2343 dmu_buf_t *dbp; 2344 dnode_t *dn; 2345 int flags = DB_RF_CANFAIL; 2346 2347 if (rwa->raw) 2348 flags |= DB_RF_NO_DECRYPT; 2349 2350 if (rwa->byteswap) { 2351 dmu_object_byteswap_t byteswap = 2352 DMU_OT_BYTESWAP(drrw->drr_type); 2353 dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(rrd->abd), 2354 DRR_WRITE_PAYLOAD_SIZE(drrw)); 2355 } 2356 2357 err = dmu_buf_hold_noread(rwa->os, drrw->drr_object, 2358 drrw->drr_offset, FTAG, &dbp); 2359 if (err != 0) 2360 return (err); 2361 2362 /* Try to read the object to see if it needs healing */ 2363 err = dbuf_read((dmu_buf_impl_t *)dbp, NULL, flags); 2364 /* 2365 * We only try to heal when dbuf_read() returns a ECKSUMs. 2366 * Other errors (even EIO) get returned to caller. 2367 * EIO indicates that the device is not present/accessible, 2368 * so writing to it will likely fail. 2369 * If the block is healthy, we don't want to overwrite it 2370 * unnecessarily. 2371 */ 2372 if (err != ECKSUM) { 2373 dmu_buf_rele(dbp, FTAG); 2374 return (err); 2375 } 2376 dn = dmu_buf_dnode_enter(dbp); 2377 /* Make sure the on-disk block and recv record sizes match */ 2378 if (drrw->drr_logical_size != 2379 dn->dn_datablkszsec << SPA_MINBLOCKSHIFT) { 2380 err = ENOTSUP; 2381 dmu_buf_dnode_exit(dbp); 2382 dmu_buf_rele(dbp, FTAG); 2383 return (err); 2384 } 2385 /* Get the block pointer for the corrupted block */ 2386 bp = dmu_buf_get_blkptr(dbp); 2387 err = do_corrective_recv(rwa, drrw, rrd, bp); 2388 dmu_buf_dnode_exit(dbp); 2389 dmu_buf_rele(dbp, FTAG); 2390 return (err); 2391 } 2392 2393 /* 2394 * For resuming to work, records must be in increasing order 2395 * by (object, offset). 2396 */ 2397 if (drrw->drr_object < rwa->last_object || 2398 (drrw->drr_object == rwa->last_object && 2399 drrw->drr_offset < rwa->last_offset)) { 2400 return (SET_ERROR(EINVAL)); 2401 } 2402 2403 struct receive_record_arg *first_rrd = list_head(&rwa->write_batch); 2404 struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write; 2405 uint64_t batch_size = 2406 MIN(zfs_recv_write_batch_size, DMU_MAX_ACCESS / 2); 2407 if (first_rrd != NULL && 2408 (drrw->drr_object != first_drrw->drr_object || 2409 drrw->drr_offset >= first_drrw->drr_offset + batch_size)) { 2410 err = flush_write_batch(rwa); 2411 if (err != 0) 2412 return (err); 2413 } 2414 2415 rwa->last_object = drrw->drr_object; 2416 rwa->last_offset = drrw->drr_offset; 2417 2418 if (rwa->last_object > rwa->max_object) 2419 rwa->max_object = rwa->last_object; 2420 2421 list_insert_tail(&rwa->write_batch, rrd); 2422 /* 2423 * Return EAGAIN to indicate that we will use this rrd again, 2424 * so the caller should not free it 2425 */ 2426 return (EAGAIN); 2427 } 2428 2429 static int 2430 receive_write_embedded(struct receive_writer_arg *rwa, 2431 struct drr_write_embedded *drrwe, void *data) 2432 { 2433 dmu_tx_t *tx; 2434 int err; 2435 2436 if (drrwe->drr_offset + drrwe->drr_length < drrwe->drr_offset) 2437 return (SET_ERROR(EINVAL)); 2438 2439 if (drrwe->drr_psize > BPE_PAYLOAD_SIZE) 2440 return (SET_ERROR(EINVAL)); 2441 2442 if (drrwe->drr_etype >= NUM_BP_EMBEDDED_TYPES) 2443 return (SET_ERROR(EINVAL)); 2444 if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS) 2445 return (SET_ERROR(EINVAL)); 2446 if (rwa->raw) 2447 return (SET_ERROR(EINVAL)); 2448 2449 if (drrwe->drr_object > rwa->max_object) 2450 rwa->max_object = drrwe->drr_object; 2451 2452 tx = dmu_tx_create(rwa->os); 2453 2454 dmu_tx_hold_write(tx, drrwe->drr_object, 2455 drrwe->drr_offset, drrwe->drr_length); 2456 err = dmu_tx_assign(tx, TXG_WAIT); 2457 if (err != 0) { 2458 dmu_tx_abort(tx); 2459 return (err); 2460 } 2461 2462 dmu_write_embedded(rwa->os, drrwe->drr_object, 2463 drrwe->drr_offset, data, drrwe->drr_etype, 2464 drrwe->drr_compression, drrwe->drr_lsize, drrwe->drr_psize, 2465 rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx); 2466 2467 /* See comment in restore_write. */ 2468 save_resume_state(rwa, drrwe->drr_object, drrwe->drr_offset, tx); 2469 dmu_tx_commit(tx); 2470 return (0); 2471 } 2472 2473 static int 2474 receive_spill(struct receive_writer_arg *rwa, struct drr_spill *drrs, 2475 abd_t *abd) 2476 { 2477 dmu_buf_t *db, *db_spill; 2478 int err; 2479 2480 if (drrs->drr_length < SPA_MINBLOCKSIZE || 2481 drrs->drr_length > spa_maxblocksize(dmu_objset_spa(rwa->os))) 2482 return (SET_ERROR(EINVAL)); 2483 2484 /* 2485 * This is an unmodified spill block which was added to the stream 2486 * to resolve an issue with incorrectly removing spill blocks. It 2487 * should be ignored by current versions of the code which support 2488 * the DRR_FLAG_SPILL_BLOCK flag. 2489 */ 2490 if (rwa->spill && DRR_SPILL_IS_UNMODIFIED(drrs->drr_flags)) { 2491 abd_free(abd); 2492 return (0); 2493 } 2494 2495 if (rwa->raw) { 2496 if (!DMU_OT_IS_VALID(drrs->drr_type) || 2497 drrs->drr_compressiontype >= ZIO_COMPRESS_FUNCTIONS || 2498 drrs->drr_compressed_size == 0) 2499 return (SET_ERROR(EINVAL)); 2500 } 2501 2502 if (dmu_object_info(rwa->os, drrs->drr_object, NULL) != 0) 2503 return (SET_ERROR(EINVAL)); 2504 2505 if (drrs->drr_object > rwa->max_object) 2506 rwa->max_object = drrs->drr_object; 2507 2508 VERIFY0(dmu_bonus_hold(rwa->os, drrs->drr_object, FTAG, &db)); 2509 if ((err = dmu_spill_hold_by_bonus(db, DMU_READ_NO_DECRYPT, FTAG, 2510 &db_spill)) != 0) { 2511 dmu_buf_rele(db, FTAG); 2512 return (err); 2513 } 2514 2515 dmu_tx_t *tx = dmu_tx_create(rwa->os); 2516 2517 dmu_tx_hold_spill(tx, db->db_object); 2518 2519 err = dmu_tx_assign(tx, TXG_WAIT); 2520 if (err != 0) { 2521 dmu_buf_rele(db, FTAG); 2522 dmu_buf_rele(db_spill, FTAG); 2523 dmu_tx_abort(tx); 2524 return (err); 2525 } 2526 2527 /* 2528 * Spill blocks may both grow and shrink. When a change in size 2529 * occurs any existing dbuf must be updated to match the logical 2530 * size of the provided arc_buf_t. 2531 */ 2532 if (db_spill->db_size != drrs->drr_length) { 2533 dmu_buf_will_fill(db_spill, tx); 2534 VERIFY0(dbuf_spill_set_blksz(db_spill, 2535 drrs->drr_length, tx)); 2536 } 2537 2538 arc_buf_t *abuf; 2539 if (rwa->raw) { 2540 boolean_t byteorder = ZFS_HOST_BYTEORDER ^ 2541 !!DRR_IS_RAW_BYTESWAPPED(drrs->drr_flags) ^ 2542 rwa->byteswap; 2543 2544 abuf = arc_loan_raw_buf(dmu_objset_spa(rwa->os), 2545 drrs->drr_object, byteorder, drrs->drr_salt, 2546 drrs->drr_iv, drrs->drr_mac, drrs->drr_type, 2547 drrs->drr_compressed_size, drrs->drr_length, 2548 drrs->drr_compressiontype, 0); 2549 } else { 2550 abuf = arc_loan_buf(dmu_objset_spa(rwa->os), 2551 DMU_OT_IS_METADATA(drrs->drr_type), 2552 drrs->drr_length); 2553 if (rwa->byteswap) { 2554 dmu_object_byteswap_t byteswap = 2555 DMU_OT_BYTESWAP(drrs->drr_type); 2556 dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(abd), 2557 DRR_SPILL_PAYLOAD_SIZE(drrs)); 2558 } 2559 } 2560 2561 memcpy(abuf->b_data, abd_to_buf(abd), DRR_SPILL_PAYLOAD_SIZE(drrs)); 2562 abd_free(abd); 2563 dbuf_assign_arcbuf((dmu_buf_impl_t *)db_spill, abuf, tx); 2564 2565 dmu_buf_rele(db, FTAG); 2566 dmu_buf_rele(db_spill, FTAG); 2567 2568 dmu_tx_commit(tx); 2569 return (0); 2570 } 2571 2572 noinline static int 2573 receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf) 2574 { 2575 int err; 2576 2577 if (drrf->drr_length != -1ULL && 2578 drrf->drr_offset + drrf->drr_length < drrf->drr_offset) 2579 return (SET_ERROR(EINVAL)); 2580 2581 if (dmu_object_info(rwa->os, drrf->drr_object, NULL) != 0) 2582 return (SET_ERROR(EINVAL)); 2583 2584 if (drrf->drr_object > rwa->max_object) 2585 rwa->max_object = drrf->drr_object; 2586 2587 err = dmu_free_long_range(rwa->os, drrf->drr_object, 2588 drrf->drr_offset, drrf->drr_length); 2589 2590 return (err); 2591 } 2592 2593 static int 2594 receive_object_range(struct receive_writer_arg *rwa, 2595 struct drr_object_range *drror) 2596 { 2597 /* 2598 * By default, we assume this block is in our native format 2599 * (ZFS_HOST_BYTEORDER). We then take into account whether 2600 * the send stream is byteswapped (rwa->byteswap). Finally, 2601 * we need to byteswap again if this particular block was 2602 * in non-native format on the send side. 2603 */ 2604 boolean_t byteorder = ZFS_HOST_BYTEORDER ^ rwa->byteswap ^ 2605 !!DRR_IS_RAW_BYTESWAPPED(drror->drr_flags); 2606 2607 /* 2608 * Since dnode block sizes are constant, we should not need to worry 2609 * about making sure that the dnode block size is the same on the 2610 * sending and receiving sides for the time being. For non-raw sends, 2611 * this does not matter (and in fact we do not send a DRR_OBJECT_RANGE 2612 * record at all). Raw sends require this record type because the 2613 * encryption parameters are used to protect an entire block of bonus 2614 * buffers. If the size of dnode blocks ever becomes variable, 2615 * handling will need to be added to ensure that dnode block sizes 2616 * match on the sending and receiving side. 2617 */ 2618 if (drror->drr_numslots != DNODES_PER_BLOCK || 2619 P2PHASE(drror->drr_firstobj, DNODES_PER_BLOCK) != 0 || 2620 !rwa->raw) 2621 return (SET_ERROR(EINVAL)); 2622 2623 if (drror->drr_firstobj > rwa->max_object) 2624 rwa->max_object = drror->drr_firstobj; 2625 2626 /* 2627 * The DRR_OBJECT_RANGE handling must be deferred to receive_object() 2628 * so that the block of dnodes is not written out when it's empty, 2629 * and converted to a HOLE BP. 2630 */ 2631 rwa->or_crypt_params_present = B_TRUE; 2632 rwa->or_firstobj = drror->drr_firstobj; 2633 rwa->or_numslots = drror->drr_numslots; 2634 memcpy(rwa->or_salt, drror->drr_salt, ZIO_DATA_SALT_LEN); 2635 memcpy(rwa->or_iv, drror->drr_iv, ZIO_DATA_IV_LEN); 2636 memcpy(rwa->or_mac, drror->drr_mac, ZIO_DATA_MAC_LEN); 2637 rwa->or_byteorder = byteorder; 2638 2639 rwa->or_need_sync = ORNS_MAYBE; 2640 2641 return (0); 2642 } 2643 2644 /* 2645 * Until we have the ability to redact large ranges of data efficiently, we 2646 * process these records as frees. 2647 */ 2648 noinline static int 2649 receive_redact(struct receive_writer_arg *rwa, struct drr_redact *drrr) 2650 { 2651 struct drr_free drrf = {0}; 2652 drrf.drr_length = drrr->drr_length; 2653 drrf.drr_object = drrr->drr_object; 2654 drrf.drr_offset = drrr->drr_offset; 2655 drrf.drr_toguid = drrr->drr_toguid; 2656 return (receive_free(rwa, &drrf)); 2657 } 2658 2659 /* used to destroy the drc_ds on error */ 2660 static void 2661 dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc) 2662 { 2663 dsl_dataset_t *ds = drc->drc_ds; 2664 ds_hold_flags_t dsflags; 2665 2666 dsflags = (drc->drc_raw) ? DS_HOLD_FLAG_NONE : DS_HOLD_FLAG_DECRYPT; 2667 /* 2668 * Wait for the txg sync before cleaning up the receive. For 2669 * resumable receives, this ensures that our resume state has 2670 * been written out to disk. For raw receives, this ensures 2671 * that the user accounting code will not attempt to do anything 2672 * after we stopped receiving the dataset. 2673 */ 2674 txg_wait_synced(ds->ds_dir->dd_pool, 0); 2675 ds->ds_objset->os_raw_receive = B_FALSE; 2676 2677 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2678 if (drc->drc_resumable && drc->drc_should_save && 2679 !BP_IS_HOLE(dsl_dataset_get_blkptr(ds))) { 2680 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2681 dsl_dataset_disown(ds, dsflags, dmu_recv_tag); 2682 } else { 2683 char name[ZFS_MAX_DATASET_NAME_LEN]; 2684 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2685 dsl_dataset_name(ds, name); 2686 dsl_dataset_disown(ds, dsflags, dmu_recv_tag); 2687 if (!drc->drc_heal) 2688 (void) dsl_destroy_head(name); 2689 } 2690 } 2691 2692 static void 2693 receive_cksum(dmu_recv_cookie_t *drc, int len, void *buf) 2694 { 2695 if (drc->drc_byteswap) { 2696 (void) fletcher_4_incremental_byteswap(buf, len, 2697 &drc->drc_cksum); 2698 } else { 2699 (void) fletcher_4_incremental_native(buf, len, &drc->drc_cksum); 2700 } 2701 } 2702 2703 /* 2704 * Read the payload into a buffer of size len, and update the current record's 2705 * payload field. 2706 * Allocate drc->drc_next_rrd and read the next record's header into 2707 * drc->drc_next_rrd->header. 2708 * Verify checksum of payload and next record. 2709 */ 2710 static int 2711 receive_read_payload_and_next_header(dmu_recv_cookie_t *drc, int len, void *buf) 2712 { 2713 int err; 2714 2715 if (len != 0) { 2716 ASSERT3U(len, <=, SPA_MAXBLOCKSIZE); 2717 err = receive_read(drc, len, buf); 2718 if (err != 0) 2719 return (err); 2720 receive_cksum(drc, len, buf); 2721 2722 /* note: rrd is NULL when reading the begin record's payload */ 2723 if (drc->drc_rrd != NULL) { 2724 drc->drc_rrd->payload = buf; 2725 drc->drc_rrd->payload_size = len; 2726 drc->drc_rrd->bytes_read = drc->drc_bytes_read; 2727 } 2728 } else { 2729 ASSERT3P(buf, ==, NULL); 2730 } 2731 2732 drc->drc_prev_cksum = drc->drc_cksum; 2733 2734 drc->drc_next_rrd = kmem_zalloc(sizeof (*drc->drc_next_rrd), KM_SLEEP); 2735 err = receive_read(drc, sizeof (drc->drc_next_rrd->header), 2736 &drc->drc_next_rrd->header); 2737 drc->drc_next_rrd->bytes_read = drc->drc_bytes_read; 2738 2739 if (err != 0) { 2740 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 2741 drc->drc_next_rrd = NULL; 2742 return (err); 2743 } 2744 if (drc->drc_next_rrd->header.drr_type == DRR_BEGIN) { 2745 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 2746 drc->drc_next_rrd = NULL; 2747 return (SET_ERROR(EINVAL)); 2748 } 2749 2750 /* 2751 * Note: checksum is of everything up to but not including the 2752 * checksum itself. 2753 */ 2754 ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), 2755 ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t)); 2756 receive_cksum(drc, 2757 offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), 2758 &drc->drc_next_rrd->header); 2759 2760 zio_cksum_t cksum_orig = 2761 drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum; 2762 zio_cksum_t *cksump = 2763 &drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum; 2764 2765 if (drc->drc_byteswap) 2766 byteswap_record(&drc->drc_next_rrd->header); 2767 2768 if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) && 2769 !ZIO_CHECKSUM_EQUAL(drc->drc_cksum, *cksump)) { 2770 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 2771 drc->drc_next_rrd = NULL; 2772 return (SET_ERROR(ECKSUM)); 2773 } 2774 2775 receive_cksum(drc, sizeof (cksum_orig), &cksum_orig); 2776 2777 return (0); 2778 } 2779 2780 /* 2781 * Issue the prefetch reads for any necessary indirect blocks. 2782 * 2783 * We use the object ignore list to tell us whether or not to issue prefetches 2784 * for a given object. We do this for both correctness (in case the blocksize 2785 * of an object has changed) and performance (if the object doesn't exist, don't 2786 * needlessly try to issue prefetches). We also trim the list as we go through 2787 * the stream to prevent it from growing to an unbounded size. 2788 * 2789 * The object numbers within will always be in sorted order, and any write 2790 * records we see will also be in sorted order, but they're not sorted with 2791 * respect to each other (i.e. we can get several object records before 2792 * receiving each object's write records). As a result, once we've reached a 2793 * given object number, we can safely remove any reference to lower object 2794 * numbers in the ignore list. In practice, we receive up to 32 object records 2795 * before receiving write records, so the list can have up to 32 nodes in it. 2796 */ 2797 static void 2798 receive_read_prefetch(dmu_recv_cookie_t *drc, uint64_t object, uint64_t offset, 2799 uint64_t length) 2800 { 2801 if (!objlist_exists(drc->drc_ignore_objlist, object)) { 2802 dmu_prefetch(drc->drc_os, object, 1, offset, length, 2803 ZIO_PRIORITY_SYNC_READ); 2804 } 2805 } 2806 2807 /* 2808 * Read records off the stream, issuing any necessary prefetches. 2809 */ 2810 static int 2811 receive_read_record(dmu_recv_cookie_t *drc) 2812 { 2813 int err; 2814 2815 switch (drc->drc_rrd->header.drr_type) { 2816 case DRR_OBJECT: 2817 { 2818 struct drr_object *drro = 2819 &drc->drc_rrd->header.drr_u.drr_object; 2820 uint32_t size = DRR_OBJECT_PAYLOAD_SIZE(drro); 2821 void *buf = NULL; 2822 dmu_object_info_t doi; 2823 2824 if (size != 0) 2825 buf = kmem_zalloc(size, KM_SLEEP); 2826 2827 err = receive_read_payload_and_next_header(drc, size, buf); 2828 if (err != 0) { 2829 kmem_free(buf, size); 2830 return (err); 2831 } 2832 err = dmu_object_info(drc->drc_os, drro->drr_object, &doi); 2833 /* 2834 * See receive_read_prefetch for an explanation why we're 2835 * storing this object in the ignore_obj_list. 2836 */ 2837 if (err == ENOENT || err == EEXIST || 2838 (err == 0 && doi.doi_data_block_size != drro->drr_blksz)) { 2839 objlist_insert(drc->drc_ignore_objlist, 2840 drro->drr_object); 2841 err = 0; 2842 } 2843 return (err); 2844 } 2845 case DRR_FREEOBJECTS: 2846 { 2847 err = receive_read_payload_and_next_header(drc, 0, NULL); 2848 return (err); 2849 } 2850 case DRR_WRITE: 2851 { 2852 struct drr_write *drrw = &drc->drc_rrd->header.drr_u.drr_write; 2853 int size = DRR_WRITE_PAYLOAD_SIZE(drrw); 2854 abd_t *abd = abd_alloc_linear(size, B_FALSE); 2855 err = receive_read_payload_and_next_header(drc, size, 2856 abd_to_buf(abd)); 2857 if (err != 0) { 2858 abd_free(abd); 2859 return (err); 2860 } 2861 drc->drc_rrd->abd = abd; 2862 receive_read_prefetch(drc, drrw->drr_object, drrw->drr_offset, 2863 drrw->drr_logical_size); 2864 return (err); 2865 } 2866 case DRR_WRITE_EMBEDDED: 2867 { 2868 struct drr_write_embedded *drrwe = 2869 &drc->drc_rrd->header.drr_u.drr_write_embedded; 2870 uint32_t size = P2ROUNDUP(drrwe->drr_psize, 8); 2871 void *buf = kmem_zalloc(size, KM_SLEEP); 2872 2873 err = receive_read_payload_and_next_header(drc, size, buf); 2874 if (err != 0) { 2875 kmem_free(buf, size); 2876 return (err); 2877 } 2878 2879 receive_read_prefetch(drc, drrwe->drr_object, drrwe->drr_offset, 2880 drrwe->drr_length); 2881 return (err); 2882 } 2883 case DRR_FREE: 2884 case DRR_REDACT: 2885 { 2886 /* 2887 * It might be beneficial to prefetch indirect blocks here, but 2888 * we don't really have the data to decide for sure. 2889 */ 2890 err = receive_read_payload_and_next_header(drc, 0, NULL); 2891 return (err); 2892 } 2893 case DRR_END: 2894 { 2895 struct drr_end *drre = &drc->drc_rrd->header.drr_u.drr_end; 2896 if (!ZIO_CHECKSUM_EQUAL(drc->drc_prev_cksum, 2897 drre->drr_checksum)) 2898 return (SET_ERROR(ECKSUM)); 2899 return (0); 2900 } 2901 case DRR_SPILL: 2902 { 2903 struct drr_spill *drrs = &drc->drc_rrd->header.drr_u.drr_spill; 2904 int size = DRR_SPILL_PAYLOAD_SIZE(drrs); 2905 abd_t *abd = abd_alloc_linear(size, B_FALSE); 2906 err = receive_read_payload_and_next_header(drc, size, 2907 abd_to_buf(abd)); 2908 if (err != 0) 2909 abd_free(abd); 2910 else 2911 drc->drc_rrd->abd = abd; 2912 return (err); 2913 } 2914 case DRR_OBJECT_RANGE: 2915 { 2916 err = receive_read_payload_and_next_header(drc, 0, NULL); 2917 return (err); 2918 2919 } 2920 default: 2921 return (SET_ERROR(EINVAL)); 2922 } 2923 } 2924 2925 2926 2927 static void 2928 dprintf_drr(struct receive_record_arg *rrd, int err) 2929 { 2930 #ifdef ZFS_DEBUG 2931 switch (rrd->header.drr_type) { 2932 case DRR_OBJECT: 2933 { 2934 struct drr_object *drro = &rrd->header.drr_u.drr_object; 2935 dprintf("drr_type = OBJECT obj = %llu type = %u " 2936 "bonustype = %u blksz = %u bonuslen = %u cksumtype = %u " 2937 "compress = %u dn_slots = %u err = %d\n", 2938 (u_longlong_t)drro->drr_object, drro->drr_type, 2939 drro->drr_bonustype, drro->drr_blksz, drro->drr_bonuslen, 2940 drro->drr_checksumtype, drro->drr_compress, 2941 drro->drr_dn_slots, err); 2942 break; 2943 } 2944 case DRR_FREEOBJECTS: 2945 { 2946 struct drr_freeobjects *drrfo = 2947 &rrd->header.drr_u.drr_freeobjects; 2948 dprintf("drr_type = FREEOBJECTS firstobj = %llu " 2949 "numobjs = %llu err = %d\n", 2950 (u_longlong_t)drrfo->drr_firstobj, 2951 (u_longlong_t)drrfo->drr_numobjs, err); 2952 break; 2953 } 2954 case DRR_WRITE: 2955 { 2956 struct drr_write *drrw = &rrd->header.drr_u.drr_write; 2957 dprintf("drr_type = WRITE obj = %llu type = %u offset = %llu " 2958 "lsize = %llu cksumtype = %u flags = %u " 2959 "compress = %u psize = %llu err = %d\n", 2960 (u_longlong_t)drrw->drr_object, drrw->drr_type, 2961 (u_longlong_t)drrw->drr_offset, 2962 (u_longlong_t)drrw->drr_logical_size, 2963 drrw->drr_checksumtype, drrw->drr_flags, 2964 drrw->drr_compressiontype, 2965 (u_longlong_t)drrw->drr_compressed_size, err); 2966 break; 2967 } 2968 case DRR_WRITE_BYREF: 2969 { 2970 struct drr_write_byref *drrwbr = 2971 &rrd->header.drr_u.drr_write_byref; 2972 dprintf("drr_type = WRITE_BYREF obj = %llu offset = %llu " 2973 "length = %llu toguid = %llx refguid = %llx " 2974 "refobject = %llu refoffset = %llu cksumtype = %u " 2975 "flags = %u err = %d\n", 2976 (u_longlong_t)drrwbr->drr_object, 2977 (u_longlong_t)drrwbr->drr_offset, 2978 (u_longlong_t)drrwbr->drr_length, 2979 (u_longlong_t)drrwbr->drr_toguid, 2980 (u_longlong_t)drrwbr->drr_refguid, 2981 (u_longlong_t)drrwbr->drr_refobject, 2982 (u_longlong_t)drrwbr->drr_refoffset, 2983 drrwbr->drr_checksumtype, drrwbr->drr_flags, err); 2984 break; 2985 } 2986 case DRR_WRITE_EMBEDDED: 2987 { 2988 struct drr_write_embedded *drrwe = 2989 &rrd->header.drr_u.drr_write_embedded; 2990 dprintf("drr_type = WRITE_EMBEDDED obj = %llu offset = %llu " 2991 "length = %llu compress = %u etype = %u lsize = %u " 2992 "psize = %u err = %d\n", 2993 (u_longlong_t)drrwe->drr_object, 2994 (u_longlong_t)drrwe->drr_offset, 2995 (u_longlong_t)drrwe->drr_length, 2996 drrwe->drr_compression, drrwe->drr_etype, 2997 drrwe->drr_lsize, drrwe->drr_psize, err); 2998 break; 2999 } 3000 case DRR_FREE: 3001 { 3002 struct drr_free *drrf = &rrd->header.drr_u.drr_free; 3003 dprintf("drr_type = FREE obj = %llu offset = %llu " 3004 "length = %lld err = %d\n", 3005 (u_longlong_t)drrf->drr_object, 3006 (u_longlong_t)drrf->drr_offset, 3007 (longlong_t)drrf->drr_length, 3008 err); 3009 break; 3010 } 3011 case DRR_SPILL: 3012 { 3013 struct drr_spill *drrs = &rrd->header.drr_u.drr_spill; 3014 dprintf("drr_type = SPILL obj = %llu length = %llu " 3015 "err = %d\n", (u_longlong_t)drrs->drr_object, 3016 (u_longlong_t)drrs->drr_length, err); 3017 break; 3018 } 3019 case DRR_OBJECT_RANGE: 3020 { 3021 struct drr_object_range *drror = 3022 &rrd->header.drr_u.drr_object_range; 3023 dprintf("drr_type = OBJECT_RANGE firstobj = %llu " 3024 "numslots = %llu flags = %u err = %d\n", 3025 (u_longlong_t)drror->drr_firstobj, 3026 (u_longlong_t)drror->drr_numslots, 3027 drror->drr_flags, err); 3028 break; 3029 } 3030 default: 3031 return; 3032 } 3033 #endif 3034 } 3035 3036 /* 3037 * Commit the records to the pool. 3038 */ 3039 static int 3040 receive_process_record(struct receive_writer_arg *rwa, 3041 struct receive_record_arg *rrd) 3042 { 3043 int err; 3044 3045 /* Processing in order, therefore bytes_read should be increasing. */ 3046 ASSERT3U(rrd->bytes_read, >=, rwa->bytes_read); 3047 rwa->bytes_read = rrd->bytes_read; 3048 3049 /* We can only heal write records; other ones get ignored */ 3050 if (rwa->heal && rrd->header.drr_type != DRR_WRITE) { 3051 if (rrd->abd != NULL) { 3052 abd_free(rrd->abd); 3053 rrd->abd = NULL; 3054 } else if (rrd->payload != NULL) { 3055 kmem_free(rrd->payload, rrd->payload_size); 3056 rrd->payload = NULL; 3057 } 3058 return (0); 3059 } 3060 3061 if (!rwa->heal && rrd->header.drr_type != DRR_WRITE) { 3062 err = flush_write_batch(rwa); 3063 if (err != 0) { 3064 if (rrd->abd != NULL) { 3065 abd_free(rrd->abd); 3066 rrd->abd = NULL; 3067 rrd->payload = NULL; 3068 } else if (rrd->payload != NULL) { 3069 kmem_free(rrd->payload, rrd->payload_size); 3070 rrd->payload = NULL; 3071 } 3072 3073 return (err); 3074 } 3075 } 3076 3077 switch (rrd->header.drr_type) { 3078 case DRR_OBJECT: 3079 { 3080 struct drr_object *drro = &rrd->header.drr_u.drr_object; 3081 err = receive_object(rwa, drro, rrd->payload); 3082 kmem_free(rrd->payload, rrd->payload_size); 3083 rrd->payload = NULL; 3084 break; 3085 } 3086 case DRR_FREEOBJECTS: 3087 { 3088 struct drr_freeobjects *drrfo = 3089 &rrd->header.drr_u.drr_freeobjects; 3090 err = receive_freeobjects(rwa, drrfo); 3091 break; 3092 } 3093 case DRR_WRITE: 3094 { 3095 err = receive_process_write_record(rwa, rrd); 3096 if (rwa->heal) { 3097 /* 3098 * If healing - always free the abd after processing 3099 */ 3100 abd_free(rrd->abd); 3101 rrd->abd = NULL; 3102 } else if (err != EAGAIN) { 3103 /* 3104 * On success, a non-healing 3105 * receive_process_write_record() returns 3106 * EAGAIN to indicate that we do not want to free 3107 * the rrd or arc_buf. 3108 */ 3109 ASSERT(err != 0); 3110 abd_free(rrd->abd); 3111 rrd->abd = NULL; 3112 } 3113 break; 3114 } 3115 case DRR_WRITE_EMBEDDED: 3116 { 3117 struct drr_write_embedded *drrwe = 3118 &rrd->header.drr_u.drr_write_embedded; 3119 err = receive_write_embedded(rwa, drrwe, rrd->payload); 3120 kmem_free(rrd->payload, rrd->payload_size); 3121 rrd->payload = NULL; 3122 break; 3123 } 3124 case DRR_FREE: 3125 { 3126 struct drr_free *drrf = &rrd->header.drr_u.drr_free; 3127 err = receive_free(rwa, drrf); 3128 break; 3129 } 3130 case DRR_SPILL: 3131 { 3132 struct drr_spill *drrs = &rrd->header.drr_u.drr_spill; 3133 err = receive_spill(rwa, drrs, rrd->abd); 3134 if (err != 0) 3135 abd_free(rrd->abd); 3136 rrd->abd = NULL; 3137 rrd->payload = NULL; 3138 break; 3139 } 3140 case DRR_OBJECT_RANGE: 3141 { 3142 struct drr_object_range *drror = 3143 &rrd->header.drr_u.drr_object_range; 3144 err = receive_object_range(rwa, drror); 3145 break; 3146 } 3147 case DRR_REDACT: 3148 { 3149 struct drr_redact *drrr = &rrd->header.drr_u.drr_redact; 3150 err = receive_redact(rwa, drrr); 3151 break; 3152 } 3153 default: 3154 err = (SET_ERROR(EINVAL)); 3155 } 3156 3157 if (err != 0) 3158 dprintf_drr(rrd, err); 3159 3160 return (err); 3161 } 3162 3163 /* 3164 * dmu_recv_stream's worker thread; pull records off the queue, and then call 3165 * receive_process_record When we're done, signal the main thread and exit. 3166 */ 3167 static __attribute__((noreturn)) void 3168 receive_writer_thread(void *arg) 3169 { 3170 struct receive_writer_arg *rwa = arg; 3171 struct receive_record_arg *rrd; 3172 fstrans_cookie_t cookie = spl_fstrans_mark(); 3173 3174 for (rrd = bqueue_dequeue(&rwa->q); !rrd->eos_marker; 3175 rrd = bqueue_dequeue(&rwa->q)) { 3176 /* 3177 * If there's an error, the main thread will stop putting things 3178 * on the queue, but we need to clear everything in it before we 3179 * can exit. 3180 */ 3181 int err = 0; 3182 if (rwa->err == 0) { 3183 err = receive_process_record(rwa, rrd); 3184 } else if (rrd->abd != NULL) { 3185 abd_free(rrd->abd); 3186 rrd->abd = NULL; 3187 rrd->payload = NULL; 3188 } else if (rrd->payload != NULL) { 3189 kmem_free(rrd->payload, rrd->payload_size); 3190 rrd->payload = NULL; 3191 } 3192 /* 3193 * EAGAIN indicates that this record has been saved (on 3194 * raw->write_batch), and will be used again, so we don't 3195 * free it. 3196 * When healing data we always need to free the record. 3197 */ 3198 if (err != EAGAIN || rwa->heal) { 3199 if (rwa->err == 0) 3200 rwa->err = err; 3201 kmem_free(rrd, sizeof (*rrd)); 3202 } 3203 } 3204 kmem_free(rrd, sizeof (*rrd)); 3205 3206 if (rwa->heal) { 3207 zio_wait(rwa->heal_pio); 3208 } else { 3209 int err = flush_write_batch(rwa); 3210 if (rwa->err == 0) 3211 rwa->err = err; 3212 } 3213 mutex_enter(&rwa->mutex); 3214 rwa->done = B_TRUE; 3215 cv_signal(&rwa->cv); 3216 mutex_exit(&rwa->mutex); 3217 spl_fstrans_unmark(cookie); 3218 thread_exit(); 3219 } 3220 3221 static int 3222 resume_check(dmu_recv_cookie_t *drc, nvlist_t *begin_nvl) 3223 { 3224 uint64_t val; 3225 objset_t *mos = dmu_objset_pool(drc->drc_os)->dp_meta_objset; 3226 uint64_t dsobj = dmu_objset_id(drc->drc_os); 3227 uint64_t resume_obj, resume_off; 3228 3229 if (nvlist_lookup_uint64(begin_nvl, 3230 "resume_object", &resume_obj) != 0 || 3231 nvlist_lookup_uint64(begin_nvl, 3232 "resume_offset", &resume_off) != 0) { 3233 return (SET_ERROR(EINVAL)); 3234 } 3235 VERIFY0(zap_lookup(mos, dsobj, 3236 DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val)); 3237 if (resume_obj != val) 3238 return (SET_ERROR(EINVAL)); 3239 VERIFY0(zap_lookup(mos, dsobj, 3240 DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val)); 3241 if (resume_off != val) 3242 return (SET_ERROR(EINVAL)); 3243 3244 return (0); 3245 } 3246 3247 /* 3248 * Read in the stream's records, one by one, and apply them to the pool. There 3249 * are two threads involved; the thread that calls this function will spin up a 3250 * worker thread, read the records off the stream one by one, and issue 3251 * prefetches for any necessary indirect blocks. It will then push the records 3252 * onto an internal blocking queue. The worker thread will pull the records off 3253 * the queue, and actually write the data into the DMU. This way, the worker 3254 * thread doesn't have to wait for reads to complete, since everything it needs 3255 * (the indirect blocks) will be prefetched. 3256 * 3257 * NB: callers *must* call dmu_recv_end() if this succeeds. 3258 */ 3259 int 3260 dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp) 3261 { 3262 int err = 0; 3263 struct receive_writer_arg *rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP); 3264 3265 if (dsl_dataset_has_resume_receive_state(drc->drc_ds)) { 3266 uint64_t bytes = 0; 3267 (void) zap_lookup(drc->drc_ds->ds_dir->dd_pool->dp_meta_objset, 3268 drc->drc_ds->ds_object, DS_FIELD_RESUME_BYTES, 3269 sizeof (bytes), 1, &bytes); 3270 drc->drc_bytes_read += bytes; 3271 } 3272 3273 drc->drc_ignore_objlist = objlist_create(); 3274 3275 /* these were verified in dmu_recv_begin */ 3276 ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==, 3277 DMU_SUBSTREAM); 3278 ASSERT3U(drc->drc_drrb->drr_type, <, DMU_OST_NUMTYPES); 3279 3280 ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT); 3281 ASSERT0(drc->drc_os->os_encrypted && 3282 (drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)); 3283 3284 /* handle DSL encryption key payload */ 3285 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) { 3286 nvlist_t *keynvl = NULL; 3287 3288 ASSERT(drc->drc_os->os_encrypted); 3289 ASSERT(drc->drc_raw); 3290 3291 err = nvlist_lookup_nvlist(drc->drc_begin_nvl, "crypt_keydata", 3292 &keynvl); 3293 if (err != 0) 3294 goto out; 3295 3296 if (!drc->drc_heal) { 3297 /* 3298 * If this is a new dataset we set the key immediately. 3299 * Otherwise we don't want to change the key until we 3300 * are sure the rest of the receive succeeded so we 3301 * stash the keynvl away until then. 3302 */ 3303 err = dsl_crypto_recv_raw(spa_name(drc->drc_os->os_spa), 3304 drc->drc_ds->ds_object, drc->drc_fromsnapobj, 3305 drc->drc_drrb->drr_type, keynvl, drc->drc_newfs); 3306 if (err != 0) 3307 goto out; 3308 } 3309 3310 /* see comment in dmu_recv_end_sync() */ 3311 drc->drc_ivset_guid = 0; 3312 (void) nvlist_lookup_uint64(keynvl, "to_ivset_guid", 3313 &drc->drc_ivset_guid); 3314 3315 if (!drc->drc_newfs) 3316 drc->drc_keynvl = fnvlist_dup(keynvl); 3317 } 3318 3319 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) { 3320 err = resume_check(drc, drc->drc_begin_nvl); 3321 if (err != 0) 3322 goto out; 3323 } 3324 3325 /* 3326 * For compatibility with recursive send streams, we do this here, 3327 * rather than in dmu_recv_begin. If we pull the next header too 3328 * early, and it's the END record, we break the `recv_skip` logic. 3329 */ 3330 if (drc->drc_drr_begin->drr_payloadlen == 0) { 3331 err = receive_read_payload_and_next_header(drc, 0, NULL); 3332 if (err != 0) 3333 goto out; 3334 } 3335 3336 /* 3337 * If we failed before this point we will clean up any new resume 3338 * state that was created. Now that we've gotten past the initial 3339 * checks we are ok to retain that resume state. 3340 */ 3341 drc->drc_should_save = B_TRUE; 3342 3343 (void) bqueue_init(&rwa->q, zfs_recv_queue_ff, 3344 MAX(zfs_recv_queue_length, 2 * zfs_max_recordsize), 3345 offsetof(struct receive_record_arg, node)); 3346 cv_init(&rwa->cv, NULL, CV_DEFAULT, NULL); 3347 mutex_init(&rwa->mutex, NULL, MUTEX_DEFAULT, NULL); 3348 rwa->os = drc->drc_os; 3349 rwa->byteswap = drc->drc_byteswap; 3350 rwa->heal = drc->drc_heal; 3351 rwa->tofs = drc->drc_tofs; 3352 rwa->resumable = drc->drc_resumable; 3353 rwa->raw = drc->drc_raw; 3354 rwa->spill = drc->drc_spill; 3355 rwa->full = (drc->drc_drr_begin->drr_u.drr_begin.drr_fromguid == 0); 3356 rwa->os->os_raw_receive = drc->drc_raw; 3357 if (drc->drc_heal) { 3358 rwa->heal_pio = zio_root(drc->drc_os->os_spa, NULL, NULL, 3359 ZIO_FLAG_GODFATHER); 3360 } 3361 list_create(&rwa->write_batch, sizeof (struct receive_record_arg), 3362 offsetof(struct receive_record_arg, node.bqn_node)); 3363 3364 (void) thread_create(NULL, 0, receive_writer_thread, rwa, 0, curproc, 3365 TS_RUN, minclsyspri); 3366 /* 3367 * We're reading rwa->err without locks, which is safe since we are the 3368 * only reader, and the worker thread is the only writer. It's ok if we 3369 * miss a write for an iteration or two of the loop, since the writer 3370 * thread will keep freeing records we send it until we send it an eos 3371 * marker. 3372 * 3373 * We can leave this loop in 3 ways: First, if rwa->err is 3374 * non-zero. In that case, the writer thread will free the rrd we just 3375 * pushed. Second, if we're interrupted; in that case, either it's the 3376 * first loop and drc->drc_rrd was never allocated, or it's later, and 3377 * drc->drc_rrd has been handed off to the writer thread who will free 3378 * it. Finally, if receive_read_record fails or we're at the end of the 3379 * stream, then we free drc->drc_rrd and exit. 3380 */ 3381 while (rwa->err == 0) { 3382 if (issig(JUSTLOOKING) && issig(FORREAL)) { 3383 err = SET_ERROR(EINTR); 3384 break; 3385 } 3386 3387 ASSERT3P(drc->drc_rrd, ==, NULL); 3388 drc->drc_rrd = drc->drc_next_rrd; 3389 drc->drc_next_rrd = NULL; 3390 /* Allocates and loads header into drc->drc_next_rrd */ 3391 err = receive_read_record(drc); 3392 3393 if (drc->drc_rrd->header.drr_type == DRR_END || err != 0) { 3394 kmem_free(drc->drc_rrd, sizeof (*drc->drc_rrd)); 3395 drc->drc_rrd = NULL; 3396 break; 3397 } 3398 3399 bqueue_enqueue(&rwa->q, drc->drc_rrd, 3400 sizeof (struct receive_record_arg) + 3401 drc->drc_rrd->payload_size); 3402 drc->drc_rrd = NULL; 3403 } 3404 3405 ASSERT3P(drc->drc_rrd, ==, NULL); 3406 drc->drc_rrd = kmem_zalloc(sizeof (*drc->drc_rrd), KM_SLEEP); 3407 drc->drc_rrd->eos_marker = B_TRUE; 3408 bqueue_enqueue_flush(&rwa->q, drc->drc_rrd, 1); 3409 3410 mutex_enter(&rwa->mutex); 3411 while (!rwa->done) { 3412 /* 3413 * We need to use cv_wait_sig() so that any process that may 3414 * be sleeping here can still fork. 3415 */ 3416 (void) cv_wait_sig(&rwa->cv, &rwa->mutex); 3417 } 3418 mutex_exit(&rwa->mutex); 3419 3420 /* 3421 * If we are receiving a full stream as a clone, all object IDs which 3422 * are greater than the maximum ID referenced in the stream are 3423 * by definition unused and must be freed. 3424 */ 3425 if (drc->drc_clone && drc->drc_drrb->drr_fromguid == 0) { 3426 uint64_t obj = rwa->max_object + 1; 3427 int free_err = 0; 3428 int next_err = 0; 3429 3430 while (next_err == 0) { 3431 free_err = dmu_free_long_object(rwa->os, obj); 3432 if (free_err != 0 && free_err != ENOENT) 3433 break; 3434 3435 next_err = dmu_object_next(rwa->os, &obj, FALSE, 0); 3436 } 3437 3438 if (err == 0) { 3439 if (free_err != 0 && free_err != ENOENT) 3440 err = free_err; 3441 else if (next_err != ESRCH) 3442 err = next_err; 3443 } 3444 } 3445 3446 cv_destroy(&rwa->cv); 3447 mutex_destroy(&rwa->mutex); 3448 bqueue_destroy(&rwa->q); 3449 list_destroy(&rwa->write_batch); 3450 if (err == 0) 3451 err = rwa->err; 3452 3453 out: 3454 /* 3455 * If we hit an error before we started the receive_writer_thread 3456 * we need to clean up the next_rrd we create by processing the 3457 * DRR_BEGIN record. 3458 */ 3459 if (drc->drc_next_rrd != NULL) 3460 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 3461 3462 /* 3463 * The objset will be invalidated by dmu_recv_end() when we do 3464 * dsl_dataset_clone_swap_sync_impl(). 3465 */ 3466 drc->drc_os = NULL; 3467 3468 kmem_free(rwa, sizeof (*rwa)); 3469 nvlist_free(drc->drc_begin_nvl); 3470 3471 if (err != 0) { 3472 /* 3473 * Clean up references. If receive is not resumable, 3474 * destroy what we created, so we don't leave it in 3475 * the inconsistent state. 3476 */ 3477 dmu_recv_cleanup_ds(drc); 3478 nvlist_free(drc->drc_keynvl); 3479 } 3480 3481 objlist_destroy(drc->drc_ignore_objlist); 3482 drc->drc_ignore_objlist = NULL; 3483 *voffp = drc->drc_voff; 3484 return (err); 3485 } 3486 3487 static int 3488 dmu_recv_end_check(void *arg, dmu_tx_t *tx) 3489 { 3490 dmu_recv_cookie_t *drc = arg; 3491 dsl_pool_t *dp = dmu_tx_pool(tx); 3492 int error; 3493 3494 ASSERT3P(drc->drc_ds->ds_owner, ==, dmu_recv_tag); 3495 3496 if (drc->drc_heal) { 3497 error = 0; 3498 } else if (!drc->drc_newfs) { 3499 dsl_dataset_t *origin_head; 3500 3501 error = dsl_dataset_hold(dp, drc->drc_tofs, FTAG, &origin_head); 3502 if (error != 0) 3503 return (error); 3504 if (drc->drc_force) { 3505 /* 3506 * We will destroy any snapshots in tofs (i.e. before 3507 * origin_head) that are after the origin (which is 3508 * the snap before drc_ds, because drc_ds can not 3509 * have any snaps of its own). 3510 */ 3511 uint64_t obj; 3512 3513 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj; 3514 while (obj != 3515 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) { 3516 dsl_dataset_t *snap; 3517 error = dsl_dataset_hold_obj(dp, obj, FTAG, 3518 &snap); 3519 if (error != 0) 3520 break; 3521 if (snap->ds_dir != origin_head->ds_dir) 3522 error = SET_ERROR(EINVAL); 3523 if (error == 0) { 3524 error = dsl_destroy_snapshot_check_impl( 3525 snap, B_FALSE); 3526 } 3527 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj; 3528 dsl_dataset_rele(snap, FTAG); 3529 if (error != 0) 3530 break; 3531 } 3532 if (error != 0) { 3533 dsl_dataset_rele(origin_head, FTAG); 3534 return (error); 3535 } 3536 } 3537 if (drc->drc_keynvl != NULL) { 3538 error = dsl_crypto_recv_raw_key_check(drc->drc_ds, 3539 drc->drc_keynvl, tx); 3540 if (error != 0) { 3541 dsl_dataset_rele(origin_head, FTAG); 3542 return (error); 3543 } 3544 } 3545 3546 error = dsl_dataset_clone_swap_check_impl(drc->drc_ds, 3547 origin_head, drc->drc_force, drc->drc_owner, tx); 3548 if (error != 0) { 3549 dsl_dataset_rele(origin_head, FTAG); 3550 return (error); 3551 } 3552 error = dsl_dataset_snapshot_check_impl(origin_head, 3553 drc->drc_tosnap, tx, B_TRUE, 1, 3554 drc->drc_cred, drc->drc_proc); 3555 dsl_dataset_rele(origin_head, FTAG); 3556 if (error != 0) 3557 return (error); 3558 3559 error = dsl_destroy_head_check_impl(drc->drc_ds, 1); 3560 } else { 3561 error = dsl_dataset_snapshot_check_impl(drc->drc_ds, 3562 drc->drc_tosnap, tx, B_TRUE, 1, 3563 drc->drc_cred, drc->drc_proc); 3564 } 3565 return (error); 3566 } 3567 3568 static void 3569 dmu_recv_end_sync(void *arg, dmu_tx_t *tx) 3570 { 3571 dmu_recv_cookie_t *drc = arg; 3572 dsl_pool_t *dp = dmu_tx_pool(tx); 3573 boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0; 3574 uint64_t newsnapobj = 0; 3575 3576 spa_history_log_internal_ds(drc->drc_ds, "finish receiving", 3577 tx, "snap=%s", drc->drc_tosnap); 3578 drc->drc_ds->ds_objset->os_raw_receive = B_FALSE; 3579 3580 if (drc->drc_heal) { 3581 if (drc->drc_keynvl != NULL) { 3582 nvlist_free(drc->drc_keynvl); 3583 drc->drc_keynvl = NULL; 3584 } 3585 } else if (!drc->drc_newfs) { 3586 dsl_dataset_t *origin_head; 3587 3588 VERIFY0(dsl_dataset_hold(dp, drc->drc_tofs, FTAG, 3589 &origin_head)); 3590 3591 if (drc->drc_force) { 3592 /* 3593 * Destroy any snapshots of drc_tofs (origin_head) 3594 * after the origin (the snap before drc_ds). 3595 */ 3596 uint64_t obj; 3597 3598 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj; 3599 while (obj != 3600 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) { 3601 dsl_dataset_t *snap; 3602 VERIFY0(dsl_dataset_hold_obj(dp, obj, FTAG, 3603 &snap)); 3604 ASSERT3P(snap->ds_dir, ==, origin_head->ds_dir); 3605 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj; 3606 dsl_destroy_snapshot_sync_impl(snap, 3607 B_FALSE, tx); 3608 dsl_dataset_rele(snap, FTAG); 3609 } 3610 } 3611 if (drc->drc_keynvl != NULL) { 3612 dsl_crypto_recv_raw_key_sync(drc->drc_ds, 3613 drc->drc_keynvl, tx); 3614 nvlist_free(drc->drc_keynvl); 3615 drc->drc_keynvl = NULL; 3616 } 3617 3618 VERIFY3P(drc->drc_ds->ds_prev, ==, 3619 origin_head->ds_prev); 3620 3621 dsl_dataset_clone_swap_sync_impl(drc->drc_ds, 3622 origin_head, tx); 3623 /* 3624 * The objset was evicted by dsl_dataset_clone_swap_sync_impl, 3625 * so drc_os is no longer valid. 3626 */ 3627 drc->drc_os = NULL; 3628 3629 dsl_dataset_snapshot_sync_impl(origin_head, 3630 drc->drc_tosnap, tx); 3631 3632 /* set snapshot's creation time and guid */ 3633 dmu_buf_will_dirty(origin_head->ds_prev->ds_dbuf, tx); 3634 dsl_dataset_phys(origin_head->ds_prev)->ds_creation_time = 3635 drc->drc_drrb->drr_creation_time; 3636 dsl_dataset_phys(origin_head->ds_prev)->ds_guid = 3637 drc->drc_drrb->drr_toguid; 3638 dsl_dataset_phys(origin_head->ds_prev)->ds_flags &= 3639 ~DS_FLAG_INCONSISTENT; 3640 3641 dmu_buf_will_dirty(origin_head->ds_dbuf, tx); 3642 dsl_dataset_phys(origin_head)->ds_flags &= 3643 ~DS_FLAG_INCONSISTENT; 3644 3645 newsnapobj = 3646 dsl_dataset_phys(origin_head)->ds_prev_snap_obj; 3647 3648 dsl_dataset_rele(origin_head, FTAG); 3649 dsl_destroy_head_sync_impl(drc->drc_ds, tx); 3650 3651 if (drc->drc_owner != NULL) 3652 VERIFY3P(origin_head->ds_owner, ==, drc->drc_owner); 3653 } else { 3654 dsl_dataset_t *ds = drc->drc_ds; 3655 3656 dsl_dataset_snapshot_sync_impl(ds, drc->drc_tosnap, tx); 3657 3658 /* set snapshot's creation time and guid */ 3659 dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx); 3660 dsl_dataset_phys(ds->ds_prev)->ds_creation_time = 3661 drc->drc_drrb->drr_creation_time; 3662 dsl_dataset_phys(ds->ds_prev)->ds_guid = 3663 drc->drc_drrb->drr_toguid; 3664 dsl_dataset_phys(ds->ds_prev)->ds_flags &= 3665 ~DS_FLAG_INCONSISTENT; 3666 3667 dmu_buf_will_dirty(ds->ds_dbuf, tx); 3668 dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT; 3669 if (dsl_dataset_has_resume_receive_state(ds)) { 3670 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3671 DS_FIELD_RESUME_FROMGUID, tx); 3672 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3673 DS_FIELD_RESUME_OBJECT, tx); 3674 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3675 DS_FIELD_RESUME_OFFSET, tx); 3676 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3677 DS_FIELD_RESUME_BYTES, tx); 3678 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3679 DS_FIELD_RESUME_TOGUID, tx); 3680 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3681 DS_FIELD_RESUME_TONAME, tx); 3682 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3683 DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx); 3684 } 3685 newsnapobj = 3686 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj; 3687 } 3688 3689 /* 3690 * If this is a raw receive, the crypt_keydata nvlist will include 3691 * a to_ivset_guid for us to set on the new snapshot. This value 3692 * will override the value generated by the snapshot code. However, 3693 * this value may not be present, because older implementations of 3694 * the raw send code did not include this value, and we are still 3695 * allowed to receive them if the zfs_disable_ivset_guid_check 3696 * tunable is set, in which case we will leave the newly-generated 3697 * value. 3698 */ 3699 if (!drc->drc_heal && drc->drc_raw && drc->drc_ivset_guid != 0) { 3700 dmu_object_zapify(dp->dp_meta_objset, newsnapobj, 3701 DMU_OT_DSL_DATASET, tx); 3702 VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj, 3703 DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1, 3704 &drc->drc_ivset_guid, tx)); 3705 } 3706 3707 /* 3708 * Release the hold from dmu_recv_begin. This must be done before 3709 * we return to open context, so that when we free the dataset's dnode 3710 * we can evict its bonus buffer. Since the dataset may be destroyed 3711 * at this point (and therefore won't have a valid pointer to the spa) 3712 * we release the key mapping manually here while we do have a valid 3713 * pointer, if it exists. 3714 */ 3715 if (!drc->drc_raw && encrypted) { 3716 (void) spa_keystore_remove_mapping(dmu_tx_pool(tx)->dp_spa, 3717 drc->drc_ds->ds_object, drc->drc_ds); 3718 } 3719 dsl_dataset_disown(drc->drc_ds, 0, dmu_recv_tag); 3720 drc->drc_ds = NULL; 3721 } 3722 3723 static int dmu_recv_end_modified_blocks = 3; 3724 3725 static int 3726 dmu_recv_existing_end(dmu_recv_cookie_t *drc) 3727 { 3728 #ifdef _KERNEL 3729 /* 3730 * We will be destroying the ds; make sure its origin is unmounted if 3731 * necessary. 3732 */ 3733 char name[ZFS_MAX_DATASET_NAME_LEN]; 3734 dsl_dataset_name(drc->drc_ds, name); 3735 zfs_destroy_unmount_origin(name); 3736 #endif 3737 3738 return (dsl_sync_task(drc->drc_tofs, 3739 dmu_recv_end_check, dmu_recv_end_sync, drc, 3740 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL)); 3741 } 3742 3743 static int 3744 dmu_recv_new_end(dmu_recv_cookie_t *drc) 3745 { 3746 return (dsl_sync_task(drc->drc_tofs, 3747 dmu_recv_end_check, dmu_recv_end_sync, drc, 3748 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL)); 3749 } 3750 3751 int 3752 dmu_recv_end(dmu_recv_cookie_t *drc, void *owner) 3753 { 3754 int error; 3755 3756 drc->drc_owner = owner; 3757 3758 if (drc->drc_newfs) 3759 error = dmu_recv_new_end(drc); 3760 else 3761 error = dmu_recv_existing_end(drc); 3762 3763 if (error != 0) { 3764 dmu_recv_cleanup_ds(drc); 3765 nvlist_free(drc->drc_keynvl); 3766 } else if (!drc->drc_heal) { 3767 if (drc->drc_newfs) { 3768 zvol_create_minor(drc->drc_tofs); 3769 } 3770 char *snapname = kmem_asprintf("%s@%s", 3771 drc->drc_tofs, drc->drc_tosnap); 3772 zvol_create_minor(snapname); 3773 kmem_strfree(snapname); 3774 } 3775 return (error); 3776 } 3777 3778 /* 3779 * Return TRUE if this objset is currently being received into. 3780 */ 3781 boolean_t 3782 dmu_objset_is_receiving(objset_t *os) 3783 { 3784 return (os->os_dsl_dataset != NULL && 3785 os->os_dsl_dataset->ds_owner == dmu_recv_tag); 3786 } 3787 3788 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_length, UINT, ZMOD_RW, 3789 "Maximum receive queue length"); 3790 3791 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_ff, UINT, ZMOD_RW, 3792 "Receive queue fill fraction"); 3793 3794 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, write_batch_size, UINT, ZMOD_RW, 3795 "Maximum amount of writes to batch into one transaction"); 3796 3797 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, best_effort_corrective, INT, ZMOD_RW, 3798 "Ignore errors during corrective receive"); 3799 /* END CSTYLED */ 3800