1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2014, Joyent, Inc. All rights reserved. 26 * Copyright 2014 HybridCluster. All rights reserved. 27 * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved. 28 * Copyright (c) 2019, Klara Inc. 29 * Copyright (c) 2019, Allan Jude 30 * Copyright (c) 2019 Datto Inc. 31 * Copyright (c) 2022 Axcient. 32 */ 33 34 #include <sys/arc.h> 35 #include <sys/spa_impl.h> 36 #include <sys/dmu.h> 37 #include <sys/dmu_impl.h> 38 #include <sys/dmu_send.h> 39 #include <sys/dmu_recv.h> 40 #include <sys/dmu_tx.h> 41 #include <sys/dbuf.h> 42 #include <sys/dnode.h> 43 #include <sys/zfs_context.h> 44 #include <sys/dmu_objset.h> 45 #include <sys/dmu_traverse.h> 46 #include <sys/dsl_dataset.h> 47 #include <sys/dsl_dir.h> 48 #include <sys/dsl_prop.h> 49 #include <sys/dsl_pool.h> 50 #include <sys/dsl_synctask.h> 51 #include <sys/zfs_ioctl.h> 52 #include <sys/zap.h> 53 #include <sys/zvol.h> 54 #include <sys/zio_checksum.h> 55 #include <sys/zfs_znode.h> 56 #include <zfs_fletcher.h> 57 #include <sys/avl.h> 58 #include <sys/ddt.h> 59 #include <sys/zfs_onexit.h> 60 #include <sys/dsl_destroy.h> 61 #include <sys/blkptr.h> 62 #include <sys/dsl_bookmark.h> 63 #include <sys/zfeature.h> 64 #include <sys/bqueue.h> 65 #include <sys/objlist.h> 66 #ifdef _KERNEL 67 #include <sys/zfs_vfsops.h> 68 #endif 69 #include <sys/zfs_file.h> 70 71 static uint_t zfs_recv_queue_length = SPA_MAXBLOCKSIZE; 72 static uint_t zfs_recv_queue_ff = 20; 73 static uint_t zfs_recv_write_batch_size = 1024 * 1024; 74 static int zfs_recv_best_effort_corrective = 0; 75 76 static const void *const dmu_recv_tag = "dmu_recv_tag"; 77 const char *const recv_clone_name = "%recv"; 78 79 typedef enum { 80 ORNS_NO, 81 ORNS_YES, 82 ORNS_MAYBE 83 } or_need_sync_t; 84 85 static int receive_read_payload_and_next_header(dmu_recv_cookie_t *ra, int len, 86 void *buf); 87 88 struct receive_record_arg { 89 dmu_replay_record_t header; 90 void *payload; /* Pointer to a buffer containing the payload */ 91 /* 92 * If the record is a WRITE or SPILL, pointer to the abd containing the 93 * payload. 94 */ 95 abd_t *abd; 96 int payload_size; 97 uint64_t bytes_read; /* bytes read from stream when record created */ 98 boolean_t eos_marker; /* Marks the end of the stream */ 99 bqueue_node_t node; 100 }; 101 102 struct receive_writer_arg { 103 objset_t *os; 104 boolean_t byteswap; 105 bqueue_t q; 106 107 /* 108 * These three members are used to signal to the main thread when 109 * we're done. 110 */ 111 kmutex_t mutex; 112 kcondvar_t cv; 113 boolean_t done; 114 115 int err; 116 const char *tofs; 117 boolean_t heal; 118 boolean_t resumable; 119 boolean_t raw; /* DMU_BACKUP_FEATURE_RAW set */ 120 boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */ 121 boolean_t full; /* this is a full send stream */ 122 uint64_t last_object; 123 uint64_t last_offset; 124 uint64_t max_object; /* highest object ID referenced in stream */ 125 uint64_t bytes_read; /* bytes read when current record created */ 126 127 list_t write_batch; 128 129 /* Encryption parameters for the last received DRR_OBJECT_RANGE */ 130 boolean_t or_crypt_params_present; 131 uint64_t or_firstobj; 132 uint64_t or_numslots; 133 uint8_t or_salt[ZIO_DATA_SALT_LEN]; 134 uint8_t or_iv[ZIO_DATA_IV_LEN]; 135 uint8_t or_mac[ZIO_DATA_MAC_LEN]; 136 boolean_t or_byteorder; 137 zio_t *heal_pio; 138 139 /* Keep track of DRR_FREEOBJECTS right after DRR_OBJECT_RANGE */ 140 or_need_sync_t or_need_sync; 141 }; 142 143 typedef struct dmu_recv_begin_arg { 144 const char *drba_origin; 145 dmu_recv_cookie_t *drba_cookie; 146 cred_t *drba_cred; 147 proc_t *drba_proc; 148 dsl_crypto_params_t *drba_dcp; 149 } dmu_recv_begin_arg_t; 150 151 static void 152 byteswap_record(dmu_replay_record_t *drr) 153 { 154 #define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X)) 155 #define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X)) 156 drr->drr_type = BSWAP_32(drr->drr_type); 157 drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen); 158 159 switch (drr->drr_type) { 160 case DRR_BEGIN: 161 DO64(drr_begin.drr_magic); 162 DO64(drr_begin.drr_versioninfo); 163 DO64(drr_begin.drr_creation_time); 164 DO32(drr_begin.drr_type); 165 DO32(drr_begin.drr_flags); 166 DO64(drr_begin.drr_toguid); 167 DO64(drr_begin.drr_fromguid); 168 break; 169 case DRR_OBJECT: 170 DO64(drr_object.drr_object); 171 DO32(drr_object.drr_type); 172 DO32(drr_object.drr_bonustype); 173 DO32(drr_object.drr_blksz); 174 DO32(drr_object.drr_bonuslen); 175 DO32(drr_object.drr_raw_bonuslen); 176 DO64(drr_object.drr_toguid); 177 DO64(drr_object.drr_maxblkid); 178 break; 179 case DRR_FREEOBJECTS: 180 DO64(drr_freeobjects.drr_firstobj); 181 DO64(drr_freeobjects.drr_numobjs); 182 DO64(drr_freeobjects.drr_toguid); 183 break; 184 case DRR_WRITE: 185 DO64(drr_write.drr_object); 186 DO32(drr_write.drr_type); 187 DO64(drr_write.drr_offset); 188 DO64(drr_write.drr_logical_size); 189 DO64(drr_write.drr_toguid); 190 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum); 191 DO64(drr_write.drr_key.ddk_prop); 192 DO64(drr_write.drr_compressed_size); 193 break; 194 case DRR_WRITE_EMBEDDED: 195 DO64(drr_write_embedded.drr_object); 196 DO64(drr_write_embedded.drr_offset); 197 DO64(drr_write_embedded.drr_length); 198 DO64(drr_write_embedded.drr_toguid); 199 DO32(drr_write_embedded.drr_lsize); 200 DO32(drr_write_embedded.drr_psize); 201 break; 202 case DRR_FREE: 203 DO64(drr_free.drr_object); 204 DO64(drr_free.drr_offset); 205 DO64(drr_free.drr_length); 206 DO64(drr_free.drr_toguid); 207 break; 208 case DRR_SPILL: 209 DO64(drr_spill.drr_object); 210 DO64(drr_spill.drr_length); 211 DO64(drr_spill.drr_toguid); 212 DO64(drr_spill.drr_compressed_size); 213 DO32(drr_spill.drr_type); 214 break; 215 case DRR_OBJECT_RANGE: 216 DO64(drr_object_range.drr_firstobj); 217 DO64(drr_object_range.drr_numslots); 218 DO64(drr_object_range.drr_toguid); 219 break; 220 case DRR_REDACT: 221 DO64(drr_redact.drr_object); 222 DO64(drr_redact.drr_offset); 223 DO64(drr_redact.drr_length); 224 DO64(drr_redact.drr_toguid); 225 break; 226 case DRR_END: 227 DO64(drr_end.drr_toguid); 228 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum); 229 break; 230 default: 231 break; 232 } 233 234 if (drr->drr_type != DRR_BEGIN) { 235 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum); 236 } 237 238 #undef DO64 239 #undef DO32 240 } 241 242 static boolean_t 243 redact_snaps_contains(uint64_t *snaps, uint64_t num_snaps, uint64_t guid) 244 { 245 for (int i = 0; i < num_snaps; i++) { 246 if (snaps[i] == guid) 247 return (B_TRUE); 248 } 249 return (B_FALSE); 250 } 251 252 /* 253 * Check that the new stream we're trying to receive is redacted with respect to 254 * a subset of the snapshots that the origin was redacted with respect to. For 255 * the reasons behind this, see the man page on redacted zfs sends and receives. 256 */ 257 static boolean_t 258 compatible_redact_snaps(uint64_t *origin_snaps, uint64_t origin_num_snaps, 259 uint64_t *redact_snaps, uint64_t num_redact_snaps) 260 { 261 /* 262 * Short circuit the comparison; if we are redacted with respect to 263 * more snapshots than the origin, we can't be redacted with respect 264 * to a subset. 265 */ 266 if (num_redact_snaps > origin_num_snaps) { 267 return (B_FALSE); 268 } 269 270 for (int i = 0; i < num_redact_snaps; i++) { 271 if (!redact_snaps_contains(origin_snaps, origin_num_snaps, 272 redact_snaps[i])) { 273 return (B_FALSE); 274 } 275 } 276 return (B_TRUE); 277 } 278 279 static boolean_t 280 redact_check(dmu_recv_begin_arg_t *drba, dsl_dataset_t *origin) 281 { 282 uint64_t *origin_snaps; 283 uint64_t origin_num_snaps; 284 dmu_recv_cookie_t *drc = drba->drba_cookie; 285 struct drr_begin *drrb = drc->drc_drrb; 286 int featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); 287 int err = 0; 288 boolean_t ret = B_TRUE; 289 uint64_t *redact_snaps; 290 uint_t numredactsnaps; 291 292 /* 293 * If this is a full send stream, we're safe no matter what. 294 */ 295 if (drrb->drr_fromguid == 0) 296 return (ret); 297 298 VERIFY(dsl_dataset_get_uint64_array_feature(origin, 299 SPA_FEATURE_REDACTED_DATASETS, &origin_num_snaps, &origin_snaps)); 300 301 if (nvlist_lookup_uint64_array(drc->drc_begin_nvl, 302 BEGINNV_REDACT_FROM_SNAPS, &redact_snaps, &numredactsnaps) == 303 0) { 304 /* 305 * If the send stream was sent from the redaction bookmark or 306 * the redacted version of the dataset, then we're safe. Verify 307 * that this is from the a compatible redaction bookmark or 308 * redacted dataset. 309 */ 310 if (!compatible_redact_snaps(origin_snaps, origin_num_snaps, 311 redact_snaps, numredactsnaps)) { 312 err = EINVAL; 313 } 314 } else if (featureflags & DMU_BACKUP_FEATURE_REDACTED) { 315 /* 316 * If the stream is redacted, it must be redacted with respect 317 * to a subset of what the origin is redacted with respect to. 318 * See case number 2 in the zfs man page section on redacted zfs 319 * send. 320 */ 321 err = nvlist_lookup_uint64_array(drc->drc_begin_nvl, 322 BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps); 323 324 if (err != 0 || !compatible_redact_snaps(origin_snaps, 325 origin_num_snaps, redact_snaps, numredactsnaps)) { 326 err = EINVAL; 327 } 328 } else if (!redact_snaps_contains(origin_snaps, origin_num_snaps, 329 drrb->drr_toguid)) { 330 /* 331 * If the stream isn't redacted but the origin is, this must be 332 * one of the snapshots the origin is redacted with respect to. 333 * See case number 1 in the zfs man page section on redacted zfs 334 * send. 335 */ 336 err = EINVAL; 337 } 338 339 if (err != 0) 340 ret = B_FALSE; 341 return (ret); 342 } 343 344 /* 345 * If we previously received a stream with --large-block, we don't support 346 * receiving an incremental on top of it without --large-block. This avoids 347 * forcing a read-modify-write or trying to re-aggregate a string of WRITE 348 * records. 349 */ 350 static int 351 recv_check_large_blocks(dsl_dataset_t *ds, uint64_t featureflags) 352 { 353 if (dsl_dataset_feature_is_active(ds, SPA_FEATURE_LARGE_BLOCKS) && 354 !(featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS)) 355 return (SET_ERROR(ZFS_ERR_STREAM_LARGE_BLOCK_MISMATCH)); 356 return (0); 357 } 358 359 static int 360 recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds, 361 uint64_t fromguid, uint64_t featureflags) 362 { 363 uint64_t obj; 364 uint64_t children; 365 int error; 366 dsl_dataset_t *snap; 367 dsl_pool_t *dp = ds->ds_dir->dd_pool; 368 boolean_t encrypted = ds->ds_dir->dd_crypto_obj != 0; 369 boolean_t raw = (featureflags & DMU_BACKUP_FEATURE_RAW) != 0; 370 boolean_t embed = (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) != 0; 371 372 /* Temporary clone name must not exist. */ 373 error = zap_lookup(dp->dp_meta_objset, 374 dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, recv_clone_name, 375 8, 1, &obj); 376 if (error != ENOENT) 377 return (error == 0 ? SET_ERROR(EBUSY) : error); 378 379 /* Resume state must not be set. */ 380 if (dsl_dataset_has_resume_receive_state(ds)) 381 return (SET_ERROR(EBUSY)); 382 383 /* New snapshot name must not exist if we're not healing it. */ 384 error = zap_lookup(dp->dp_meta_objset, 385 dsl_dataset_phys(ds)->ds_snapnames_zapobj, 386 drba->drba_cookie->drc_tosnap, 8, 1, &obj); 387 if (drba->drba_cookie->drc_heal) { 388 if (error != 0) 389 return (error); 390 } else if (error != ENOENT) { 391 return (error == 0 ? SET_ERROR(EEXIST) : error); 392 } 393 394 /* Must not have children if receiving a ZVOL. */ 395 error = zap_count(dp->dp_meta_objset, 396 dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, &children); 397 if (error != 0) 398 return (error); 399 if (drba->drba_cookie->drc_drrb->drr_type != DMU_OST_ZFS && 400 children > 0) 401 return (SET_ERROR(ZFS_ERR_WRONG_PARENT)); 402 403 /* 404 * Check snapshot limit before receiving. We'll recheck again at the 405 * end, but might as well abort before receiving if we're already over 406 * the limit. 407 * 408 * Note that we do not check the file system limit with 409 * dsl_dir_fscount_check because the temporary %clones don't count 410 * against that limit. 411 */ 412 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, ZFS_PROP_SNAPSHOT_LIMIT, 413 NULL, drba->drba_cred, drba->drba_proc); 414 if (error != 0) 415 return (error); 416 417 if (drba->drba_cookie->drc_heal) { 418 /* Encryption is incompatible with embedded data. */ 419 if (encrypted && embed) 420 return (SET_ERROR(EINVAL)); 421 422 /* Healing is not supported when in 'force' mode. */ 423 if (drba->drba_cookie->drc_force) 424 return (SET_ERROR(EINVAL)); 425 426 /* Must have keys loaded if doing encrypted non-raw recv. */ 427 if (encrypted && !raw) { 428 if (spa_keystore_lookup_key(dp->dp_spa, ds->ds_object, 429 NULL, NULL) != 0) 430 return (SET_ERROR(EACCES)); 431 } 432 433 error = dsl_dataset_hold_obj(dp, obj, FTAG, &snap); 434 if (error != 0) 435 return (error); 436 437 /* 438 * When not doing best effort corrective recv healing can only 439 * be done if the send stream is for the same snapshot as the 440 * one we are trying to heal. 441 */ 442 if (zfs_recv_best_effort_corrective == 0 && 443 drba->drba_cookie->drc_drrb->drr_toguid != 444 dsl_dataset_phys(snap)->ds_guid) { 445 dsl_dataset_rele(snap, FTAG); 446 return (SET_ERROR(ENOTSUP)); 447 } 448 dsl_dataset_rele(snap, FTAG); 449 } else if (fromguid != 0) { 450 /* Sanity check the incremental recv */ 451 uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj; 452 453 /* Can't perform a raw receive on top of a non-raw receive */ 454 if (!encrypted && raw) 455 return (SET_ERROR(EINVAL)); 456 457 /* Encryption is incompatible with embedded data */ 458 if (encrypted && embed) 459 return (SET_ERROR(EINVAL)); 460 461 /* Find snapshot in this dir that matches fromguid. */ 462 while (obj != 0) { 463 error = dsl_dataset_hold_obj(dp, obj, FTAG, 464 &snap); 465 if (error != 0) 466 return (SET_ERROR(ENODEV)); 467 if (snap->ds_dir != ds->ds_dir) { 468 dsl_dataset_rele(snap, FTAG); 469 return (SET_ERROR(ENODEV)); 470 } 471 if (dsl_dataset_phys(snap)->ds_guid == fromguid) 472 break; 473 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj; 474 dsl_dataset_rele(snap, FTAG); 475 } 476 if (obj == 0) 477 return (SET_ERROR(ENODEV)); 478 479 if (drba->drba_cookie->drc_force) { 480 drba->drba_cookie->drc_fromsnapobj = obj; 481 } else { 482 /* 483 * If we are not forcing, there must be no 484 * changes since fromsnap. Raw sends have an 485 * additional constraint that requires that 486 * no "noop" snapshots exist between fromsnap 487 * and tosnap for the IVset checking code to 488 * work properly. 489 */ 490 if (dsl_dataset_modified_since_snap(ds, snap) || 491 (raw && 492 dsl_dataset_phys(ds)->ds_prev_snap_obj != 493 snap->ds_object)) { 494 dsl_dataset_rele(snap, FTAG); 495 return (SET_ERROR(ETXTBSY)); 496 } 497 drba->drba_cookie->drc_fromsnapobj = 498 ds->ds_prev->ds_object; 499 } 500 501 if (dsl_dataset_feature_is_active(snap, 502 SPA_FEATURE_REDACTED_DATASETS) && !redact_check(drba, 503 snap)) { 504 dsl_dataset_rele(snap, FTAG); 505 return (SET_ERROR(EINVAL)); 506 } 507 508 error = recv_check_large_blocks(snap, featureflags); 509 if (error != 0) { 510 dsl_dataset_rele(snap, FTAG); 511 return (error); 512 } 513 514 dsl_dataset_rele(snap, FTAG); 515 } else { 516 /* If full and not healing then must be forced. */ 517 if (!drba->drba_cookie->drc_force) 518 return (SET_ERROR(EEXIST)); 519 520 /* 521 * We don't support using zfs recv -F to blow away 522 * encrypted filesystems. This would require the 523 * dsl dir to point to the old encryption key and 524 * the new one at the same time during the receive. 525 */ 526 if ((!encrypted && raw) || encrypted) 527 return (SET_ERROR(EINVAL)); 528 529 /* 530 * Perform the same encryption checks we would if 531 * we were creating a new dataset from scratch. 532 */ 533 if (!raw) { 534 boolean_t will_encrypt; 535 536 error = dmu_objset_create_crypt_check( 537 ds->ds_dir->dd_parent, drba->drba_dcp, 538 &will_encrypt); 539 if (error != 0) 540 return (error); 541 542 if (will_encrypt && embed) 543 return (SET_ERROR(EINVAL)); 544 } 545 } 546 547 return (0); 548 } 549 550 /* 551 * Check that any feature flags used in the data stream we're receiving are 552 * supported by the pool we are receiving into. 553 * 554 * Note that some of the features we explicitly check here have additional 555 * (implicit) features they depend on, but those dependencies are enforced 556 * through the zfeature_register() calls declaring the features that we 557 * explicitly check. 558 */ 559 static int 560 recv_begin_check_feature_flags_impl(uint64_t featureflags, spa_t *spa) 561 { 562 /* 563 * Check if there are any unsupported feature flags. 564 */ 565 if (!DMU_STREAM_SUPPORTED(featureflags)) { 566 return (SET_ERROR(ZFS_ERR_UNKNOWN_SEND_STREAM_FEATURE)); 567 } 568 569 /* Verify pool version supports SA if SA_SPILL feature set */ 570 if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) && 571 spa_version(spa) < SPA_VERSION_SA) 572 return (SET_ERROR(ENOTSUP)); 573 574 /* 575 * LZ4 compressed, ZSTD compressed, embedded, mooched, large blocks, 576 * and large_dnodes in the stream can only be used if those pool 577 * features are enabled because we don't attempt to decompress / 578 * un-embed / un-mooch / split up the blocks / dnodes during the 579 * receive process. 580 */ 581 if ((featureflags & DMU_BACKUP_FEATURE_LZ4) && 582 !spa_feature_is_enabled(spa, SPA_FEATURE_LZ4_COMPRESS)) 583 return (SET_ERROR(ENOTSUP)); 584 if ((featureflags & DMU_BACKUP_FEATURE_ZSTD) && 585 !spa_feature_is_enabled(spa, SPA_FEATURE_ZSTD_COMPRESS)) 586 return (SET_ERROR(ENOTSUP)); 587 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) && 588 !spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) 589 return (SET_ERROR(ENOTSUP)); 590 if ((featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) && 591 !spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) 592 return (SET_ERROR(ENOTSUP)); 593 if ((featureflags & DMU_BACKUP_FEATURE_LARGE_DNODE) && 594 !spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) 595 return (SET_ERROR(ENOTSUP)); 596 597 /* 598 * Receiving redacted streams requires that redacted datasets are 599 * enabled. 600 */ 601 if ((featureflags & DMU_BACKUP_FEATURE_REDACTED) && 602 !spa_feature_is_enabled(spa, SPA_FEATURE_REDACTED_DATASETS)) 603 return (SET_ERROR(ENOTSUP)); 604 605 return (0); 606 } 607 608 static int 609 dmu_recv_begin_check(void *arg, dmu_tx_t *tx) 610 { 611 dmu_recv_begin_arg_t *drba = arg; 612 dsl_pool_t *dp = dmu_tx_pool(tx); 613 struct drr_begin *drrb = drba->drba_cookie->drc_drrb; 614 uint64_t fromguid = drrb->drr_fromguid; 615 int flags = drrb->drr_flags; 616 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 617 int error; 618 uint64_t featureflags = drba->drba_cookie->drc_featureflags; 619 dsl_dataset_t *ds; 620 const char *tofs = drba->drba_cookie->drc_tofs; 621 622 /* already checked */ 623 ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); 624 ASSERT(!(featureflags & DMU_BACKUP_FEATURE_RESUMING)); 625 626 if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == 627 DMU_COMPOUNDSTREAM || 628 drrb->drr_type >= DMU_OST_NUMTYPES || 629 ((flags & DRR_FLAG_CLONE) && drba->drba_origin == NULL)) 630 return (SET_ERROR(EINVAL)); 631 632 error = recv_begin_check_feature_flags_impl(featureflags, dp->dp_spa); 633 if (error != 0) 634 return (error); 635 636 /* Resumable receives require extensible datasets */ 637 if (drba->drba_cookie->drc_resumable && 638 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EXTENSIBLE_DATASET)) 639 return (SET_ERROR(ENOTSUP)); 640 641 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 642 /* raw receives require the encryption feature */ 643 if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION)) 644 return (SET_ERROR(ENOTSUP)); 645 646 /* embedded data is incompatible with encryption and raw recv */ 647 if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) 648 return (SET_ERROR(EINVAL)); 649 650 /* raw receives require spill block allocation flag */ 651 if (!(flags & DRR_FLAG_SPILL_BLOCK)) 652 return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING)); 653 } else { 654 /* 655 * We support unencrypted datasets below encrypted ones now, 656 * so add the DS_HOLD_FLAG_DECRYPT flag only if we are dealing 657 * with a dataset we may encrypt. 658 */ 659 if (drba->drba_dcp == NULL || 660 drba->drba_dcp->cp_crypt != ZIO_CRYPT_OFF) { 661 dsflags |= DS_HOLD_FLAG_DECRYPT; 662 } 663 } 664 665 error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds); 666 if (error == 0) { 667 /* target fs already exists; recv into temp clone */ 668 669 /* Can't recv a clone into an existing fs */ 670 if (flags & DRR_FLAG_CLONE || drba->drba_origin) { 671 dsl_dataset_rele_flags(ds, dsflags, FTAG); 672 return (SET_ERROR(EINVAL)); 673 } 674 675 error = recv_begin_check_existing_impl(drba, ds, fromguid, 676 featureflags); 677 dsl_dataset_rele_flags(ds, dsflags, FTAG); 678 } else if (error == ENOENT) { 679 /* target fs does not exist; must be a full backup or clone */ 680 char buf[ZFS_MAX_DATASET_NAME_LEN]; 681 objset_t *os; 682 683 /* healing recv must be done "into" an existing snapshot */ 684 if (drba->drba_cookie->drc_heal == B_TRUE) 685 return (SET_ERROR(ENOTSUP)); 686 687 /* 688 * If it's a non-clone incremental, we are missing the 689 * target fs, so fail the recv. 690 */ 691 if (fromguid != 0 && !((flags & DRR_FLAG_CLONE) || 692 drba->drba_origin)) 693 return (SET_ERROR(ENOENT)); 694 695 /* 696 * If we're receiving a full send as a clone, and it doesn't 697 * contain all the necessary free records and freeobject 698 * records, reject it. 699 */ 700 if (fromguid == 0 && drba->drba_origin != NULL && 701 !(flags & DRR_FLAG_FREERECORDS)) 702 return (SET_ERROR(EINVAL)); 703 704 /* Open the parent of tofs */ 705 ASSERT3U(strlen(tofs), <, sizeof (buf)); 706 (void) strlcpy(buf, tofs, strrchr(tofs, '/') - tofs + 1); 707 error = dsl_dataset_hold(dp, buf, FTAG, &ds); 708 if (error != 0) 709 return (error); 710 711 if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0 && 712 drba->drba_origin == NULL) { 713 boolean_t will_encrypt; 714 715 /* 716 * Check that we aren't breaking any encryption rules 717 * and that we have all the parameters we need to 718 * create an encrypted dataset if necessary. If we are 719 * making an encrypted dataset the stream can't have 720 * embedded data. 721 */ 722 error = dmu_objset_create_crypt_check(ds->ds_dir, 723 drba->drba_dcp, &will_encrypt); 724 if (error != 0) { 725 dsl_dataset_rele(ds, FTAG); 726 return (error); 727 } 728 729 if (will_encrypt && 730 (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) { 731 dsl_dataset_rele(ds, FTAG); 732 return (SET_ERROR(EINVAL)); 733 } 734 } 735 736 /* 737 * Check filesystem and snapshot limits before receiving. We'll 738 * recheck snapshot limits again at the end (we create the 739 * filesystems and increment those counts during begin_sync). 740 */ 741 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, 742 ZFS_PROP_FILESYSTEM_LIMIT, NULL, 743 drba->drba_cred, drba->drba_proc); 744 if (error != 0) { 745 dsl_dataset_rele(ds, FTAG); 746 return (error); 747 } 748 749 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, 750 ZFS_PROP_SNAPSHOT_LIMIT, NULL, 751 drba->drba_cred, drba->drba_proc); 752 if (error != 0) { 753 dsl_dataset_rele(ds, FTAG); 754 return (error); 755 } 756 757 /* can't recv below anything but filesystems (eg. no ZVOLs) */ 758 error = dmu_objset_from_ds(ds, &os); 759 if (error != 0) { 760 dsl_dataset_rele(ds, FTAG); 761 return (error); 762 } 763 if (dmu_objset_type(os) != DMU_OST_ZFS) { 764 dsl_dataset_rele(ds, FTAG); 765 return (SET_ERROR(ZFS_ERR_WRONG_PARENT)); 766 } 767 768 if (drba->drba_origin != NULL) { 769 dsl_dataset_t *origin; 770 error = dsl_dataset_hold_flags(dp, drba->drba_origin, 771 dsflags, FTAG, &origin); 772 if (error != 0) { 773 dsl_dataset_rele(ds, FTAG); 774 return (error); 775 } 776 if (!origin->ds_is_snapshot) { 777 dsl_dataset_rele_flags(origin, dsflags, FTAG); 778 dsl_dataset_rele(ds, FTAG); 779 return (SET_ERROR(EINVAL)); 780 } 781 if (dsl_dataset_phys(origin)->ds_guid != fromguid && 782 fromguid != 0) { 783 dsl_dataset_rele_flags(origin, dsflags, FTAG); 784 dsl_dataset_rele(ds, FTAG); 785 return (SET_ERROR(ENODEV)); 786 } 787 788 if (origin->ds_dir->dd_crypto_obj != 0 && 789 (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) { 790 dsl_dataset_rele_flags(origin, dsflags, FTAG); 791 dsl_dataset_rele(ds, FTAG); 792 return (SET_ERROR(EINVAL)); 793 } 794 795 /* 796 * If the origin is redacted we need to verify that this 797 * send stream can safely be received on top of the 798 * origin. 799 */ 800 if (dsl_dataset_feature_is_active(origin, 801 SPA_FEATURE_REDACTED_DATASETS)) { 802 if (!redact_check(drba, origin)) { 803 dsl_dataset_rele_flags(origin, dsflags, 804 FTAG); 805 dsl_dataset_rele_flags(ds, dsflags, 806 FTAG); 807 return (SET_ERROR(EINVAL)); 808 } 809 } 810 811 error = recv_check_large_blocks(ds, featureflags); 812 if (error != 0) { 813 dsl_dataset_rele_flags(origin, dsflags, FTAG); 814 dsl_dataset_rele_flags(ds, dsflags, FTAG); 815 return (error); 816 } 817 818 dsl_dataset_rele_flags(origin, dsflags, FTAG); 819 } 820 821 dsl_dataset_rele(ds, FTAG); 822 error = 0; 823 } 824 return (error); 825 } 826 827 static void 828 dmu_recv_begin_sync(void *arg, dmu_tx_t *tx) 829 { 830 dmu_recv_begin_arg_t *drba = arg; 831 dsl_pool_t *dp = dmu_tx_pool(tx); 832 objset_t *mos = dp->dp_meta_objset; 833 dmu_recv_cookie_t *drc = drba->drba_cookie; 834 struct drr_begin *drrb = drc->drc_drrb; 835 const char *tofs = drc->drc_tofs; 836 uint64_t featureflags = drc->drc_featureflags; 837 dsl_dataset_t *ds, *newds; 838 objset_t *os; 839 uint64_t dsobj; 840 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 841 int error; 842 uint64_t crflags = 0; 843 dsl_crypto_params_t dummy_dcp = { 0 }; 844 dsl_crypto_params_t *dcp = drba->drba_dcp; 845 846 if (drrb->drr_flags & DRR_FLAG_CI_DATA) 847 crflags |= DS_FLAG_CI_DATASET; 848 849 if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0) 850 dsflags |= DS_HOLD_FLAG_DECRYPT; 851 852 /* 853 * Raw, non-incremental recvs always use a dummy dcp with 854 * the raw cmd set. Raw incremental recvs do not use a dcp 855 * since the encryption parameters are already set in stone. 856 */ 857 if (dcp == NULL && drrb->drr_fromguid == 0 && 858 drba->drba_origin == NULL) { 859 ASSERT3P(dcp, ==, NULL); 860 dcp = &dummy_dcp; 861 862 if (featureflags & DMU_BACKUP_FEATURE_RAW) 863 dcp->cp_cmd = DCP_CMD_RAW_RECV; 864 } 865 866 error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds); 867 if (error == 0) { 868 /* Create temporary clone unless we're doing corrective recv */ 869 dsl_dataset_t *snap = NULL; 870 871 if (drba->drba_cookie->drc_fromsnapobj != 0) { 872 VERIFY0(dsl_dataset_hold_obj(dp, 873 drba->drba_cookie->drc_fromsnapobj, FTAG, &snap)); 874 ASSERT3P(dcp, ==, NULL); 875 } 876 if (drc->drc_heal) { 877 /* When healing we want to use the provided snapshot */ 878 VERIFY0(dsl_dataset_snap_lookup(ds, drc->drc_tosnap, 879 &dsobj)); 880 } else { 881 dsobj = dsl_dataset_create_sync(ds->ds_dir, 882 recv_clone_name, snap, crflags, drba->drba_cred, 883 dcp, tx); 884 } 885 if (drba->drba_cookie->drc_fromsnapobj != 0) 886 dsl_dataset_rele(snap, FTAG); 887 dsl_dataset_rele_flags(ds, dsflags, FTAG); 888 } else { 889 dsl_dir_t *dd; 890 const char *tail; 891 dsl_dataset_t *origin = NULL; 892 893 VERIFY0(dsl_dir_hold(dp, tofs, FTAG, &dd, &tail)); 894 895 if (drba->drba_origin != NULL) { 896 VERIFY0(dsl_dataset_hold(dp, drba->drba_origin, 897 FTAG, &origin)); 898 ASSERT3P(dcp, ==, NULL); 899 } 900 901 /* Create new dataset. */ 902 dsobj = dsl_dataset_create_sync(dd, strrchr(tofs, '/') + 1, 903 origin, crflags, drba->drba_cred, dcp, tx); 904 if (origin != NULL) 905 dsl_dataset_rele(origin, FTAG); 906 dsl_dir_rele(dd, FTAG); 907 drc->drc_newfs = B_TRUE; 908 } 909 VERIFY0(dsl_dataset_own_obj_force(dp, dsobj, dsflags, dmu_recv_tag, 910 &newds)); 911 if (dsl_dataset_feature_is_active(newds, 912 SPA_FEATURE_REDACTED_DATASETS)) { 913 /* 914 * If the origin dataset is redacted, the child will be redacted 915 * when we create it. We clear the new dataset's 916 * redaction info; if it should be redacted, we'll fill 917 * in its information later. 918 */ 919 dsl_dataset_deactivate_feature(newds, 920 SPA_FEATURE_REDACTED_DATASETS, tx); 921 } 922 VERIFY0(dmu_objset_from_ds(newds, &os)); 923 924 if (drc->drc_resumable) { 925 dsl_dataset_zapify(newds, tx); 926 if (drrb->drr_fromguid != 0) { 927 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_FROMGUID, 928 8, 1, &drrb->drr_fromguid, tx)); 929 } 930 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TOGUID, 931 8, 1, &drrb->drr_toguid, tx)); 932 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TONAME, 933 1, strlen(drrb->drr_toname) + 1, drrb->drr_toname, tx)); 934 uint64_t one = 1; 935 uint64_t zero = 0; 936 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OBJECT, 937 8, 1, &one, tx)); 938 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OFFSET, 939 8, 1, &zero, tx)); 940 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_BYTES, 941 8, 1, &zero, tx)); 942 if (featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) { 943 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_LARGEBLOCK, 944 8, 1, &one, tx)); 945 } 946 if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) { 947 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_EMBEDOK, 948 8, 1, &one, tx)); 949 } 950 if (featureflags & DMU_BACKUP_FEATURE_COMPRESSED) { 951 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_COMPRESSOK, 952 8, 1, &one, tx)); 953 } 954 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 955 VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_RAWOK, 956 8, 1, &one, tx)); 957 } 958 959 uint64_t *redact_snaps; 960 uint_t numredactsnaps; 961 if (nvlist_lookup_uint64_array(drc->drc_begin_nvl, 962 BEGINNV_REDACT_FROM_SNAPS, &redact_snaps, 963 &numredactsnaps) == 0) { 964 VERIFY0(zap_add(mos, dsobj, 965 DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, 966 sizeof (*redact_snaps), numredactsnaps, 967 redact_snaps, tx)); 968 } 969 } 970 971 /* 972 * Usually the os->os_encrypted value is tied to the presence of a 973 * DSL Crypto Key object in the dd. However, that will not be received 974 * until dmu_recv_stream(), so we set the value manually for now. 975 */ 976 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 977 os->os_encrypted = B_TRUE; 978 drba->drba_cookie->drc_raw = B_TRUE; 979 } 980 981 if (featureflags & DMU_BACKUP_FEATURE_REDACTED) { 982 uint64_t *redact_snaps; 983 uint_t numredactsnaps; 984 VERIFY0(nvlist_lookup_uint64_array(drc->drc_begin_nvl, 985 BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps)); 986 dsl_dataset_activate_redaction(newds, redact_snaps, 987 numredactsnaps, tx); 988 } 989 990 dmu_buf_will_dirty(newds->ds_dbuf, tx); 991 dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT; 992 993 /* 994 * If we actually created a non-clone, we need to create the objset 995 * in our new dataset. If this is a raw send we postpone this until 996 * dmu_recv_stream() so that we can allocate the metadnode with the 997 * properties from the DRR_BEGIN payload. 998 */ 999 rrw_enter(&newds->ds_bp_rwlock, RW_READER, FTAG); 1000 if (BP_IS_HOLE(dsl_dataset_get_blkptr(newds)) && 1001 (featureflags & DMU_BACKUP_FEATURE_RAW) == 0 && 1002 !drc->drc_heal) { 1003 (void) dmu_objset_create_impl(dp->dp_spa, 1004 newds, dsl_dataset_get_blkptr(newds), drrb->drr_type, tx); 1005 } 1006 rrw_exit(&newds->ds_bp_rwlock, FTAG); 1007 1008 drba->drba_cookie->drc_ds = newds; 1009 drba->drba_cookie->drc_os = os; 1010 1011 spa_history_log_internal_ds(newds, "receive", tx, " "); 1012 } 1013 1014 static int 1015 dmu_recv_resume_begin_check(void *arg, dmu_tx_t *tx) 1016 { 1017 dmu_recv_begin_arg_t *drba = arg; 1018 dmu_recv_cookie_t *drc = drba->drba_cookie; 1019 dsl_pool_t *dp = dmu_tx_pool(tx); 1020 struct drr_begin *drrb = drc->drc_drrb; 1021 int error; 1022 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 1023 dsl_dataset_t *ds; 1024 const char *tofs = drc->drc_tofs; 1025 1026 /* already checked */ 1027 ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); 1028 ASSERT(drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING); 1029 1030 if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == 1031 DMU_COMPOUNDSTREAM || 1032 drrb->drr_type >= DMU_OST_NUMTYPES) 1033 return (SET_ERROR(EINVAL)); 1034 1035 /* 1036 * This is mostly a sanity check since we should have already done these 1037 * checks during a previous attempt to receive the data. 1038 */ 1039 error = recv_begin_check_feature_flags_impl(drc->drc_featureflags, 1040 dp->dp_spa); 1041 if (error != 0) 1042 return (error); 1043 1044 /* 6 extra bytes for /%recv */ 1045 char recvname[ZFS_MAX_DATASET_NAME_LEN + 6]; 1046 1047 (void) snprintf(recvname, sizeof (recvname), "%s/%s", 1048 tofs, recv_clone_name); 1049 1050 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) { 1051 /* raw receives require spill block allocation flag */ 1052 if (!(drrb->drr_flags & DRR_FLAG_SPILL_BLOCK)) 1053 return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING)); 1054 } else { 1055 dsflags |= DS_HOLD_FLAG_DECRYPT; 1056 } 1057 1058 boolean_t recvexist = B_TRUE; 1059 if (dsl_dataset_hold_flags(dp, recvname, dsflags, FTAG, &ds) != 0) { 1060 /* %recv does not exist; continue in tofs */ 1061 recvexist = B_FALSE; 1062 error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds); 1063 if (error != 0) 1064 return (error); 1065 } 1066 1067 /* 1068 * Resume of full/newfs recv on existing dataset should be done with 1069 * force flag 1070 */ 1071 if (recvexist && drrb->drr_fromguid == 0 && !drc->drc_force) { 1072 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1073 return (SET_ERROR(ZFS_ERR_RESUME_EXISTS)); 1074 } 1075 1076 /* check that ds is marked inconsistent */ 1077 if (!DS_IS_INCONSISTENT(ds)) { 1078 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1079 return (SET_ERROR(EINVAL)); 1080 } 1081 1082 /* check that there is resuming data, and that the toguid matches */ 1083 if (!dsl_dataset_is_zapified(ds)) { 1084 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1085 return (SET_ERROR(EINVAL)); 1086 } 1087 uint64_t val; 1088 error = zap_lookup(dp->dp_meta_objset, ds->ds_object, 1089 DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val); 1090 if (error != 0 || drrb->drr_toguid != val) { 1091 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1092 return (SET_ERROR(EINVAL)); 1093 } 1094 1095 /* 1096 * Check if the receive is still running. If so, it will be owned. 1097 * Note that nothing else can own the dataset (e.g. after the receive 1098 * fails) because it will be marked inconsistent. 1099 */ 1100 if (dsl_dataset_has_owner(ds)) { 1101 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1102 return (SET_ERROR(EBUSY)); 1103 } 1104 1105 /* There should not be any snapshots of this fs yet. */ 1106 if (ds->ds_prev != NULL && ds->ds_prev->ds_dir == ds->ds_dir) { 1107 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1108 return (SET_ERROR(EINVAL)); 1109 } 1110 1111 /* 1112 * Note: resume point will be checked when we process the first WRITE 1113 * record. 1114 */ 1115 1116 /* check that the origin matches */ 1117 val = 0; 1118 (void) zap_lookup(dp->dp_meta_objset, ds->ds_object, 1119 DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val); 1120 if (drrb->drr_fromguid != val) { 1121 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1122 return (SET_ERROR(EINVAL)); 1123 } 1124 1125 if (ds->ds_prev != NULL && drrb->drr_fromguid != 0) 1126 drc->drc_fromsnapobj = ds->ds_prev->ds_object; 1127 1128 /* 1129 * If we're resuming, and the send is redacted, then the original send 1130 * must have been redacted, and must have been redacted with respect to 1131 * the same snapshots. 1132 */ 1133 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_REDACTED) { 1134 uint64_t num_ds_redact_snaps; 1135 uint64_t *ds_redact_snaps; 1136 1137 uint_t num_stream_redact_snaps; 1138 uint64_t *stream_redact_snaps; 1139 1140 if (nvlist_lookup_uint64_array(drc->drc_begin_nvl, 1141 BEGINNV_REDACT_SNAPS, &stream_redact_snaps, 1142 &num_stream_redact_snaps) != 0) { 1143 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1144 return (SET_ERROR(EINVAL)); 1145 } 1146 1147 if (!dsl_dataset_get_uint64_array_feature(ds, 1148 SPA_FEATURE_REDACTED_DATASETS, &num_ds_redact_snaps, 1149 &ds_redact_snaps)) { 1150 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1151 return (SET_ERROR(EINVAL)); 1152 } 1153 1154 for (int i = 0; i < num_ds_redact_snaps; i++) { 1155 if (!redact_snaps_contains(ds_redact_snaps, 1156 num_ds_redact_snaps, stream_redact_snaps[i])) { 1157 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1158 return (SET_ERROR(EINVAL)); 1159 } 1160 } 1161 } 1162 1163 error = recv_check_large_blocks(ds, drc->drc_featureflags); 1164 if (error != 0) { 1165 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1166 return (error); 1167 } 1168 1169 dsl_dataset_rele_flags(ds, dsflags, FTAG); 1170 return (0); 1171 } 1172 1173 static void 1174 dmu_recv_resume_begin_sync(void *arg, dmu_tx_t *tx) 1175 { 1176 dmu_recv_begin_arg_t *drba = arg; 1177 dsl_pool_t *dp = dmu_tx_pool(tx); 1178 const char *tofs = drba->drba_cookie->drc_tofs; 1179 uint64_t featureflags = drba->drba_cookie->drc_featureflags; 1180 dsl_dataset_t *ds; 1181 ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE; 1182 /* 6 extra bytes for /%recv */ 1183 char recvname[ZFS_MAX_DATASET_NAME_LEN + 6]; 1184 1185 (void) snprintf(recvname, sizeof (recvname), "%s/%s", tofs, 1186 recv_clone_name); 1187 1188 if (featureflags & DMU_BACKUP_FEATURE_RAW) { 1189 drba->drba_cookie->drc_raw = B_TRUE; 1190 } else { 1191 dsflags |= DS_HOLD_FLAG_DECRYPT; 1192 } 1193 1194 if (dsl_dataset_own_force(dp, recvname, dsflags, dmu_recv_tag, &ds) 1195 != 0) { 1196 /* %recv does not exist; continue in tofs */ 1197 VERIFY0(dsl_dataset_own_force(dp, tofs, dsflags, dmu_recv_tag, 1198 &ds)); 1199 drba->drba_cookie->drc_newfs = B_TRUE; 1200 } 1201 1202 ASSERT(DS_IS_INCONSISTENT(ds)); 1203 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 1204 ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)) || 1205 drba->drba_cookie->drc_raw); 1206 rrw_exit(&ds->ds_bp_rwlock, FTAG); 1207 1208 drba->drba_cookie->drc_ds = ds; 1209 VERIFY0(dmu_objset_from_ds(ds, &drba->drba_cookie->drc_os)); 1210 drba->drba_cookie->drc_should_save = B_TRUE; 1211 1212 spa_history_log_internal_ds(ds, "resume receive", tx, " "); 1213 } 1214 1215 /* 1216 * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin() 1217 * succeeds; otherwise we will leak the holds on the datasets. 1218 */ 1219 int 1220 dmu_recv_begin(const char *tofs, const char *tosnap, 1221 dmu_replay_record_t *drr_begin, boolean_t force, boolean_t heal, 1222 boolean_t resumable, nvlist_t *localprops, nvlist_t *hidden_args, 1223 const char *origin, dmu_recv_cookie_t *drc, zfs_file_t *fp, 1224 offset_t *voffp) 1225 { 1226 dmu_recv_begin_arg_t drba = { 0 }; 1227 int err = 0; 1228 1229 memset(drc, 0, sizeof (dmu_recv_cookie_t)); 1230 drc->drc_drr_begin = drr_begin; 1231 drc->drc_drrb = &drr_begin->drr_u.drr_begin; 1232 drc->drc_tosnap = tosnap; 1233 drc->drc_tofs = tofs; 1234 drc->drc_force = force; 1235 drc->drc_heal = heal; 1236 drc->drc_resumable = resumable; 1237 drc->drc_cred = CRED(); 1238 drc->drc_proc = curproc; 1239 drc->drc_clone = (origin != NULL); 1240 1241 if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) { 1242 drc->drc_byteswap = B_TRUE; 1243 (void) fletcher_4_incremental_byteswap(drr_begin, 1244 sizeof (dmu_replay_record_t), &drc->drc_cksum); 1245 byteswap_record(drr_begin); 1246 } else if (drc->drc_drrb->drr_magic == DMU_BACKUP_MAGIC) { 1247 (void) fletcher_4_incremental_native(drr_begin, 1248 sizeof (dmu_replay_record_t), &drc->drc_cksum); 1249 } else { 1250 return (SET_ERROR(EINVAL)); 1251 } 1252 1253 drc->drc_fp = fp; 1254 drc->drc_voff = *voffp; 1255 drc->drc_featureflags = 1256 DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo); 1257 1258 uint32_t payloadlen = drc->drc_drr_begin->drr_payloadlen; 1259 1260 /* 1261 * Since OpenZFS 2.0.0, we have enforced a 64MB limit in userspace 1262 * configurable via ZFS_SENDRECV_MAX_NVLIST. We enforce 256MB as a hard 1263 * upper limit. Systems with less than 1GB of RAM will see a lower 1264 * limit from `arc_all_memory() / 4`. 1265 */ 1266 if (payloadlen > (MIN((1U << 28), arc_all_memory() / 4))) 1267 return (E2BIG); 1268 1269 1270 if (payloadlen != 0) { 1271 void *payload = vmem_alloc(payloadlen, KM_SLEEP); 1272 /* 1273 * For compatibility with recursive send streams, we don't do 1274 * this here if the stream could be part of a package. Instead, 1275 * we'll do it in dmu_recv_stream. If we pull the next header 1276 * too early, and it's the END record, we break the `recv_skip` 1277 * logic. 1278 */ 1279 1280 err = receive_read_payload_and_next_header(drc, payloadlen, 1281 payload); 1282 if (err != 0) { 1283 vmem_free(payload, payloadlen); 1284 return (err); 1285 } 1286 err = nvlist_unpack(payload, payloadlen, &drc->drc_begin_nvl, 1287 KM_SLEEP); 1288 vmem_free(payload, payloadlen); 1289 if (err != 0) { 1290 kmem_free(drc->drc_next_rrd, 1291 sizeof (*drc->drc_next_rrd)); 1292 return (err); 1293 } 1294 } 1295 1296 if (drc->drc_drrb->drr_flags & DRR_FLAG_SPILL_BLOCK) 1297 drc->drc_spill = B_TRUE; 1298 1299 drba.drba_origin = origin; 1300 drba.drba_cookie = drc; 1301 drba.drba_cred = CRED(); 1302 drba.drba_proc = curproc; 1303 1304 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) { 1305 err = dsl_sync_task(tofs, 1306 dmu_recv_resume_begin_check, dmu_recv_resume_begin_sync, 1307 &drba, 5, ZFS_SPACE_CHECK_NORMAL); 1308 } else { 1309 /* 1310 * For non-raw, non-incremental, non-resuming receives the 1311 * user can specify encryption parameters on the command line 1312 * with "zfs recv -o". For these receives we create a dcp and 1313 * pass it to the sync task. Creating the dcp will implicitly 1314 * remove the encryption params from the localprops nvlist, 1315 * which avoids errors when trying to set these normally 1316 * read-only properties. Any other kind of receive that 1317 * attempts to set these properties will fail as a result. 1318 */ 1319 if ((DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo) & 1320 DMU_BACKUP_FEATURE_RAW) == 0 && 1321 origin == NULL && drc->drc_drrb->drr_fromguid == 0) { 1322 err = dsl_crypto_params_create_nvlist(DCP_CMD_NONE, 1323 localprops, hidden_args, &drba.drba_dcp); 1324 } 1325 1326 if (err == 0) { 1327 err = dsl_sync_task(tofs, 1328 dmu_recv_begin_check, dmu_recv_begin_sync, 1329 &drba, 5, ZFS_SPACE_CHECK_NORMAL); 1330 dsl_crypto_params_free(drba.drba_dcp, !!err); 1331 } 1332 } 1333 1334 if (err != 0) { 1335 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 1336 nvlist_free(drc->drc_begin_nvl); 1337 } 1338 return (err); 1339 } 1340 1341 /* 1342 * Holds data need for corrective recv callback 1343 */ 1344 typedef struct cr_cb_data { 1345 uint64_t size; 1346 zbookmark_phys_t zb; 1347 spa_t *spa; 1348 } cr_cb_data_t; 1349 1350 static void 1351 corrective_read_done(zio_t *zio) 1352 { 1353 cr_cb_data_t *data = zio->io_private; 1354 /* Corruption corrected; update error log if needed */ 1355 if (zio->io_error == 0) 1356 spa_remove_error(data->spa, &data->zb, &zio->io_bp->blk_birth); 1357 kmem_free(data, sizeof (cr_cb_data_t)); 1358 abd_free(zio->io_abd); 1359 } 1360 1361 /* 1362 * zio_rewrite the data pointed to by bp with the data from the rrd's abd. 1363 */ 1364 static int 1365 do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw, 1366 struct receive_record_arg *rrd, blkptr_t *bp) 1367 { 1368 int err; 1369 zio_t *io; 1370 zbookmark_phys_t zb; 1371 dnode_t *dn; 1372 abd_t *abd = rrd->abd; 1373 zio_cksum_t bp_cksum = bp->blk_cksum; 1374 zio_flag_t flags = ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_RETRY | 1375 ZIO_FLAG_CANFAIL; 1376 1377 if (rwa->raw) 1378 flags |= ZIO_FLAG_RAW; 1379 1380 err = dnode_hold(rwa->os, drrw->drr_object, FTAG, &dn); 1381 if (err != 0) 1382 return (err); 1383 SET_BOOKMARK(&zb, dmu_objset_id(rwa->os), drrw->drr_object, 0, 1384 dbuf_whichblock(dn, 0, drrw->drr_offset)); 1385 dnode_rele(dn, FTAG); 1386 1387 if (!rwa->raw && DRR_WRITE_COMPRESSED(drrw)) { 1388 /* Decompress the stream data */ 1389 abd_t *dabd = abd_alloc_linear( 1390 drrw->drr_logical_size, B_FALSE); 1391 err = zio_decompress_data(drrw->drr_compressiontype, 1392 abd, abd_to_buf(dabd), abd_get_size(abd), 1393 abd_get_size(dabd), NULL); 1394 1395 if (err != 0) { 1396 abd_free(dabd); 1397 return (err); 1398 } 1399 /* Swap in the newly decompressed data into the abd */ 1400 abd_free(abd); 1401 abd = dabd; 1402 } 1403 1404 if (!rwa->raw && BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 1405 /* Recompress the data */ 1406 abd_t *cabd = abd_alloc_linear(BP_GET_PSIZE(bp), 1407 B_FALSE); 1408 void *buf = abd_to_buf(cabd); 1409 uint64_t csize = zio_compress_data(BP_GET_COMPRESS(bp), 1410 abd, &buf, abd_get_size(abd), 1411 rwa->os->os_complevel); 1412 abd_zero_off(cabd, csize, BP_GET_PSIZE(bp) - csize); 1413 /* Swap in newly compressed data into the abd */ 1414 abd_free(abd); 1415 abd = cabd; 1416 flags |= ZIO_FLAG_RAW_COMPRESS; 1417 } 1418 1419 /* 1420 * The stream is not encrypted but the data on-disk is. 1421 * We need to re-encrypt the buf using the same 1422 * encryption type, salt, iv, and mac that was used to encrypt 1423 * the block previosly. 1424 */ 1425 if (!rwa->raw && BP_USES_CRYPT(bp)) { 1426 dsl_dataset_t *ds; 1427 dsl_crypto_key_t *dck = NULL; 1428 uint8_t salt[ZIO_DATA_SALT_LEN]; 1429 uint8_t iv[ZIO_DATA_IV_LEN]; 1430 uint8_t mac[ZIO_DATA_MAC_LEN]; 1431 boolean_t no_crypt = B_FALSE; 1432 dsl_pool_t *dp = dmu_objset_pool(rwa->os); 1433 abd_t *eabd = abd_alloc_linear(BP_GET_PSIZE(bp), B_FALSE); 1434 1435 zio_crypt_decode_params_bp(bp, salt, iv); 1436 zio_crypt_decode_mac_bp(bp, mac); 1437 1438 dsl_pool_config_enter(dp, FTAG); 1439 err = dsl_dataset_hold_flags(dp, rwa->tofs, 1440 DS_HOLD_FLAG_DECRYPT, FTAG, &ds); 1441 if (err != 0) { 1442 dsl_pool_config_exit(dp, FTAG); 1443 abd_free(eabd); 1444 return (SET_ERROR(EACCES)); 1445 } 1446 1447 /* Look up the key from the spa's keystore */ 1448 err = spa_keystore_lookup_key(rwa->os->os_spa, 1449 zb.zb_objset, FTAG, &dck); 1450 if (err != 0) { 1451 dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, 1452 FTAG); 1453 dsl_pool_config_exit(dp, FTAG); 1454 abd_free(eabd); 1455 return (SET_ERROR(EACCES)); 1456 } 1457 1458 err = zio_do_crypt_abd(B_TRUE, &dck->dck_key, 1459 BP_GET_TYPE(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, 1460 mac, abd_get_size(abd), abd, eabd, &no_crypt); 1461 1462 spa_keystore_dsl_key_rele(rwa->os->os_spa, dck, FTAG); 1463 dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, FTAG); 1464 dsl_pool_config_exit(dp, FTAG); 1465 1466 ASSERT0(no_crypt); 1467 if (err != 0) { 1468 abd_free(eabd); 1469 return (err); 1470 } 1471 /* Swap in the newly encrypted data into the abd */ 1472 abd_free(abd); 1473 abd = eabd; 1474 1475 /* 1476 * We want to prevent zio_rewrite() from trying to 1477 * encrypt the data again 1478 */ 1479 flags |= ZIO_FLAG_RAW_ENCRYPT; 1480 } 1481 rrd->abd = abd; 1482 1483 io = zio_rewrite(NULL, rwa->os->os_spa, bp->blk_birth, bp, abd, 1484 BP_GET_PSIZE(bp), NULL, NULL, ZIO_PRIORITY_SYNC_WRITE, flags, &zb); 1485 1486 ASSERT(abd_get_size(abd) == BP_GET_LSIZE(bp) || 1487 abd_get_size(abd) == BP_GET_PSIZE(bp)); 1488 1489 /* compute new bp checksum value and make sure it matches the old one */ 1490 zio_checksum_compute(io, BP_GET_CHECKSUM(bp), abd, abd_get_size(abd)); 1491 if (!ZIO_CHECKSUM_EQUAL(bp_cksum, io->io_bp->blk_cksum)) { 1492 zio_destroy(io); 1493 if (zfs_recv_best_effort_corrective != 0) 1494 return (0); 1495 return (SET_ERROR(ECKSUM)); 1496 } 1497 1498 /* Correct the corruption in place */ 1499 err = zio_wait(io); 1500 if (err == 0) { 1501 cr_cb_data_t *cb_data = 1502 kmem_alloc(sizeof (cr_cb_data_t), KM_SLEEP); 1503 cb_data->spa = rwa->os->os_spa; 1504 cb_data->size = drrw->drr_logical_size; 1505 cb_data->zb = zb; 1506 /* Test if healing worked by re-reading the bp */ 1507 err = zio_wait(zio_read(rwa->heal_pio, rwa->os->os_spa, bp, 1508 abd_alloc_for_io(drrw->drr_logical_size, B_FALSE), 1509 drrw->drr_logical_size, corrective_read_done, 1510 cb_data, ZIO_PRIORITY_ASYNC_READ, flags, NULL)); 1511 } 1512 if (err != 0 && zfs_recv_best_effort_corrective != 0) 1513 err = 0; 1514 1515 return (err); 1516 } 1517 1518 static int 1519 receive_read(dmu_recv_cookie_t *drc, int len, void *buf) 1520 { 1521 int done = 0; 1522 1523 /* 1524 * The code doesn't rely on this (lengths being multiples of 8). See 1525 * comment in dump_bytes. 1526 */ 1527 ASSERT(len % 8 == 0 || 1528 (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) != 0); 1529 1530 while (done < len) { 1531 ssize_t resid = len - done; 1532 zfs_file_t *fp = drc->drc_fp; 1533 int err = zfs_file_read(fp, (char *)buf + done, 1534 len - done, &resid); 1535 if (err == 0 && resid == len - done) { 1536 /* 1537 * Note: ECKSUM or ZFS_ERR_STREAM_TRUNCATED indicates 1538 * that the receive was interrupted and can 1539 * potentially be resumed. 1540 */ 1541 err = SET_ERROR(ZFS_ERR_STREAM_TRUNCATED); 1542 } 1543 drc->drc_voff += len - done - resid; 1544 done = len - resid; 1545 if (err != 0) 1546 return (err); 1547 } 1548 1549 drc->drc_bytes_read += len; 1550 1551 ASSERT3U(done, ==, len); 1552 return (0); 1553 } 1554 1555 static inline uint8_t 1556 deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size) 1557 { 1558 if (bonus_type == DMU_OT_SA) { 1559 return (1); 1560 } else { 1561 return (1 + 1562 ((DN_OLD_MAX_BONUSLEN - 1563 MIN(DN_OLD_MAX_BONUSLEN, bonus_size)) >> SPA_BLKPTRSHIFT)); 1564 } 1565 } 1566 1567 static void 1568 save_resume_state(struct receive_writer_arg *rwa, 1569 uint64_t object, uint64_t offset, dmu_tx_t *tx) 1570 { 1571 int txgoff = dmu_tx_get_txg(tx) & TXG_MASK; 1572 1573 if (!rwa->resumable) 1574 return; 1575 1576 /* 1577 * We use ds_resume_bytes[] != 0 to indicate that we need to 1578 * update this on disk, so it must not be 0. 1579 */ 1580 ASSERT(rwa->bytes_read != 0); 1581 1582 /* 1583 * We only resume from write records, which have a valid 1584 * (non-meta-dnode) object number. 1585 */ 1586 ASSERT(object != 0); 1587 1588 /* 1589 * For resuming to work correctly, we must receive records in order, 1590 * sorted by object,offset. This is checked by the callers, but 1591 * assert it here for good measure. 1592 */ 1593 ASSERT3U(object, >=, rwa->os->os_dsl_dataset->ds_resume_object[txgoff]); 1594 ASSERT(object != rwa->os->os_dsl_dataset->ds_resume_object[txgoff] || 1595 offset >= rwa->os->os_dsl_dataset->ds_resume_offset[txgoff]); 1596 ASSERT3U(rwa->bytes_read, >=, 1597 rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff]); 1598 1599 rwa->os->os_dsl_dataset->ds_resume_object[txgoff] = object; 1600 rwa->os->os_dsl_dataset->ds_resume_offset[txgoff] = offset; 1601 rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff] = rwa->bytes_read; 1602 } 1603 1604 static int 1605 receive_object_is_same_generation(objset_t *os, uint64_t object, 1606 dmu_object_type_t old_bonus_type, dmu_object_type_t new_bonus_type, 1607 const void *new_bonus, boolean_t *samegenp) 1608 { 1609 zfs_file_info_t zoi; 1610 int err; 1611 1612 dmu_buf_t *old_bonus_dbuf; 1613 err = dmu_bonus_hold(os, object, FTAG, &old_bonus_dbuf); 1614 if (err != 0) 1615 return (err); 1616 err = dmu_get_file_info(os, old_bonus_type, old_bonus_dbuf->db_data, 1617 &zoi); 1618 dmu_buf_rele(old_bonus_dbuf, FTAG); 1619 if (err != 0) 1620 return (err); 1621 uint64_t old_gen = zoi.zfi_generation; 1622 1623 err = dmu_get_file_info(os, new_bonus_type, new_bonus, &zoi); 1624 if (err != 0) 1625 return (err); 1626 uint64_t new_gen = zoi.zfi_generation; 1627 1628 *samegenp = (old_gen == new_gen); 1629 return (0); 1630 } 1631 1632 static int 1633 receive_handle_existing_object(const struct receive_writer_arg *rwa, 1634 const struct drr_object *drro, const dmu_object_info_t *doi, 1635 const void *bonus_data, 1636 uint64_t *object_to_hold, uint32_t *new_blksz) 1637 { 1638 uint32_t indblksz = drro->drr_indblkshift ? 1639 1ULL << drro->drr_indblkshift : 0; 1640 int nblkptr = deduce_nblkptr(drro->drr_bonustype, 1641 drro->drr_bonuslen); 1642 uint8_t dn_slots = drro->drr_dn_slots != 0 ? 1643 drro->drr_dn_slots : DNODE_MIN_SLOTS; 1644 boolean_t do_free_range = B_FALSE; 1645 int err; 1646 1647 *object_to_hold = drro->drr_object; 1648 1649 /* nblkptr should be bounded by the bonus size and type */ 1650 if (rwa->raw && nblkptr != drro->drr_nblkptr) 1651 return (SET_ERROR(EINVAL)); 1652 1653 /* 1654 * After the previous send stream, the sending system may 1655 * have freed this object, and then happened to re-allocate 1656 * this object number in a later txg. In this case, we are 1657 * receiving a different logical file, and the block size may 1658 * appear to be different. i.e. we may have a different 1659 * block size for this object than what the send stream says. 1660 * In this case we need to remove the object's contents, 1661 * so that its structure can be changed and then its contents 1662 * entirely replaced by subsequent WRITE records. 1663 * 1664 * If this is a -L (--large-block) incremental stream, and 1665 * the previous stream was not -L, the block size may appear 1666 * to increase. i.e. we may have a smaller block size for 1667 * this object than what the send stream says. In this case 1668 * we need to keep the object's contents and block size 1669 * intact, so that we don't lose parts of the object's 1670 * contents that are not changed by this incremental send 1671 * stream. 1672 * 1673 * We can distinguish between the two above cases by using 1674 * the ZPL's generation number (see 1675 * receive_object_is_same_generation()). However, we only 1676 * want to rely on the generation number when absolutely 1677 * necessary, because with raw receives, the generation is 1678 * encrypted. We also want to minimize dependence on the 1679 * ZPL, so that other types of datasets can also be received 1680 * (e.g. ZVOLs, although note that ZVOLS currently do not 1681 * reallocate their objects or change their structure). 1682 * Therefore, we check a number of different cases where we 1683 * know it is safe to discard the object's contents, before 1684 * using the ZPL's generation number to make the above 1685 * distinction. 1686 */ 1687 if (drro->drr_blksz != doi->doi_data_block_size) { 1688 if (rwa->raw) { 1689 /* 1690 * RAW streams always have large blocks, so 1691 * we are sure that the data is not needed 1692 * due to changing --large-block to be on. 1693 * Which is fortunate since the bonus buffer 1694 * (which contains the ZPL generation) is 1695 * encrypted, and the key might not be 1696 * loaded. 1697 */ 1698 do_free_range = B_TRUE; 1699 } else if (rwa->full) { 1700 /* 1701 * This is a full send stream, so it always 1702 * replaces what we have. Even if the 1703 * generation numbers happen to match, this 1704 * can not actually be the same logical file. 1705 * This is relevant when receiving a full 1706 * send as a clone. 1707 */ 1708 do_free_range = B_TRUE; 1709 } else if (drro->drr_type != 1710 DMU_OT_PLAIN_FILE_CONTENTS || 1711 doi->doi_type != DMU_OT_PLAIN_FILE_CONTENTS) { 1712 /* 1713 * PLAIN_FILE_CONTENTS are the only type of 1714 * objects that have ever been stored with 1715 * large blocks, so we don't need the special 1716 * logic below. ZAP blocks can shrink (when 1717 * there's only one block), so we don't want 1718 * to hit the error below about block size 1719 * only increasing. 1720 */ 1721 do_free_range = B_TRUE; 1722 } else if (doi->doi_max_offset <= 1723 doi->doi_data_block_size) { 1724 /* 1725 * There is only one block. We can free it, 1726 * because its contents will be replaced by a 1727 * WRITE record. This can not be the no-L -> 1728 * -L case, because the no-L case would have 1729 * resulted in multiple blocks. If we 1730 * supported -L -> no-L, it would not be safe 1731 * to free the file's contents. Fortunately, 1732 * that is not allowed (see 1733 * recv_check_large_blocks()). 1734 */ 1735 do_free_range = B_TRUE; 1736 } else { 1737 boolean_t is_same_gen; 1738 err = receive_object_is_same_generation(rwa->os, 1739 drro->drr_object, doi->doi_bonus_type, 1740 drro->drr_bonustype, bonus_data, &is_same_gen); 1741 if (err != 0) 1742 return (SET_ERROR(EINVAL)); 1743 1744 if (is_same_gen) { 1745 /* 1746 * This is the same logical file, and 1747 * the block size must be increasing. 1748 * It could only decrease if 1749 * --large-block was changed to be 1750 * off, which is checked in 1751 * recv_check_large_blocks(). 1752 */ 1753 if (drro->drr_blksz <= 1754 doi->doi_data_block_size) 1755 return (SET_ERROR(EINVAL)); 1756 /* 1757 * We keep the existing blocksize and 1758 * contents. 1759 */ 1760 *new_blksz = 1761 doi->doi_data_block_size; 1762 } else { 1763 do_free_range = B_TRUE; 1764 } 1765 } 1766 } 1767 1768 /* nblkptr can only decrease if the object was reallocated */ 1769 if (nblkptr < doi->doi_nblkptr) 1770 do_free_range = B_TRUE; 1771 1772 /* number of slots can only change on reallocation */ 1773 if (dn_slots != doi->doi_dnodesize >> DNODE_SHIFT) 1774 do_free_range = B_TRUE; 1775 1776 /* 1777 * For raw sends we also check a few other fields to 1778 * ensure we are preserving the objset structure exactly 1779 * as it was on the receive side: 1780 * - A changed indirect block size 1781 * - A smaller nlevels 1782 */ 1783 if (rwa->raw) { 1784 if (indblksz != doi->doi_metadata_block_size) 1785 do_free_range = B_TRUE; 1786 if (drro->drr_nlevels < doi->doi_indirection) 1787 do_free_range = B_TRUE; 1788 } 1789 1790 if (do_free_range) { 1791 err = dmu_free_long_range(rwa->os, drro->drr_object, 1792 0, DMU_OBJECT_END); 1793 if (err != 0) 1794 return (SET_ERROR(EINVAL)); 1795 } 1796 1797 /* 1798 * The dmu does not currently support decreasing nlevels or changing 1799 * indirect block size if there is already one, same as changing the 1800 * number of of dnode slots on an object. For non-raw sends this 1801 * does not matter and the new object can just use the previous one's 1802 * parameters. For raw sends, however, the structure of the received 1803 * dnode (including indirects and dnode slots) must match that of the 1804 * send side. Therefore, instead of using dmu_object_reclaim(), we 1805 * must free the object completely and call dmu_object_claim_dnsize() 1806 * instead. 1807 */ 1808 if ((rwa->raw && ((doi->doi_indirection > 1 && 1809 indblksz != doi->doi_metadata_block_size) || 1810 drro->drr_nlevels < doi->doi_indirection)) || 1811 dn_slots != doi->doi_dnodesize >> DNODE_SHIFT) { 1812 err = dmu_free_long_object(rwa->os, drro->drr_object); 1813 if (err != 0) 1814 return (SET_ERROR(EINVAL)); 1815 1816 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1817 *object_to_hold = DMU_NEW_OBJECT; 1818 } 1819 1820 /* 1821 * For raw receives, free everything beyond the new incoming 1822 * maxblkid. Normally this would be done with a DRR_FREE 1823 * record that would come after this DRR_OBJECT record is 1824 * processed. However, for raw receives we manually set the 1825 * maxblkid from the drr_maxblkid and so we must first free 1826 * everything above that blkid to ensure the DMU is always 1827 * consistent with itself. We will never free the first block 1828 * of the object here because a maxblkid of 0 could indicate 1829 * an object with a single block or one with no blocks. This 1830 * free may be skipped when dmu_free_long_range() was called 1831 * above since it covers the entire object's contents. 1832 */ 1833 if (rwa->raw && *object_to_hold != DMU_NEW_OBJECT && !do_free_range) { 1834 err = dmu_free_long_range(rwa->os, drro->drr_object, 1835 (drro->drr_maxblkid + 1) * doi->doi_data_block_size, 1836 DMU_OBJECT_END); 1837 if (err != 0) 1838 return (SET_ERROR(EINVAL)); 1839 } 1840 return (0); 1841 } 1842 1843 noinline static int 1844 receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, 1845 void *data) 1846 { 1847 dmu_object_info_t doi; 1848 dmu_tx_t *tx; 1849 int err; 1850 uint32_t new_blksz = drro->drr_blksz; 1851 uint8_t dn_slots = drro->drr_dn_slots != 0 ? 1852 drro->drr_dn_slots : DNODE_MIN_SLOTS; 1853 1854 if (drro->drr_type == DMU_OT_NONE || 1855 !DMU_OT_IS_VALID(drro->drr_type) || 1856 !DMU_OT_IS_VALID(drro->drr_bonustype) || 1857 drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS || 1858 drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS || 1859 P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) || 1860 drro->drr_blksz < SPA_MINBLOCKSIZE || 1861 drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(rwa->os)) || 1862 drro->drr_bonuslen > 1863 DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(rwa->os))) || 1864 dn_slots > 1865 (spa_maxdnodesize(dmu_objset_spa(rwa->os)) >> DNODE_SHIFT)) { 1866 return (SET_ERROR(EINVAL)); 1867 } 1868 1869 if (rwa->raw) { 1870 /* 1871 * We should have received a DRR_OBJECT_RANGE record 1872 * containing this block and stored it in rwa. 1873 */ 1874 if (drro->drr_object < rwa->or_firstobj || 1875 drro->drr_object >= rwa->or_firstobj + rwa->or_numslots || 1876 drro->drr_raw_bonuslen < drro->drr_bonuslen || 1877 drro->drr_indblkshift > SPA_MAXBLOCKSHIFT || 1878 drro->drr_nlevels > DN_MAX_LEVELS || 1879 drro->drr_nblkptr > DN_MAX_NBLKPTR || 1880 DN_SLOTS_TO_BONUSLEN(dn_slots) < 1881 drro->drr_raw_bonuslen) 1882 return (SET_ERROR(EINVAL)); 1883 } else { 1884 /* 1885 * The DRR_OBJECT_SPILL flag is valid when the DRR_BEGIN 1886 * record indicates this by setting DRR_FLAG_SPILL_BLOCK. 1887 */ 1888 if (((drro->drr_flags & ~(DRR_OBJECT_SPILL))) || 1889 (!rwa->spill && DRR_OBJECT_HAS_SPILL(drro->drr_flags))) { 1890 return (SET_ERROR(EINVAL)); 1891 } 1892 1893 if (drro->drr_raw_bonuslen != 0 || drro->drr_nblkptr != 0 || 1894 drro->drr_indblkshift != 0 || drro->drr_nlevels != 0) { 1895 return (SET_ERROR(EINVAL)); 1896 } 1897 } 1898 1899 err = dmu_object_info(rwa->os, drro->drr_object, &doi); 1900 1901 if (err != 0 && err != ENOENT && err != EEXIST) 1902 return (SET_ERROR(EINVAL)); 1903 1904 if (drro->drr_object > rwa->max_object) 1905 rwa->max_object = drro->drr_object; 1906 1907 /* 1908 * If we are losing blkptrs or changing the block size this must 1909 * be a new file instance. We must clear out the previous file 1910 * contents before we can change this type of metadata in the dnode. 1911 * Raw receives will also check that the indirect structure of the 1912 * dnode hasn't changed. 1913 */ 1914 uint64_t object_to_hold; 1915 if (err == 0) { 1916 err = receive_handle_existing_object(rwa, drro, &doi, data, 1917 &object_to_hold, &new_blksz); 1918 if (err != 0) 1919 return (err); 1920 } else if (err == EEXIST) { 1921 /* 1922 * The object requested is currently an interior slot of a 1923 * multi-slot dnode. This will be resolved when the next txg 1924 * is synced out, since the send stream will have told us 1925 * to free this slot when we freed the associated dnode 1926 * earlier in the stream. 1927 */ 1928 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1929 1930 if (dmu_object_info(rwa->os, drro->drr_object, NULL) != ENOENT) 1931 return (SET_ERROR(EINVAL)); 1932 1933 /* object was freed and we are about to allocate a new one */ 1934 object_to_hold = DMU_NEW_OBJECT; 1935 } else { 1936 /* 1937 * If the only record in this range so far was DRR_FREEOBJECTS 1938 * with at least one actually freed object, it's possible that 1939 * the block will now be converted to a hole. We need to wait 1940 * for the txg to sync to prevent races. 1941 */ 1942 if (rwa->or_need_sync == ORNS_YES) 1943 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1944 1945 /* object is free and we are about to allocate a new one */ 1946 object_to_hold = DMU_NEW_OBJECT; 1947 } 1948 1949 /* Only relevant for the first object in the range */ 1950 rwa->or_need_sync = ORNS_NO; 1951 1952 /* 1953 * If this is a multi-slot dnode there is a chance that this 1954 * object will expand into a slot that is already used by 1955 * another object from the previous snapshot. We must free 1956 * these objects before we attempt to allocate the new dnode. 1957 */ 1958 if (dn_slots > 1) { 1959 boolean_t need_sync = B_FALSE; 1960 1961 for (uint64_t slot = drro->drr_object + 1; 1962 slot < drro->drr_object + dn_slots; 1963 slot++) { 1964 dmu_object_info_t slot_doi; 1965 1966 err = dmu_object_info(rwa->os, slot, &slot_doi); 1967 if (err == ENOENT || err == EEXIST) 1968 continue; 1969 else if (err != 0) 1970 return (err); 1971 1972 err = dmu_free_long_object(rwa->os, slot); 1973 if (err != 0) 1974 return (err); 1975 1976 need_sync = B_TRUE; 1977 } 1978 1979 if (need_sync) 1980 txg_wait_synced(dmu_objset_pool(rwa->os), 0); 1981 } 1982 1983 tx = dmu_tx_create(rwa->os); 1984 dmu_tx_hold_bonus(tx, object_to_hold); 1985 dmu_tx_hold_write(tx, object_to_hold, 0, 0); 1986 err = dmu_tx_assign(tx, TXG_WAIT); 1987 if (err != 0) { 1988 dmu_tx_abort(tx); 1989 return (err); 1990 } 1991 1992 if (object_to_hold == DMU_NEW_OBJECT) { 1993 /* Currently free, wants to be allocated */ 1994 err = dmu_object_claim_dnsize(rwa->os, drro->drr_object, 1995 drro->drr_type, new_blksz, 1996 drro->drr_bonustype, drro->drr_bonuslen, 1997 dn_slots << DNODE_SHIFT, tx); 1998 } else if (drro->drr_type != doi.doi_type || 1999 new_blksz != doi.doi_data_block_size || 2000 drro->drr_bonustype != doi.doi_bonus_type || 2001 drro->drr_bonuslen != doi.doi_bonus_size) { 2002 /* Currently allocated, but with different properties */ 2003 err = dmu_object_reclaim_dnsize(rwa->os, drro->drr_object, 2004 drro->drr_type, new_blksz, 2005 drro->drr_bonustype, drro->drr_bonuslen, 2006 dn_slots << DNODE_SHIFT, rwa->spill ? 2007 DRR_OBJECT_HAS_SPILL(drro->drr_flags) : B_FALSE, tx); 2008 } else if (rwa->spill && !DRR_OBJECT_HAS_SPILL(drro->drr_flags)) { 2009 /* 2010 * Currently allocated, the existing version of this object 2011 * may reference a spill block that is no longer allocated 2012 * at the source and needs to be freed. 2013 */ 2014 err = dmu_object_rm_spill(rwa->os, drro->drr_object, tx); 2015 } 2016 2017 if (err != 0) { 2018 dmu_tx_commit(tx); 2019 return (SET_ERROR(EINVAL)); 2020 } 2021 2022 if (rwa->or_crypt_params_present) { 2023 /* 2024 * Set the crypt params for the buffer associated with this 2025 * range of dnodes. This causes the blkptr_t to have the 2026 * same crypt params (byteorder, salt, iv, mac) as on the 2027 * sending side. 2028 * 2029 * Since we are committing this tx now, it is possible for 2030 * the dnode block to end up on-disk with the incorrect MAC, 2031 * if subsequent objects in this block are received in a 2032 * different txg. However, since the dataset is marked as 2033 * inconsistent, no code paths will do a non-raw read (or 2034 * decrypt the block / verify the MAC). The receive code and 2035 * scrub code can safely do raw reads and verify the 2036 * checksum. They don't need to verify the MAC. 2037 */ 2038 dmu_buf_t *db = NULL; 2039 uint64_t offset = rwa->or_firstobj * DNODE_MIN_SIZE; 2040 2041 err = dmu_buf_hold_by_dnode(DMU_META_DNODE(rwa->os), 2042 offset, FTAG, &db, DMU_READ_PREFETCH | DMU_READ_NO_DECRYPT); 2043 if (err != 0) { 2044 dmu_tx_commit(tx); 2045 return (SET_ERROR(EINVAL)); 2046 } 2047 2048 dmu_buf_set_crypt_params(db, rwa->or_byteorder, 2049 rwa->or_salt, rwa->or_iv, rwa->or_mac, tx); 2050 2051 dmu_buf_rele(db, FTAG); 2052 2053 rwa->or_crypt_params_present = B_FALSE; 2054 } 2055 2056 dmu_object_set_checksum(rwa->os, drro->drr_object, 2057 drro->drr_checksumtype, tx); 2058 dmu_object_set_compress(rwa->os, drro->drr_object, 2059 drro->drr_compress, tx); 2060 2061 /* handle more restrictive dnode structuring for raw recvs */ 2062 if (rwa->raw) { 2063 /* 2064 * Set the indirect block size, block shift, nlevels. 2065 * This will not fail because we ensured all of the 2066 * blocks were freed earlier if this is a new object. 2067 * For non-new objects block size and indirect block 2068 * shift cannot change and nlevels can only increase. 2069 */ 2070 ASSERT3U(new_blksz, ==, drro->drr_blksz); 2071 VERIFY0(dmu_object_set_blocksize(rwa->os, drro->drr_object, 2072 drro->drr_blksz, drro->drr_indblkshift, tx)); 2073 VERIFY0(dmu_object_set_nlevels(rwa->os, drro->drr_object, 2074 drro->drr_nlevels, tx)); 2075 2076 /* 2077 * Set the maxblkid. This will always succeed because 2078 * we freed all blocks beyond the new maxblkid above. 2079 */ 2080 VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object, 2081 drro->drr_maxblkid, tx)); 2082 } 2083 2084 if (data != NULL) { 2085 dmu_buf_t *db; 2086 dnode_t *dn; 2087 uint32_t flags = DMU_READ_NO_PREFETCH; 2088 2089 if (rwa->raw) 2090 flags |= DMU_READ_NO_DECRYPT; 2091 2092 VERIFY0(dnode_hold(rwa->os, drro->drr_object, FTAG, &dn)); 2093 VERIFY0(dmu_bonus_hold_by_dnode(dn, FTAG, &db, flags)); 2094 2095 dmu_buf_will_dirty(db, tx); 2096 2097 ASSERT3U(db->db_size, >=, drro->drr_bonuslen); 2098 memcpy(db->db_data, data, DRR_OBJECT_PAYLOAD_SIZE(drro)); 2099 2100 /* 2101 * Raw bonus buffers have their byteorder determined by the 2102 * DRR_OBJECT_RANGE record. 2103 */ 2104 if (rwa->byteswap && !rwa->raw) { 2105 dmu_object_byteswap_t byteswap = 2106 DMU_OT_BYTESWAP(drro->drr_bonustype); 2107 dmu_ot_byteswap[byteswap].ob_func(db->db_data, 2108 DRR_OBJECT_PAYLOAD_SIZE(drro)); 2109 } 2110 dmu_buf_rele(db, FTAG); 2111 dnode_rele(dn, FTAG); 2112 } 2113 dmu_tx_commit(tx); 2114 2115 return (0); 2116 } 2117 2118 noinline static int 2119 receive_freeobjects(struct receive_writer_arg *rwa, 2120 struct drr_freeobjects *drrfo) 2121 { 2122 uint64_t obj; 2123 int next_err = 0; 2124 2125 if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj) 2126 return (SET_ERROR(EINVAL)); 2127 2128 for (obj = drrfo->drr_firstobj == 0 ? 1 : drrfo->drr_firstobj; 2129 obj < drrfo->drr_firstobj + drrfo->drr_numobjs && 2130 obj < DN_MAX_OBJECT && next_err == 0; 2131 next_err = dmu_object_next(rwa->os, &obj, FALSE, 0)) { 2132 dmu_object_info_t doi; 2133 int err; 2134 2135 err = dmu_object_info(rwa->os, obj, &doi); 2136 if (err == ENOENT) 2137 continue; 2138 else if (err != 0) 2139 return (err); 2140 2141 err = dmu_free_long_object(rwa->os, obj); 2142 2143 if (err != 0) 2144 return (err); 2145 2146 if (rwa->or_need_sync == ORNS_MAYBE) 2147 rwa->or_need_sync = ORNS_YES; 2148 } 2149 if (next_err != ESRCH) 2150 return (next_err); 2151 return (0); 2152 } 2153 2154 /* 2155 * Note: if this fails, the caller will clean up any records left on the 2156 * rwa->write_batch list. 2157 */ 2158 static int 2159 flush_write_batch_impl(struct receive_writer_arg *rwa) 2160 { 2161 dnode_t *dn; 2162 int err; 2163 2164 if (dnode_hold(rwa->os, rwa->last_object, FTAG, &dn) != 0) 2165 return (SET_ERROR(EINVAL)); 2166 2167 struct receive_record_arg *last_rrd = list_tail(&rwa->write_batch); 2168 struct drr_write *last_drrw = &last_rrd->header.drr_u.drr_write; 2169 2170 struct receive_record_arg *first_rrd = list_head(&rwa->write_batch); 2171 struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write; 2172 2173 ASSERT3U(rwa->last_object, ==, last_drrw->drr_object); 2174 ASSERT3U(rwa->last_offset, ==, last_drrw->drr_offset); 2175 2176 dmu_tx_t *tx = dmu_tx_create(rwa->os); 2177 dmu_tx_hold_write_by_dnode(tx, dn, first_drrw->drr_offset, 2178 last_drrw->drr_offset - first_drrw->drr_offset + 2179 last_drrw->drr_logical_size); 2180 err = dmu_tx_assign(tx, TXG_WAIT); 2181 if (err != 0) { 2182 dmu_tx_abort(tx); 2183 dnode_rele(dn, FTAG); 2184 return (err); 2185 } 2186 2187 struct receive_record_arg *rrd; 2188 while ((rrd = list_head(&rwa->write_batch)) != NULL) { 2189 struct drr_write *drrw = &rrd->header.drr_u.drr_write; 2190 abd_t *abd = rrd->abd; 2191 2192 ASSERT3U(drrw->drr_object, ==, rwa->last_object); 2193 2194 if (drrw->drr_logical_size != dn->dn_datablksz) { 2195 /* 2196 * The WRITE record is larger than the object's block 2197 * size. We must be receiving an incremental 2198 * large-block stream into a dataset that previously did 2199 * a non-large-block receive. Lightweight writes must 2200 * be exactly one block, so we need to decompress the 2201 * data (if compressed) and do a normal dmu_write(). 2202 */ 2203 ASSERT3U(drrw->drr_logical_size, >, dn->dn_datablksz); 2204 if (DRR_WRITE_COMPRESSED(drrw)) { 2205 abd_t *decomp_abd = 2206 abd_alloc_linear(drrw->drr_logical_size, 2207 B_FALSE); 2208 2209 err = zio_decompress_data( 2210 drrw->drr_compressiontype, 2211 abd, abd_to_buf(decomp_abd), 2212 abd_get_size(abd), 2213 abd_get_size(decomp_abd), NULL); 2214 2215 if (err == 0) { 2216 dmu_write_by_dnode(dn, 2217 drrw->drr_offset, 2218 drrw->drr_logical_size, 2219 abd_to_buf(decomp_abd), tx); 2220 } 2221 abd_free(decomp_abd); 2222 } else { 2223 dmu_write_by_dnode(dn, 2224 drrw->drr_offset, 2225 drrw->drr_logical_size, 2226 abd_to_buf(abd), tx); 2227 } 2228 if (err == 0) 2229 abd_free(abd); 2230 } else { 2231 zio_prop_t zp = {0}; 2232 dmu_write_policy(rwa->os, dn, 0, 0, &zp); 2233 2234 zio_flag_t zio_flags = 0; 2235 2236 if (rwa->raw) { 2237 zp.zp_encrypt = B_TRUE; 2238 zp.zp_compress = drrw->drr_compressiontype; 2239 zp.zp_byteorder = ZFS_HOST_BYTEORDER ^ 2240 !!DRR_IS_RAW_BYTESWAPPED(drrw->drr_flags) ^ 2241 rwa->byteswap; 2242 memcpy(zp.zp_salt, drrw->drr_salt, 2243 ZIO_DATA_SALT_LEN); 2244 memcpy(zp.zp_iv, drrw->drr_iv, 2245 ZIO_DATA_IV_LEN); 2246 memcpy(zp.zp_mac, drrw->drr_mac, 2247 ZIO_DATA_MAC_LEN); 2248 if (DMU_OT_IS_ENCRYPTED(zp.zp_type)) { 2249 zp.zp_nopwrite = B_FALSE; 2250 zp.zp_copies = MIN(zp.zp_copies, 2251 SPA_DVAS_PER_BP - 1); 2252 } 2253 zio_flags |= ZIO_FLAG_RAW; 2254 } else if (DRR_WRITE_COMPRESSED(drrw)) { 2255 ASSERT3U(drrw->drr_compressed_size, >, 0); 2256 ASSERT3U(drrw->drr_logical_size, >=, 2257 drrw->drr_compressed_size); 2258 zp.zp_compress = drrw->drr_compressiontype; 2259 zio_flags |= ZIO_FLAG_RAW_COMPRESS; 2260 } else if (rwa->byteswap) { 2261 /* 2262 * Note: compressed blocks never need to be 2263 * byteswapped, because WRITE records for 2264 * metadata blocks are never compressed. The 2265 * exception is raw streams, which are written 2266 * in the original byteorder, and the byteorder 2267 * bit is preserved in the BP by setting 2268 * zp_byteorder above. 2269 */ 2270 dmu_object_byteswap_t byteswap = 2271 DMU_OT_BYTESWAP(drrw->drr_type); 2272 dmu_ot_byteswap[byteswap].ob_func( 2273 abd_to_buf(abd), 2274 DRR_WRITE_PAYLOAD_SIZE(drrw)); 2275 } 2276 2277 /* 2278 * Since this data can't be read until the receive 2279 * completes, we can do a "lightweight" write for 2280 * improved performance. 2281 */ 2282 err = dmu_lightweight_write_by_dnode(dn, 2283 drrw->drr_offset, abd, &zp, zio_flags, tx); 2284 } 2285 2286 if (err != 0) { 2287 /* 2288 * This rrd is left on the list, so the caller will 2289 * free it (and the abd). 2290 */ 2291 break; 2292 } 2293 2294 /* 2295 * Note: If the receive fails, we want the resume stream to 2296 * start with the same record that we last successfully 2297 * received (as opposed to the next record), so that we can 2298 * verify that we are resuming from the correct location. 2299 */ 2300 save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx); 2301 2302 list_remove(&rwa->write_batch, rrd); 2303 kmem_free(rrd, sizeof (*rrd)); 2304 } 2305 2306 dmu_tx_commit(tx); 2307 dnode_rele(dn, FTAG); 2308 return (err); 2309 } 2310 2311 noinline static int 2312 flush_write_batch(struct receive_writer_arg *rwa) 2313 { 2314 if (list_is_empty(&rwa->write_batch)) 2315 return (0); 2316 int err = rwa->err; 2317 if (err == 0) 2318 err = flush_write_batch_impl(rwa); 2319 if (err != 0) { 2320 struct receive_record_arg *rrd; 2321 while ((rrd = list_remove_head(&rwa->write_batch)) != NULL) { 2322 abd_free(rrd->abd); 2323 kmem_free(rrd, sizeof (*rrd)); 2324 } 2325 } 2326 ASSERT(list_is_empty(&rwa->write_batch)); 2327 return (err); 2328 } 2329 2330 noinline static int 2331 receive_process_write_record(struct receive_writer_arg *rwa, 2332 struct receive_record_arg *rrd) 2333 { 2334 int err = 0; 2335 2336 ASSERT3U(rrd->header.drr_type, ==, DRR_WRITE); 2337 struct drr_write *drrw = &rrd->header.drr_u.drr_write; 2338 2339 if (drrw->drr_offset + drrw->drr_logical_size < drrw->drr_offset || 2340 !DMU_OT_IS_VALID(drrw->drr_type)) 2341 return (SET_ERROR(EINVAL)); 2342 2343 if (rwa->heal) { 2344 blkptr_t *bp; 2345 dmu_buf_t *dbp; 2346 dnode_t *dn; 2347 int flags = DB_RF_CANFAIL; 2348 2349 if (rwa->raw) 2350 flags |= DB_RF_NO_DECRYPT; 2351 2352 if (rwa->byteswap) { 2353 dmu_object_byteswap_t byteswap = 2354 DMU_OT_BYTESWAP(drrw->drr_type); 2355 dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(rrd->abd), 2356 DRR_WRITE_PAYLOAD_SIZE(drrw)); 2357 } 2358 2359 err = dmu_buf_hold_noread(rwa->os, drrw->drr_object, 2360 drrw->drr_offset, FTAG, &dbp); 2361 if (err != 0) 2362 return (err); 2363 2364 /* Try to read the object to see if it needs healing */ 2365 err = dbuf_read((dmu_buf_impl_t *)dbp, NULL, flags); 2366 /* 2367 * We only try to heal when dbuf_read() returns a ECKSUMs. 2368 * Other errors (even EIO) get returned to caller. 2369 * EIO indicates that the device is not present/accessible, 2370 * so writing to it will likely fail. 2371 * If the block is healthy, we don't want to overwrite it 2372 * unnecessarily. 2373 */ 2374 if (err != ECKSUM) { 2375 dmu_buf_rele(dbp, FTAG); 2376 return (err); 2377 } 2378 dn = dmu_buf_dnode_enter(dbp); 2379 /* Make sure the on-disk block and recv record sizes match */ 2380 if (drrw->drr_logical_size != 2381 dn->dn_datablkszsec << SPA_MINBLOCKSHIFT) { 2382 err = ENOTSUP; 2383 dmu_buf_dnode_exit(dbp); 2384 dmu_buf_rele(dbp, FTAG); 2385 return (err); 2386 } 2387 /* Get the block pointer for the corrupted block */ 2388 bp = dmu_buf_get_blkptr(dbp); 2389 err = do_corrective_recv(rwa, drrw, rrd, bp); 2390 dmu_buf_dnode_exit(dbp); 2391 dmu_buf_rele(dbp, FTAG); 2392 return (err); 2393 } 2394 2395 /* 2396 * For resuming to work, records must be in increasing order 2397 * by (object, offset). 2398 */ 2399 if (drrw->drr_object < rwa->last_object || 2400 (drrw->drr_object == rwa->last_object && 2401 drrw->drr_offset < rwa->last_offset)) { 2402 return (SET_ERROR(EINVAL)); 2403 } 2404 2405 struct receive_record_arg *first_rrd = list_head(&rwa->write_batch); 2406 struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write; 2407 uint64_t batch_size = 2408 MIN(zfs_recv_write_batch_size, DMU_MAX_ACCESS / 2); 2409 if (first_rrd != NULL && 2410 (drrw->drr_object != first_drrw->drr_object || 2411 drrw->drr_offset >= first_drrw->drr_offset + batch_size)) { 2412 err = flush_write_batch(rwa); 2413 if (err != 0) 2414 return (err); 2415 } 2416 2417 rwa->last_object = drrw->drr_object; 2418 rwa->last_offset = drrw->drr_offset; 2419 2420 if (rwa->last_object > rwa->max_object) 2421 rwa->max_object = rwa->last_object; 2422 2423 list_insert_tail(&rwa->write_batch, rrd); 2424 /* 2425 * Return EAGAIN to indicate that we will use this rrd again, 2426 * so the caller should not free it 2427 */ 2428 return (EAGAIN); 2429 } 2430 2431 static int 2432 receive_write_embedded(struct receive_writer_arg *rwa, 2433 struct drr_write_embedded *drrwe, void *data) 2434 { 2435 dmu_tx_t *tx; 2436 int err; 2437 2438 if (drrwe->drr_offset + drrwe->drr_length < drrwe->drr_offset) 2439 return (SET_ERROR(EINVAL)); 2440 2441 if (drrwe->drr_psize > BPE_PAYLOAD_SIZE) 2442 return (SET_ERROR(EINVAL)); 2443 2444 if (drrwe->drr_etype >= NUM_BP_EMBEDDED_TYPES) 2445 return (SET_ERROR(EINVAL)); 2446 if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS) 2447 return (SET_ERROR(EINVAL)); 2448 if (rwa->raw) 2449 return (SET_ERROR(EINVAL)); 2450 2451 if (drrwe->drr_object > rwa->max_object) 2452 rwa->max_object = drrwe->drr_object; 2453 2454 tx = dmu_tx_create(rwa->os); 2455 2456 dmu_tx_hold_write(tx, drrwe->drr_object, 2457 drrwe->drr_offset, drrwe->drr_length); 2458 err = dmu_tx_assign(tx, TXG_WAIT); 2459 if (err != 0) { 2460 dmu_tx_abort(tx); 2461 return (err); 2462 } 2463 2464 dmu_write_embedded(rwa->os, drrwe->drr_object, 2465 drrwe->drr_offset, data, drrwe->drr_etype, 2466 drrwe->drr_compression, drrwe->drr_lsize, drrwe->drr_psize, 2467 rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx); 2468 2469 /* See comment in restore_write. */ 2470 save_resume_state(rwa, drrwe->drr_object, drrwe->drr_offset, tx); 2471 dmu_tx_commit(tx); 2472 return (0); 2473 } 2474 2475 static int 2476 receive_spill(struct receive_writer_arg *rwa, struct drr_spill *drrs, 2477 abd_t *abd) 2478 { 2479 dmu_buf_t *db, *db_spill; 2480 int err; 2481 2482 if (drrs->drr_length < SPA_MINBLOCKSIZE || 2483 drrs->drr_length > spa_maxblocksize(dmu_objset_spa(rwa->os))) 2484 return (SET_ERROR(EINVAL)); 2485 2486 /* 2487 * This is an unmodified spill block which was added to the stream 2488 * to resolve an issue with incorrectly removing spill blocks. It 2489 * should be ignored by current versions of the code which support 2490 * the DRR_FLAG_SPILL_BLOCK flag. 2491 */ 2492 if (rwa->spill && DRR_SPILL_IS_UNMODIFIED(drrs->drr_flags)) { 2493 abd_free(abd); 2494 return (0); 2495 } 2496 2497 if (rwa->raw) { 2498 if (!DMU_OT_IS_VALID(drrs->drr_type) || 2499 drrs->drr_compressiontype >= ZIO_COMPRESS_FUNCTIONS || 2500 drrs->drr_compressed_size == 0) 2501 return (SET_ERROR(EINVAL)); 2502 } 2503 2504 if (dmu_object_info(rwa->os, drrs->drr_object, NULL) != 0) 2505 return (SET_ERROR(EINVAL)); 2506 2507 if (drrs->drr_object > rwa->max_object) 2508 rwa->max_object = drrs->drr_object; 2509 2510 VERIFY0(dmu_bonus_hold(rwa->os, drrs->drr_object, FTAG, &db)); 2511 if ((err = dmu_spill_hold_by_bonus(db, DMU_READ_NO_DECRYPT, FTAG, 2512 &db_spill)) != 0) { 2513 dmu_buf_rele(db, FTAG); 2514 return (err); 2515 } 2516 2517 dmu_tx_t *tx = dmu_tx_create(rwa->os); 2518 2519 dmu_tx_hold_spill(tx, db->db_object); 2520 2521 err = dmu_tx_assign(tx, TXG_WAIT); 2522 if (err != 0) { 2523 dmu_buf_rele(db, FTAG); 2524 dmu_buf_rele(db_spill, FTAG); 2525 dmu_tx_abort(tx); 2526 return (err); 2527 } 2528 2529 /* 2530 * Spill blocks may both grow and shrink. When a change in size 2531 * occurs any existing dbuf must be updated to match the logical 2532 * size of the provided arc_buf_t. 2533 */ 2534 if (db_spill->db_size != drrs->drr_length) { 2535 dmu_buf_will_fill(db_spill, tx); 2536 VERIFY0(dbuf_spill_set_blksz(db_spill, 2537 drrs->drr_length, tx)); 2538 } 2539 2540 arc_buf_t *abuf; 2541 if (rwa->raw) { 2542 boolean_t byteorder = ZFS_HOST_BYTEORDER ^ 2543 !!DRR_IS_RAW_BYTESWAPPED(drrs->drr_flags) ^ 2544 rwa->byteswap; 2545 2546 abuf = arc_loan_raw_buf(dmu_objset_spa(rwa->os), 2547 drrs->drr_object, byteorder, drrs->drr_salt, 2548 drrs->drr_iv, drrs->drr_mac, drrs->drr_type, 2549 drrs->drr_compressed_size, drrs->drr_length, 2550 drrs->drr_compressiontype, 0); 2551 } else { 2552 abuf = arc_loan_buf(dmu_objset_spa(rwa->os), 2553 DMU_OT_IS_METADATA(drrs->drr_type), 2554 drrs->drr_length); 2555 if (rwa->byteswap) { 2556 dmu_object_byteswap_t byteswap = 2557 DMU_OT_BYTESWAP(drrs->drr_type); 2558 dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(abd), 2559 DRR_SPILL_PAYLOAD_SIZE(drrs)); 2560 } 2561 } 2562 2563 memcpy(abuf->b_data, abd_to_buf(abd), DRR_SPILL_PAYLOAD_SIZE(drrs)); 2564 abd_free(abd); 2565 dbuf_assign_arcbuf((dmu_buf_impl_t *)db_spill, abuf, tx); 2566 2567 dmu_buf_rele(db, FTAG); 2568 dmu_buf_rele(db_spill, FTAG); 2569 2570 dmu_tx_commit(tx); 2571 return (0); 2572 } 2573 2574 noinline static int 2575 receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf) 2576 { 2577 int err; 2578 2579 if (drrf->drr_length != -1ULL && 2580 drrf->drr_offset + drrf->drr_length < drrf->drr_offset) 2581 return (SET_ERROR(EINVAL)); 2582 2583 if (dmu_object_info(rwa->os, drrf->drr_object, NULL) != 0) 2584 return (SET_ERROR(EINVAL)); 2585 2586 if (drrf->drr_object > rwa->max_object) 2587 rwa->max_object = drrf->drr_object; 2588 2589 err = dmu_free_long_range(rwa->os, drrf->drr_object, 2590 drrf->drr_offset, drrf->drr_length); 2591 2592 return (err); 2593 } 2594 2595 static int 2596 receive_object_range(struct receive_writer_arg *rwa, 2597 struct drr_object_range *drror) 2598 { 2599 /* 2600 * By default, we assume this block is in our native format 2601 * (ZFS_HOST_BYTEORDER). We then take into account whether 2602 * the send stream is byteswapped (rwa->byteswap). Finally, 2603 * we need to byteswap again if this particular block was 2604 * in non-native format on the send side. 2605 */ 2606 boolean_t byteorder = ZFS_HOST_BYTEORDER ^ rwa->byteswap ^ 2607 !!DRR_IS_RAW_BYTESWAPPED(drror->drr_flags); 2608 2609 /* 2610 * Since dnode block sizes are constant, we should not need to worry 2611 * about making sure that the dnode block size is the same on the 2612 * sending and receiving sides for the time being. For non-raw sends, 2613 * this does not matter (and in fact we do not send a DRR_OBJECT_RANGE 2614 * record at all). Raw sends require this record type because the 2615 * encryption parameters are used to protect an entire block of bonus 2616 * buffers. If the size of dnode blocks ever becomes variable, 2617 * handling will need to be added to ensure that dnode block sizes 2618 * match on the sending and receiving side. 2619 */ 2620 if (drror->drr_numslots != DNODES_PER_BLOCK || 2621 P2PHASE(drror->drr_firstobj, DNODES_PER_BLOCK) != 0 || 2622 !rwa->raw) 2623 return (SET_ERROR(EINVAL)); 2624 2625 if (drror->drr_firstobj > rwa->max_object) 2626 rwa->max_object = drror->drr_firstobj; 2627 2628 /* 2629 * The DRR_OBJECT_RANGE handling must be deferred to receive_object() 2630 * so that the block of dnodes is not written out when it's empty, 2631 * and converted to a HOLE BP. 2632 */ 2633 rwa->or_crypt_params_present = B_TRUE; 2634 rwa->or_firstobj = drror->drr_firstobj; 2635 rwa->or_numslots = drror->drr_numslots; 2636 memcpy(rwa->or_salt, drror->drr_salt, ZIO_DATA_SALT_LEN); 2637 memcpy(rwa->or_iv, drror->drr_iv, ZIO_DATA_IV_LEN); 2638 memcpy(rwa->or_mac, drror->drr_mac, ZIO_DATA_MAC_LEN); 2639 rwa->or_byteorder = byteorder; 2640 2641 rwa->or_need_sync = ORNS_MAYBE; 2642 2643 return (0); 2644 } 2645 2646 /* 2647 * Until we have the ability to redact large ranges of data efficiently, we 2648 * process these records as frees. 2649 */ 2650 noinline static int 2651 receive_redact(struct receive_writer_arg *rwa, struct drr_redact *drrr) 2652 { 2653 struct drr_free drrf = {0}; 2654 drrf.drr_length = drrr->drr_length; 2655 drrf.drr_object = drrr->drr_object; 2656 drrf.drr_offset = drrr->drr_offset; 2657 drrf.drr_toguid = drrr->drr_toguid; 2658 return (receive_free(rwa, &drrf)); 2659 } 2660 2661 /* used to destroy the drc_ds on error */ 2662 static void 2663 dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc) 2664 { 2665 dsl_dataset_t *ds = drc->drc_ds; 2666 ds_hold_flags_t dsflags; 2667 2668 dsflags = (drc->drc_raw) ? DS_HOLD_FLAG_NONE : DS_HOLD_FLAG_DECRYPT; 2669 /* 2670 * Wait for the txg sync before cleaning up the receive. For 2671 * resumable receives, this ensures that our resume state has 2672 * been written out to disk. For raw receives, this ensures 2673 * that the user accounting code will not attempt to do anything 2674 * after we stopped receiving the dataset. 2675 */ 2676 txg_wait_synced(ds->ds_dir->dd_pool, 0); 2677 ds->ds_objset->os_raw_receive = B_FALSE; 2678 2679 rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2680 if (drc->drc_resumable && drc->drc_should_save && 2681 !BP_IS_HOLE(dsl_dataset_get_blkptr(ds))) { 2682 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2683 dsl_dataset_disown(ds, dsflags, dmu_recv_tag); 2684 } else { 2685 char name[ZFS_MAX_DATASET_NAME_LEN]; 2686 rrw_exit(&ds->ds_bp_rwlock, FTAG); 2687 dsl_dataset_name(ds, name); 2688 dsl_dataset_disown(ds, dsflags, dmu_recv_tag); 2689 if (!drc->drc_heal) 2690 (void) dsl_destroy_head(name); 2691 } 2692 } 2693 2694 static void 2695 receive_cksum(dmu_recv_cookie_t *drc, int len, void *buf) 2696 { 2697 if (drc->drc_byteswap) { 2698 (void) fletcher_4_incremental_byteswap(buf, len, 2699 &drc->drc_cksum); 2700 } else { 2701 (void) fletcher_4_incremental_native(buf, len, &drc->drc_cksum); 2702 } 2703 } 2704 2705 /* 2706 * Read the payload into a buffer of size len, and update the current record's 2707 * payload field. 2708 * Allocate drc->drc_next_rrd and read the next record's header into 2709 * drc->drc_next_rrd->header. 2710 * Verify checksum of payload and next record. 2711 */ 2712 static int 2713 receive_read_payload_and_next_header(dmu_recv_cookie_t *drc, int len, void *buf) 2714 { 2715 int err; 2716 2717 if (len != 0) { 2718 ASSERT3U(len, <=, SPA_MAXBLOCKSIZE); 2719 err = receive_read(drc, len, buf); 2720 if (err != 0) 2721 return (err); 2722 receive_cksum(drc, len, buf); 2723 2724 /* note: rrd is NULL when reading the begin record's payload */ 2725 if (drc->drc_rrd != NULL) { 2726 drc->drc_rrd->payload = buf; 2727 drc->drc_rrd->payload_size = len; 2728 drc->drc_rrd->bytes_read = drc->drc_bytes_read; 2729 } 2730 } else { 2731 ASSERT3P(buf, ==, NULL); 2732 } 2733 2734 drc->drc_prev_cksum = drc->drc_cksum; 2735 2736 drc->drc_next_rrd = kmem_zalloc(sizeof (*drc->drc_next_rrd), KM_SLEEP); 2737 err = receive_read(drc, sizeof (drc->drc_next_rrd->header), 2738 &drc->drc_next_rrd->header); 2739 drc->drc_next_rrd->bytes_read = drc->drc_bytes_read; 2740 2741 if (err != 0) { 2742 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 2743 drc->drc_next_rrd = NULL; 2744 return (err); 2745 } 2746 if (drc->drc_next_rrd->header.drr_type == DRR_BEGIN) { 2747 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 2748 drc->drc_next_rrd = NULL; 2749 return (SET_ERROR(EINVAL)); 2750 } 2751 2752 /* 2753 * Note: checksum is of everything up to but not including the 2754 * checksum itself. 2755 */ 2756 ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), 2757 ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t)); 2758 receive_cksum(drc, 2759 offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum), 2760 &drc->drc_next_rrd->header); 2761 2762 zio_cksum_t cksum_orig = 2763 drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum; 2764 zio_cksum_t *cksump = 2765 &drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum; 2766 2767 if (drc->drc_byteswap) 2768 byteswap_record(&drc->drc_next_rrd->header); 2769 2770 if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) && 2771 !ZIO_CHECKSUM_EQUAL(drc->drc_cksum, *cksump)) { 2772 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 2773 drc->drc_next_rrd = NULL; 2774 return (SET_ERROR(ECKSUM)); 2775 } 2776 2777 receive_cksum(drc, sizeof (cksum_orig), &cksum_orig); 2778 2779 return (0); 2780 } 2781 2782 /* 2783 * Issue the prefetch reads for any necessary indirect blocks. 2784 * 2785 * We use the object ignore list to tell us whether or not to issue prefetches 2786 * for a given object. We do this for both correctness (in case the blocksize 2787 * of an object has changed) and performance (if the object doesn't exist, don't 2788 * needlessly try to issue prefetches). We also trim the list as we go through 2789 * the stream to prevent it from growing to an unbounded size. 2790 * 2791 * The object numbers within will always be in sorted order, and any write 2792 * records we see will also be in sorted order, but they're not sorted with 2793 * respect to each other (i.e. we can get several object records before 2794 * receiving each object's write records). As a result, once we've reached a 2795 * given object number, we can safely remove any reference to lower object 2796 * numbers in the ignore list. In practice, we receive up to 32 object records 2797 * before receiving write records, so the list can have up to 32 nodes in it. 2798 */ 2799 static void 2800 receive_read_prefetch(dmu_recv_cookie_t *drc, uint64_t object, uint64_t offset, 2801 uint64_t length) 2802 { 2803 if (!objlist_exists(drc->drc_ignore_objlist, object)) { 2804 dmu_prefetch(drc->drc_os, object, 1, offset, length, 2805 ZIO_PRIORITY_SYNC_READ); 2806 } 2807 } 2808 2809 /* 2810 * Read records off the stream, issuing any necessary prefetches. 2811 */ 2812 static int 2813 receive_read_record(dmu_recv_cookie_t *drc) 2814 { 2815 int err; 2816 2817 switch (drc->drc_rrd->header.drr_type) { 2818 case DRR_OBJECT: 2819 { 2820 struct drr_object *drro = 2821 &drc->drc_rrd->header.drr_u.drr_object; 2822 uint32_t size = DRR_OBJECT_PAYLOAD_SIZE(drro); 2823 void *buf = NULL; 2824 dmu_object_info_t doi; 2825 2826 if (size != 0) 2827 buf = kmem_zalloc(size, KM_SLEEP); 2828 2829 err = receive_read_payload_and_next_header(drc, size, buf); 2830 if (err != 0) { 2831 kmem_free(buf, size); 2832 return (err); 2833 } 2834 err = dmu_object_info(drc->drc_os, drro->drr_object, &doi); 2835 /* 2836 * See receive_read_prefetch for an explanation why we're 2837 * storing this object in the ignore_obj_list. 2838 */ 2839 if (err == ENOENT || err == EEXIST || 2840 (err == 0 && doi.doi_data_block_size != drro->drr_blksz)) { 2841 objlist_insert(drc->drc_ignore_objlist, 2842 drro->drr_object); 2843 err = 0; 2844 } 2845 return (err); 2846 } 2847 case DRR_FREEOBJECTS: 2848 { 2849 err = receive_read_payload_and_next_header(drc, 0, NULL); 2850 return (err); 2851 } 2852 case DRR_WRITE: 2853 { 2854 struct drr_write *drrw = &drc->drc_rrd->header.drr_u.drr_write; 2855 int size = DRR_WRITE_PAYLOAD_SIZE(drrw); 2856 abd_t *abd = abd_alloc_linear(size, B_FALSE); 2857 err = receive_read_payload_and_next_header(drc, size, 2858 abd_to_buf(abd)); 2859 if (err != 0) { 2860 abd_free(abd); 2861 return (err); 2862 } 2863 drc->drc_rrd->abd = abd; 2864 receive_read_prefetch(drc, drrw->drr_object, drrw->drr_offset, 2865 drrw->drr_logical_size); 2866 return (err); 2867 } 2868 case DRR_WRITE_EMBEDDED: 2869 { 2870 struct drr_write_embedded *drrwe = 2871 &drc->drc_rrd->header.drr_u.drr_write_embedded; 2872 uint32_t size = P2ROUNDUP(drrwe->drr_psize, 8); 2873 void *buf = kmem_zalloc(size, KM_SLEEP); 2874 2875 err = receive_read_payload_and_next_header(drc, size, buf); 2876 if (err != 0) { 2877 kmem_free(buf, size); 2878 return (err); 2879 } 2880 2881 receive_read_prefetch(drc, drrwe->drr_object, drrwe->drr_offset, 2882 drrwe->drr_length); 2883 return (err); 2884 } 2885 case DRR_FREE: 2886 case DRR_REDACT: 2887 { 2888 /* 2889 * It might be beneficial to prefetch indirect blocks here, but 2890 * we don't really have the data to decide for sure. 2891 */ 2892 err = receive_read_payload_and_next_header(drc, 0, NULL); 2893 return (err); 2894 } 2895 case DRR_END: 2896 { 2897 struct drr_end *drre = &drc->drc_rrd->header.drr_u.drr_end; 2898 if (!ZIO_CHECKSUM_EQUAL(drc->drc_prev_cksum, 2899 drre->drr_checksum)) 2900 return (SET_ERROR(ECKSUM)); 2901 return (0); 2902 } 2903 case DRR_SPILL: 2904 { 2905 struct drr_spill *drrs = &drc->drc_rrd->header.drr_u.drr_spill; 2906 int size = DRR_SPILL_PAYLOAD_SIZE(drrs); 2907 abd_t *abd = abd_alloc_linear(size, B_FALSE); 2908 err = receive_read_payload_and_next_header(drc, size, 2909 abd_to_buf(abd)); 2910 if (err != 0) 2911 abd_free(abd); 2912 else 2913 drc->drc_rrd->abd = abd; 2914 return (err); 2915 } 2916 case DRR_OBJECT_RANGE: 2917 { 2918 err = receive_read_payload_and_next_header(drc, 0, NULL); 2919 return (err); 2920 2921 } 2922 default: 2923 return (SET_ERROR(EINVAL)); 2924 } 2925 } 2926 2927 2928 2929 static void 2930 dprintf_drr(struct receive_record_arg *rrd, int err) 2931 { 2932 #ifdef ZFS_DEBUG 2933 switch (rrd->header.drr_type) { 2934 case DRR_OBJECT: 2935 { 2936 struct drr_object *drro = &rrd->header.drr_u.drr_object; 2937 dprintf("drr_type = OBJECT obj = %llu type = %u " 2938 "bonustype = %u blksz = %u bonuslen = %u cksumtype = %u " 2939 "compress = %u dn_slots = %u err = %d\n", 2940 (u_longlong_t)drro->drr_object, drro->drr_type, 2941 drro->drr_bonustype, drro->drr_blksz, drro->drr_bonuslen, 2942 drro->drr_checksumtype, drro->drr_compress, 2943 drro->drr_dn_slots, err); 2944 break; 2945 } 2946 case DRR_FREEOBJECTS: 2947 { 2948 struct drr_freeobjects *drrfo = 2949 &rrd->header.drr_u.drr_freeobjects; 2950 dprintf("drr_type = FREEOBJECTS firstobj = %llu " 2951 "numobjs = %llu err = %d\n", 2952 (u_longlong_t)drrfo->drr_firstobj, 2953 (u_longlong_t)drrfo->drr_numobjs, err); 2954 break; 2955 } 2956 case DRR_WRITE: 2957 { 2958 struct drr_write *drrw = &rrd->header.drr_u.drr_write; 2959 dprintf("drr_type = WRITE obj = %llu type = %u offset = %llu " 2960 "lsize = %llu cksumtype = %u flags = %u " 2961 "compress = %u psize = %llu err = %d\n", 2962 (u_longlong_t)drrw->drr_object, drrw->drr_type, 2963 (u_longlong_t)drrw->drr_offset, 2964 (u_longlong_t)drrw->drr_logical_size, 2965 drrw->drr_checksumtype, drrw->drr_flags, 2966 drrw->drr_compressiontype, 2967 (u_longlong_t)drrw->drr_compressed_size, err); 2968 break; 2969 } 2970 case DRR_WRITE_BYREF: 2971 { 2972 struct drr_write_byref *drrwbr = 2973 &rrd->header.drr_u.drr_write_byref; 2974 dprintf("drr_type = WRITE_BYREF obj = %llu offset = %llu " 2975 "length = %llu toguid = %llx refguid = %llx " 2976 "refobject = %llu refoffset = %llu cksumtype = %u " 2977 "flags = %u err = %d\n", 2978 (u_longlong_t)drrwbr->drr_object, 2979 (u_longlong_t)drrwbr->drr_offset, 2980 (u_longlong_t)drrwbr->drr_length, 2981 (u_longlong_t)drrwbr->drr_toguid, 2982 (u_longlong_t)drrwbr->drr_refguid, 2983 (u_longlong_t)drrwbr->drr_refobject, 2984 (u_longlong_t)drrwbr->drr_refoffset, 2985 drrwbr->drr_checksumtype, drrwbr->drr_flags, err); 2986 break; 2987 } 2988 case DRR_WRITE_EMBEDDED: 2989 { 2990 struct drr_write_embedded *drrwe = 2991 &rrd->header.drr_u.drr_write_embedded; 2992 dprintf("drr_type = WRITE_EMBEDDED obj = %llu offset = %llu " 2993 "length = %llu compress = %u etype = %u lsize = %u " 2994 "psize = %u err = %d\n", 2995 (u_longlong_t)drrwe->drr_object, 2996 (u_longlong_t)drrwe->drr_offset, 2997 (u_longlong_t)drrwe->drr_length, 2998 drrwe->drr_compression, drrwe->drr_etype, 2999 drrwe->drr_lsize, drrwe->drr_psize, err); 3000 break; 3001 } 3002 case DRR_FREE: 3003 { 3004 struct drr_free *drrf = &rrd->header.drr_u.drr_free; 3005 dprintf("drr_type = FREE obj = %llu offset = %llu " 3006 "length = %lld err = %d\n", 3007 (u_longlong_t)drrf->drr_object, 3008 (u_longlong_t)drrf->drr_offset, 3009 (longlong_t)drrf->drr_length, 3010 err); 3011 break; 3012 } 3013 case DRR_SPILL: 3014 { 3015 struct drr_spill *drrs = &rrd->header.drr_u.drr_spill; 3016 dprintf("drr_type = SPILL obj = %llu length = %llu " 3017 "err = %d\n", (u_longlong_t)drrs->drr_object, 3018 (u_longlong_t)drrs->drr_length, err); 3019 break; 3020 } 3021 case DRR_OBJECT_RANGE: 3022 { 3023 struct drr_object_range *drror = 3024 &rrd->header.drr_u.drr_object_range; 3025 dprintf("drr_type = OBJECT_RANGE firstobj = %llu " 3026 "numslots = %llu flags = %u err = %d\n", 3027 (u_longlong_t)drror->drr_firstobj, 3028 (u_longlong_t)drror->drr_numslots, 3029 drror->drr_flags, err); 3030 break; 3031 } 3032 default: 3033 return; 3034 } 3035 #endif 3036 } 3037 3038 /* 3039 * Commit the records to the pool. 3040 */ 3041 static int 3042 receive_process_record(struct receive_writer_arg *rwa, 3043 struct receive_record_arg *rrd) 3044 { 3045 int err; 3046 3047 /* Processing in order, therefore bytes_read should be increasing. */ 3048 ASSERT3U(rrd->bytes_read, >=, rwa->bytes_read); 3049 rwa->bytes_read = rrd->bytes_read; 3050 3051 /* We can only heal write records; other ones get ignored */ 3052 if (rwa->heal && rrd->header.drr_type != DRR_WRITE) { 3053 if (rrd->abd != NULL) { 3054 abd_free(rrd->abd); 3055 rrd->abd = NULL; 3056 } else if (rrd->payload != NULL) { 3057 kmem_free(rrd->payload, rrd->payload_size); 3058 rrd->payload = NULL; 3059 } 3060 return (0); 3061 } 3062 3063 if (!rwa->heal && rrd->header.drr_type != DRR_WRITE) { 3064 err = flush_write_batch(rwa); 3065 if (err != 0) { 3066 if (rrd->abd != NULL) { 3067 abd_free(rrd->abd); 3068 rrd->abd = NULL; 3069 rrd->payload = NULL; 3070 } else if (rrd->payload != NULL) { 3071 kmem_free(rrd->payload, rrd->payload_size); 3072 rrd->payload = NULL; 3073 } 3074 3075 return (err); 3076 } 3077 } 3078 3079 switch (rrd->header.drr_type) { 3080 case DRR_OBJECT: 3081 { 3082 struct drr_object *drro = &rrd->header.drr_u.drr_object; 3083 err = receive_object(rwa, drro, rrd->payload); 3084 kmem_free(rrd->payload, rrd->payload_size); 3085 rrd->payload = NULL; 3086 break; 3087 } 3088 case DRR_FREEOBJECTS: 3089 { 3090 struct drr_freeobjects *drrfo = 3091 &rrd->header.drr_u.drr_freeobjects; 3092 err = receive_freeobjects(rwa, drrfo); 3093 break; 3094 } 3095 case DRR_WRITE: 3096 { 3097 err = receive_process_write_record(rwa, rrd); 3098 if (rwa->heal) { 3099 /* 3100 * If healing - always free the abd after processing 3101 */ 3102 abd_free(rrd->abd); 3103 rrd->abd = NULL; 3104 } else if (err != EAGAIN) { 3105 /* 3106 * On success, a non-healing 3107 * receive_process_write_record() returns 3108 * EAGAIN to indicate that we do not want to free 3109 * the rrd or arc_buf. 3110 */ 3111 ASSERT(err != 0); 3112 abd_free(rrd->abd); 3113 rrd->abd = NULL; 3114 } 3115 break; 3116 } 3117 case DRR_WRITE_EMBEDDED: 3118 { 3119 struct drr_write_embedded *drrwe = 3120 &rrd->header.drr_u.drr_write_embedded; 3121 err = receive_write_embedded(rwa, drrwe, rrd->payload); 3122 kmem_free(rrd->payload, rrd->payload_size); 3123 rrd->payload = NULL; 3124 break; 3125 } 3126 case DRR_FREE: 3127 { 3128 struct drr_free *drrf = &rrd->header.drr_u.drr_free; 3129 err = receive_free(rwa, drrf); 3130 break; 3131 } 3132 case DRR_SPILL: 3133 { 3134 struct drr_spill *drrs = &rrd->header.drr_u.drr_spill; 3135 err = receive_spill(rwa, drrs, rrd->abd); 3136 if (err != 0) 3137 abd_free(rrd->abd); 3138 rrd->abd = NULL; 3139 rrd->payload = NULL; 3140 break; 3141 } 3142 case DRR_OBJECT_RANGE: 3143 { 3144 struct drr_object_range *drror = 3145 &rrd->header.drr_u.drr_object_range; 3146 err = receive_object_range(rwa, drror); 3147 break; 3148 } 3149 case DRR_REDACT: 3150 { 3151 struct drr_redact *drrr = &rrd->header.drr_u.drr_redact; 3152 err = receive_redact(rwa, drrr); 3153 break; 3154 } 3155 default: 3156 err = (SET_ERROR(EINVAL)); 3157 } 3158 3159 if (err != 0) 3160 dprintf_drr(rrd, err); 3161 3162 return (err); 3163 } 3164 3165 /* 3166 * dmu_recv_stream's worker thread; pull records off the queue, and then call 3167 * receive_process_record When we're done, signal the main thread and exit. 3168 */ 3169 static __attribute__((noreturn)) void 3170 receive_writer_thread(void *arg) 3171 { 3172 struct receive_writer_arg *rwa = arg; 3173 struct receive_record_arg *rrd; 3174 fstrans_cookie_t cookie = spl_fstrans_mark(); 3175 3176 for (rrd = bqueue_dequeue(&rwa->q); !rrd->eos_marker; 3177 rrd = bqueue_dequeue(&rwa->q)) { 3178 /* 3179 * If there's an error, the main thread will stop putting things 3180 * on the queue, but we need to clear everything in it before we 3181 * can exit. 3182 */ 3183 int err = 0; 3184 if (rwa->err == 0) { 3185 err = receive_process_record(rwa, rrd); 3186 } else if (rrd->abd != NULL) { 3187 abd_free(rrd->abd); 3188 rrd->abd = NULL; 3189 rrd->payload = NULL; 3190 } else if (rrd->payload != NULL) { 3191 kmem_free(rrd->payload, rrd->payload_size); 3192 rrd->payload = NULL; 3193 } 3194 /* 3195 * EAGAIN indicates that this record has been saved (on 3196 * raw->write_batch), and will be used again, so we don't 3197 * free it. 3198 * When healing data we always need to free the record. 3199 */ 3200 if (err != EAGAIN || rwa->heal) { 3201 if (rwa->err == 0) 3202 rwa->err = err; 3203 kmem_free(rrd, sizeof (*rrd)); 3204 } 3205 } 3206 kmem_free(rrd, sizeof (*rrd)); 3207 3208 if (rwa->heal) { 3209 zio_wait(rwa->heal_pio); 3210 } else { 3211 int err = flush_write_batch(rwa); 3212 if (rwa->err == 0) 3213 rwa->err = err; 3214 } 3215 mutex_enter(&rwa->mutex); 3216 rwa->done = B_TRUE; 3217 cv_signal(&rwa->cv); 3218 mutex_exit(&rwa->mutex); 3219 spl_fstrans_unmark(cookie); 3220 thread_exit(); 3221 } 3222 3223 static int 3224 resume_check(dmu_recv_cookie_t *drc, nvlist_t *begin_nvl) 3225 { 3226 uint64_t val; 3227 objset_t *mos = dmu_objset_pool(drc->drc_os)->dp_meta_objset; 3228 uint64_t dsobj = dmu_objset_id(drc->drc_os); 3229 uint64_t resume_obj, resume_off; 3230 3231 if (nvlist_lookup_uint64(begin_nvl, 3232 "resume_object", &resume_obj) != 0 || 3233 nvlist_lookup_uint64(begin_nvl, 3234 "resume_offset", &resume_off) != 0) { 3235 return (SET_ERROR(EINVAL)); 3236 } 3237 VERIFY0(zap_lookup(mos, dsobj, 3238 DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val)); 3239 if (resume_obj != val) 3240 return (SET_ERROR(EINVAL)); 3241 VERIFY0(zap_lookup(mos, dsobj, 3242 DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val)); 3243 if (resume_off != val) 3244 return (SET_ERROR(EINVAL)); 3245 3246 return (0); 3247 } 3248 3249 /* 3250 * Read in the stream's records, one by one, and apply them to the pool. There 3251 * are two threads involved; the thread that calls this function will spin up a 3252 * worker thread, read the records off the stream one by one, and issue 3253 * prefetches for any necessary indirect blocks. It will then push the records 3254 * onto an internal blocking queue. The worker thread will pull the records off 3255 * the queue, and actually write the data into the DMU. This way, the worker 3256 * thread doesn't have to wait for reads to complete, since everything it needs 3257 * (the indirect blocks) will be prefetched. 3258 * 3259 * NB: callers *must* call dmu_recv_end() if this succeeds. 3260 */ 3261 int 3262 dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp) 3263 { 3264 int err = 0; 3265 struct receive_writer_arg *rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP); 3266 3267 if (dsl_dataset_has_resume_receive_state(drc->drc_ds)) { 3268 uint64_t bytes = 0; 3269 (void) zap_lookup(drc->drc_ds->ds_dir->dd_pool->dp_meta_objset, 3270 drc->drc_ds->ds_object, DS_FIELD_RESUME_BYTES, 3271 sizeof (bytes), 1, &bytes); 3272 drc->drc_bytes_read += bytes; 3273 } 3274 3275 drc->drc_ignore_objlist = objlist_create(); 3276 3277 /* these were verified in dmu_recv_begin */ 3278 ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==, 3279 DMU_SUBSTREAM); 3280 ASSERT3U(drc->drc_drrb->drr_type, <, DMU_OST_NUMTYPES); 3281 3282 ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT); 3283 ASSERT0(drc->drc_os->os_encrypted && 3284 (drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)); 3285 3286 /* handle DSL encryption key payload */ 3287 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) { 3288 nvlist_t *keynvl = NULL; 3289 3290 ASSERT(drc->drc_os->os_encrypted); 3291 ASSERT(drc->drc_raw); 3292 3293 err = nvlist_lookup_nvlist(drc->drc_begin_nvl, "crypt_keydata", 3294 &keynvl); 3295 if (err != 0) 3296 goto out; 3297 3298 if (!drc->drc_heal) { 3299 /* 3300 * If this is a new dataset we set the key immediately. 3301 * Otherwise we don't want to change the key until we 3302 * are sure the rest of the receive succeeded so we 3303 * stash the keynvl away until then. 3304 */ 3305 err = dsl_crypto_recv_raw(spa_name(drc->drc_os->os_spa), 3306 drc->drc_ds->ds_object, drc->drc_fromsnapobj, 3307 drc->drc_drrb->drr_type, keynvl, drc->drc_newfs); 3308 if (err != 0) 3309 goto out; 3310 } 3311 3312 /* see comment in dmu_recv_end_sync() */ 3313 drc->drc_ivset_guid = 0; 3314 (void) nvlist_lookup_uint64(keynvl, "to_ivset_guid", 3315 &drc->drc_ivset_guid); 3316 3317 if (!drc->drc_newfs) 3318 drc->drc_keynvl = fnvlist_dup(keynvl); 3319 } 3320 3321 if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) { 3322 err = resume_check(drc, drc->drc_begin_nvl); 3323 if (err != 0) 3324 goto out; 3325 } 3326 3327 /* 3328 * For compatibility with recursive send streams, we do this here, 3329 * rather than in dmu_recv_begin. If we pull the next header too 3330 * early, and it's the END record, we break the `recv_skip` logic. 3331 */ 3332 if (drc->drc_drr_begin->drr_payloadlen == 0) { 3333 err = receive_read_payload_and_next_header(drc, 0, NULL); 3334 if (err != 0) 3335 goto out; 3336 } 3337 3338 /* 3339 * If we failed before this point we will clean up any new resume 3340 * state that was created. Now that we've gotten past the initial 3341 * checks we are ok to retain that resume state. 3342 */ 3343 drc->drc_should_save = B_TRUE; 3344 3345 (void) bqueue_init(&rwa->q, zfs_recv_queue_ff, 3346 MAX(zfs_recv_queue_length, 2 * zfs_max_recordsize), 3347 offsetof(struct receive_record_arg, node)); 3348 cv_init(&rwa->cv, NULL, CV_DEFAULT, NULL); 3349 mutex_init(&rwa->mutex, NULL, MUTEX_DEFAULT, NULL); 3350 rwa->os = drc->drc_os; 3351 rwa->byteswap = drc->drc_byteswap; 3352 rwa->heal = drc->drc_heal; 3353 rwa->tofs = drc->drc_tofs; 3354 rwa->resumable = drc->drc_resumable; 3355 rwa->raw = drc->drc_raw; 3356 rwa->spill = drc->drc_spill; 3357 rwa->full = (drc->drc_drr_begin->drr_u.drr_begin.drr_fromguid == 0); 3358 rwa->os->os_raw_receive = drc->drc_raw; 3359 if (drc->drc_heal) { 3360 rwa->heal_pio = zio_root(drc->drc_os->os_spa, NULL, NULL, 3361 ZIO_FLAG_GODFATHER); 3362 } 3363 list_create(&rwa->write_batch, sizeof (struct receive_record_arg), 3364 offsetof(struct receive_record_arg, node.bqn_node)); 3365 3366 (void) thread_create(NULL, 0, receive_writer_thread, rwa, 0, curproc, 3367 TS_RUN, minclsyspri); 3368 /* 3369 * We're reading rwa->err without locks, which is safe since we are the 3370 * only reader, and the worker thread is the only writer. It's ok if we 3371 * miss a write for an iteration or two of the loop, since the writer 3372 * thread will keep freeing records we send it until we send it an eos 3373 * marker. 3374 * 3375 * We can leave this loop in 3 ways: First, if rwa->err is 3376 * non-zero. In that case, the writer thread will free the rrd we just 3377 * pushed. Second, if we're interrupted; in that case, either it's the 3378 * first loop and drc->drc_rrd was never allocated, or it's later, and 3379 * drc->drc_rrd has been handed off to the writer thread who will free 3380 * it. Finally, if receive_read_record fails or we're at the end of the 3381 * stream, then we free drc->drc_rrd and exit. 3382 */ 3383 while (rwa->err == 0) { 3384 if (issig(JUSTLOOKING) && issig(FORREAL)) { 3385 err = SET_ERROR(EINTR); 3386 break; 3387 } 3388 3389 ASSERT3P(drc->drc_rrd, ==, NULL); 3390 drc->drc_rrd = drc->drc_next_rrd; 3391 drc->drc_next_rrd = NULL; 3392 /* Allocates and loads header into drc->drc_next_rrd */ 3393 err = receive_read_record(drc); 3394 3395 if (drc->drc_rrd->header.drr_type == DRR_END || err != 0) { 3396 kmem_free(drc->drc_rrd, sizeof (*drc->drc_rrd)); 3397 drc->drc_rrd = NULL; 3398 break; 3399 } 3400 3401 bqueue_enqueue(&rwa->q, drc->drc_rrd, 3402 sizeof (struct receive_record_arg) + 3403 drc->drc_rrd->payload_size); 3404 drc->drc_rrd = NULL; 3405 } 3406 3407 ASSERT3P(drc->drc_rrd, ==, NULL); 3408 drc->drc_rrd = kmem_zalloc(sizeof (*drc->drc_rrd), KM_SLEEP); 3409 drc->drc_rrd->eos_marker = B_TRUE; 3410 bqueue_enqueue_flush(&rwa->q, drc->drc_rrd, 1); 3411 3412 mutex_enter(&rwa->mutex); 3413 while (!rwa->done) { 3414 /* 3415 * We need to use cv_wait_sig() so that any process that may 3416 * be sleeping here can still fork. 3417 */ 3418 (void) cv_wait_sig(&rwa->cv, &rwa->mutex); 3419 } 3420 mutex_exit(&rwa->mutex); 3421 3422 /* 3423 * If we are receiving a full stream as a clone, all object IDs which 3424 * are greater than the maximum ID referenced in the stream are 3425 * by definition unused and must be freed. 3426 */ 3427 if (drc->drc_clone && drc->drc_drrb->drr_fromguid == 0) { 3428 uint64_t obj = rwa->max_object + 1; 3429 int free_err = 0; 3430 int next_err = 0; 3431 3432 while (next_err == 0) { 3433 free_err = dmu_free_long_object(rwa->os, obj); 3434 if (free_err != 0 && free_err != ENOENT) 3435 break; 3436 3437 next_err = dmu_object_next(rwa->os, &obj, FALSE, 0); 3438 } 3439 3440 if (err == 0) { 3441 if (free_err != 0 && free_err != ENOENT) 3442 err = free_err; 3443 else if (next_err != ESRCH) 3444 err = next_err; 3445 } 3446 } 3447 3448 cv_destroy(&rwa->cv); 3449 mutex_destroy(&rwa->mutex); 3450 bqueue_destroy(&rwa->q); 3451 list_destroy(&rwa->write_batch); 3452 if (err == 0) 3453 err = rwa->err; 3454 3455 out: 3456 /* 3457 * If we hit an error before we started the receive_writer_thread 3458 * we need to clean up the next_rrd we create by processing the 3459 * DRR_BEGIN record. 3460 */ 3461 if (drc->drc_next_rrd != NULL) 3462 kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd)); 3463 3464 /* 3465 * The objset will be invalidated by dmu_recv_end() when we do 3466 * dsl_dataset_clone_swap_sync_impl(). 3467 */ 3468 drc->drc_os = NULL; 3469 3470 kmem_free(rwa, sizeof (*rwa)); 3471 nvlist_free(drc->drc_begin_nvl); 3472 3473 if (err != 0) { 3474 /* 3475 * Clean up references. If receive is not resumable, 3476 * destroy what we created, so we don't leave it in 3477 * the inconsistent state. 3478 */ 3479 dmu_recv_cleanup_ds(drc); 3480 nvlist_free(drc->drc_keynvl); 3481 } 3482 3483 objlist_destroy(drc->drc_ignore_objlist); 3484 drc->drc_ignore_objlist = NULL; 3485 *voffp = drc->drc_voff; 3486 return (err); 3487 } 3488 3489 static int 3490 dmu_recv_end_check(void *arg, dmu_tx_t *tx) 3491 { 3492 dmu_recv_cookie_t *drc = arg; 3493 dsl_pool_t *dp = dmu_tx_pool(tx); 3494 int error; 3495 3496 ASSERT3P(drc->drc_ds->ds_owner, ==, dmu_recv_tag); 3497 3498 if (drc->drc_heal) { 3499 error = 0; 3500 } else if (!drc->drc_newfs) { 3501 dsl_dataset_t *origin_head; 3502 3503 error = dsl_dataset_hold(dp, drc->drc_tofs, FTAG, &origin_head); 3504 if (error != 0) 3505 return (error); 3506 if (drc->drc_force) { 3507 /* 3508 * We will destroy any snapshots in tofs (i.e. before 3509 * origin_head) that are after the origin (which is 3510 * the snap before drc_ds, because drc_ds can not 3511 * have any snaps of its own). 3512 */ 3513 uint64_t obj; 3514 3515 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj; 3516 while (obj != 3517 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) { 3518 dsl_dataset_t *snap; 3519 error = dsl_dataset_hold_obj(dp, obj, FTAG, 3520 &snap); 3521 if (error != 0) 3522 break; 3523 if (snap->ds_dir != origin_head->ds_dir) 3524 error = SET_ERROR(EINVAL); 3525 if (error == 0) { 3526 error = dsl_destroy_snapshot_check_impl( 3527 snap, B_FALSE); 3528 } 3529 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj; 3530 dsl_dataset_rele(snap, FTAG); 3531 if (error != 0) 3532 break; 3533 } 3534 if (error != 0) { 3535 dsl_dataset_rele(origin_head, FTAG); 3536 return (error); 3537 } 3538 } 3539 if (drc->drc_keynvl != NULL) { 3540 error = dsl_crypto_recv_raw_key_check(drc->drc_ds, 3541 drc->drc_keynvl, tx); 3542 if (error != 0) { 3543 dsl_dataset_rele(origin_head, FTAG); 3544 return (error); 3545 } 3546 } 3547 3548 error = dsl_dataset_clone_swap_check_impl(drc->drc_ds, 3549 origin_head, drc->drc_force, drc->drc_owner, tx); 3550 if (error != 0) { 3551 dsl_dataset_rele(origin_head, FTAG); 3552 return (error); 3553 } 3554 error = dsl_dataset_snapshot_check_impl(origin_head, 3555 drc->drc_tosnap, tx, B_TRUE, 1, 3556 drc->drc_cred, drc->drc_proc); 3557 dsl_dataset_rele(origin_head, FTAG); 3558 if (error != 0) 3559 return (error); 3560 3561 error = dsl_destroy_head_check_impl(drc->drc_ds, 1); 3562 } else { 3563 error = dsl_dataset_snapshot_check_impl(drc->drc_ds, 3564 drc->drc_tosnap, tx, B_TRUE, 1, 3565 drc->drc_cred, drc->drc_proc); 3566 } 3567 return (error); 3568 } 3569 3570 static void 3571 dmu_recv_end_sync(void *arg, dmu_tx_t *tx) 3572 { 3573 dmu_recv_cookie_t *drc = arg; 3574 dsl_pool_t *dp = dmu_tx_pool(tx); 3575 boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0; 3576 uint64_t newsnapobj = 0; 3577 3578 spa_history_log_internal_ds(drc->drc_ds, "finish receiving", 3579 tx, "snap=%s", drc->drc_tosnap); 3580 drc->drc_ds->ds_objset->os_raw_receive = B_FALSE; 3581 3582 if (drc->drc_heal) { 3583 if (drc->drc_keynvl != NULL) { 3584 nvlist_free(drc->drc_keynvl); 3585 drc->drc_keynvl = NULL; 3586 } 3587 } else if (!drc->drc_newfs) { 3588 dsl_dataset_t *origin_head; 3589 3590 VERIFY0(dsl_dataset_hold(dp, drc->drc_tofs, FTAG, 3591 &origin_head)); 3592 3593 if (drc->drc_force) { 3594 /* 3595 * Destroy any snapshots of drc_tofs (origin_head) 3596 * after the origin (the snap before drc_ds). 3597 */ 3598 uint64_t obj; 3599 3600 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj; 3601 while (obj != 3602 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) { 3603 dsl_dataset_t *snap; 3604 VERIFY0(dsl_dataset_hold_obj(dp, obj, FTAG, 3605 &snap)); 3606 ASSERT3P(snap->ds_dir, ==, origin_head->ds_dir); 3607 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj; 3608 dsl_destroy_snapshot_sync_impl(snap, 3609 B_FALSE, tx); 3610 dsl_dataset_rele(snap, FTAG); 3611 } 3612 } 3613 if (drc->drc_keynvl != NULL) { 3614 dsl_crypto_recv_raw_key_sync(drc->drc_ds, 3615 drc->drc_keynvl, tx); 3616 nvlist_free(drc->drc_keynvl); 3617 drc->drc_keynvl = NULL; 3618 } 3619 3620 VERIFY3P(drc->drc_ds->ds_prev, ==, 3621 origin_head->ds_prev); 3622 3623 dsl_dataset_clone_swap_sync_impl(drc->drc_ds, 3624 origin_head, tx); 3625 /* 3626 * The objset was evicted by dsl_dataset_clone_swap_sync_impl, 3627 * so drc_os is no longer valid. 3628 */ 3629 drc->drc_os = NULL; 3630 3631 dsl_dataset_snapshot_sync_impl(origin_head, 3632 drc->drc_tosnap, tx); 3633 3634 /* set snapshot's creation time and guid */ 3635 dmu_buf_will_dirty(origin_head->ds_prev->ds_dbuf, tx); 3636 dsl_dataset_phys(origin_head->ds_prev)->ds_creation_time = 3637 drc->drc_drrb->drr_creation_time; 3638 dsl_dataset_phys(origin_head->ds_prev)->ds_guid = 3639 drc->drc_drrb->drr_toguid; 3640 dsl_dataset_phys(origin_head->ds_prev)->ds_flags &= 3641 ~DS_FLAG_INCONSISTENT; 3642 3643 dmu_buf_will_dirty(origin_head->ds_dbuf, tx); 3644 dsl_dataset_phys(origin_head)->ds_flags &= 3645 ~DS_FLAG_INCONSISTENT; 3646 3647 newsnapobj = 3648 dsl_dataset_phys(origin_head)->ds_prev_snap_obj; 3649 3650 dsl_dataset_rele(origin_head, FTAG); 3651 dsl_destroy_head_sync_impl(drc->drc_ds, tx); 3652 3653 if (drc->drc_owner != NULL) 3654 VERIFY3P(origin_head->ds_owner, ==, drc->drc_owner); 3655 } else { 3656 dsl_dataset_t *ds = drc->drc_ds; 3657 3658 dsl_dataset_snapshot_sync_impl(ds, drc->drc_tosnap, tx); 3659 3660 /* set snapshot's creation time and guid */ 3661 dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx); 3662 dsl_dataset_phys(ds->ds_prev)->ds_creation_time = 3663 drc->drc_drrb->drr_creation_time; 3664 dsl_dataset_phys(ds->ds_prev)->ds_guid = 3665 drc->drc_drrb->drr_toguid; 3666 dsl_dataset_phys(ds->ds_prev)->ds_flags &= 3667 ~DS_FLAG_INCONSISTENT; 3668 3669 dmu_buf_will_dirty(ds->ds_dbuf, tx); 3670 dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT; 3671 if (dsl_dataset_has_resume_receive_state(ds)) { 3672 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3673 DS_FIELD_RESUME_FROMGUID, tx); 3674 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3675 DS_FIELD_RESUME_OBJECT, tx); 3676 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3677 DS_FIELD_RESUME_OFFSET, tx); 3678 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3679 DS_FIELD_RESUME_BYTES, tx); 3680 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3681 DS_FIELD_RESUME_TOGUID, tx); 3682 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3683 DS_FIELD_RESUME_TONAME, tx); 3684 (void) zap_remove(dp->dp_meta_objset, ds->ds_object, 3685 DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx); 3686 } 3687 newsnapobj = 3688 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj; 3689 } 3690 3691 /* 3692 * If this is a raw receive, the crypt_keydata nvlist will include 3693 * a to_ivset_guid for us to set on the new snapshot. This value 3694 * will override the value generated by the snapshot code. However, 3695 * this value may not be present, because older implementations of 3696 * the raw send code did not include this value, and we are still 3697 * allowed to receive them if the zfs_disable_ivset_guid_check 3698 * tunable is set, in which case we will leave the newly-generated 3699 * value. 3700 */ 3701 if (!drc->drc_heal && drc->drc_raw && drc->drc_ivset_guid != 0) { 3702 dmu_object_zapify(dp->dp_meta_objset, newsnapobj, 3703 DMU_OT_DSL_DATASET, tx); 3704 VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj, 3705 DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1, 3706 &drc->drc_ivset_guid, tx)); 3707 } 3708 3709 /* 3710 * Release the hold from dmu_recv_begin. This must be done before 3711 * we return to open context, so that when we free the dataset's dnode 3712 * we can evict its bonus buffer. Since the dataset may be destroyed 3713 * at this point (and therefore won't have a valid pointer to the spa) 3714 * we release the key mapping manually here while we do have a valid 3715 * pointer, if it exists. 3716 */ 3717 if (!drc->drc_raw && encrypted) { 3718 (void) spa_keystore_remove_mapping(dmu_tx_pool(tx)->dp_spa, 3719 drc->drc_ds->ds_object, drc->drc_ds); 3720 } 3721 dsl_dataset_disown(drc->drc_ds, 0, dmu_recv_tag); 3722 drc->drc_ds = NULL; 3723 } 3724 3725 static int dmu_recv_end_modified_blocks = 3; 3726 3727 static int 3728 dmu_recv_existing_end(dmu_recv_cookie_t *drc) 3729 { 3730 #ifdef _KERNEL 3731 /* 3732 * We will be destroying the ds; make sure its origin is unmounted if 3733 * necessary. 3734 */ 3735 char name[ZFS_MAX_DATASET_NAME_LEN]; 3736 dsl_dataset_name(drc->drc_ds, name); 3737 zfs_destroy_unmount_origin(name); 3738 #endif 3739 3740 return (dsl_sync_task(drc->drc_tofs, 3741 dmu_recv_end_check, dmu_recv_end_sync, drc, 3742 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL)); 3743 } 3744 3745 static int 3746 dmu_recv_new_end(dmu_recv_cookie_t *drc) 3747 { 3748 return (dsl_sync_task(drc->drc_tofs, 3749 dmu_recv_end_check, dmu_recv_end_sync, drc, 3750 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL)); 3751 } 3752 3753 int 3754 dmu_recv_end(dmu_recv_cookie_t *drc, void *owner) 3755 { 3756 int error; 3757 3758 drc->drc_owner = owner; 3759 3760 if (drc->drc_newfs) 3761 error = dmu_recv_new_end(drc); 3762 else 3763 error = dmu_recv_existing_end(drc); 3764 3765 if (error != 0) { 3766 dmu_recv_cleanup_ds(drc); 3767 nvlist_free(drc->drc_keynvl); 3768 } else if (!drc->drc_heal) { 3769 if (drc->drc_newfs) { 3770 zvol_create_minor(drc->drc_tofs); 3771 } 3772 char *snapname = kmem_asprintf("%s@%s", 3773 drc->drc_tofs, drc->drc_tosnap); 3774 zvol_create_minor(snapname); 3775 kmem_strfree(snapname); 3776 } 3777 return (error); 3778 } 3779 3780 /* 3781 * Return TRUE if this objset is currently being received into. 3782 */ 3783 boolean_t 3784 dmu_objset_is_receiving(objset_t *os) 3785 { 3786 return (os->os_dsl_dataset != NULL && 3787 os->os_dsl_dataset->ds_owner == dmu_recv_tag); 3788 } 3789 3790 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_length, UINT, ZMOD_RW, 3791 "Maximum receive queue length"); 3792 3793 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_ff, UINT, ZMOD_RW, 3794 "Receive queue fill fraction"); 3795 3796 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, write_batch_size, UINT, ZMOD_RW, 3797 "Maximum amount of writes to batch into one transaction"); 3798 3799 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, best_effort_corrective, INT, ZMOD_RW, 3800 "Ignore errors during corrective receive"); 3801 /* END CSTYLED */ 3802