xref: /freebsd/sys/contrib/openzfs/module/zfs/dsl_dataset.c (revision 924226fba12cc9a228c73b956e1b7fa24c60b055)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
25  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
26  * Copyright (c) 2014 RackTop Systems.
27  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
28  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
29  * Copyright 2016, OmniTI Computer Consulting, Inc. All rights reserved.
30  * Copyright 2017 Nexenta Systems, Inc.
31  * Copyright (c) 2019, Klara Inc.
32  * Copyright (c) 2019, Allan Jude
33  * Copyright (c) 2020 The FreeBSD Foundation [1]
34  *
35  * [1] Portions of this software were developed by Allan Jude
36  *     under sponsorship from the FreeBSD Foundation.
37  */
38 
39 #include <sys/dmu_objset.h>
40 #include <sys/dsl_dataset.h>
41 #include <sys/dsl_dir.h>
42 #include <sys/dsl_prop.h>
43 #include <sys/dsl_synctask.h>
44 #include <sys/dmu_traverse.h>
45 #include <sys/dmu_impl.h>
46 #include <sys/dmu_tx.h>
47 #include <sys/arc.h>
48 #include <sys/zio.h>
49 #include <sys/zap.h>
50 #include <sys/zfeature.h>
51 #include <sys/unique.h>
52 #include <sys/zfs_context.h>
53 #include <sys/zfs_ioctl.h>
54 #include <sys/spa.h>
55 #include <sys/spa_impl.h>
56 #include <sys/vdev.h>
57 #include <sys/zfs_znode.h>
58 #include <sys/zfs_onexit.h>
59 #include <sys/zvol.h>
60 #include <sys/dsl_scan.h>
61 #include <sys/dsl_deadlist.h>
62 #include <sys/dsl_destroy.h>
63 #include <sys/dsl_userhold.h>
64 #include <sys/dsl_bookmark.h>
65 #include <sys/policy.h>
66 #include <sys/dmu_send.h>
67 #include <sys/dmu_recv.h>
68 #include <sys/zio_compress.h>
69 #include <zfs_fletcher.h>
70 #include <sys/zio_checksum.h>
71 
72 /*
73  * The SPA supports block sizes up to 16MB.  However, very large blocks
74  * can have an impact on i/o latency (e.g. tying up a spinning disk for
75  * ~300ms), and also potentially on the memory allocator.  Therefore,
76  * we did not allow the recordsize to be set larger than zfs_max_recordsize
77  * (former default: 1MB).  Larger blocks could be created by changing this
78  * tunable, and pools with larger blocks could always be imported and used,
79  * regardless of this setting.
80  *
81  * We do, however, still limit it by default to 1M on x86_32, because Linux's
82  * 3/1 memory split doesn't leave much room for 16M chunks.
83  */
84 #ifdef _ILP32
85 int zfs_max_recordsize =  1 * 1024 * 1024;
86 #else
87 int zfs_max_recordsize = 16 * 1024 * 1024;
88 #endif
89 static int zfs_allow_redacted_dataset_mount = 0;
90 
91 #define	SWITCH64(x, y) \
92 	{ \
93 		uint64_t __tmp = (x); \
94 		(x) = (y); \
95 		(y) = __tmp; \
96 	}
97 
98 #define	DS_REF_MAX	(1ULL << 62)
99 
100 static void dsl_dataset_set_remap_deadlist_object(dsl_dataset_t *ds,
101     uint64_t obj, dmu_tx_t *tx);
102 static void dsl_dataset_unset_remap_deadlist_object(dsl_dataset_t *ds,
103     dmu_tx_t *tx);
104 
105 static void unload_zfeature(dsl_dataset_t *ds, spa_feature_t f);
106 
107 extern int spa_asize_inflation;
108 
109 static zil_header_t zero_zil;
110 
111 /*
112  * Figure out how much of this delta should be propagated to the dsl_dir
113  * layer.  If there's a refreservation, that space has already been
114  * partially accounted for in our ancestors.
115  */
116 static int64_t
117 parent_delta(dsl_dataset_t *ds, int64_t delta)
118 {
119 	dsl_dataset_phys_t *ds_phys;
120 	uint64_t old_bytes, new_bytes;
121 
122 	if (ds->ds_reserved == 0)
123 		return (delta);
124 
125 	ds_phys = dsl_dataset_phys(ds);
126 	old_bytes = MAX(ds_phys->ds_unique_bytes, ds->ds_reserved);
127 	new_bytes = MAX(ds_phys->ds_unique_bytes + delta, ds->ds_reserved);
128 
129 	ASSERT3U(ABS((int64_t)(new_bytes - old_bytes)), <=, ABS(delta));
130 	return (new_bytes - old_bytes);
131 }
132 
133 void
134 dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx)
135 {
136 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
137 	int used = bp_get_dsize_sync(spa, bp);
138 	int compressed = BP_GET_PSIZE(bp);
139 	int uncompressed = BP_GET_UCSIZE(bp);
140 	int64_t delta;
141 	spa_feature_t f;
142 
143 	dprintf_bp(bp, "ds=%p", ds);
144 
145 	ASSERT(dmu_tx_is_syncing(tx));
146 	/* It could have been compressed away to nothing */
147 	if (BP_IS_HOLE(bp) || BP_IS_REDACTED(bp))
148 		return;
149 	ASSERT(BP_GET_TYPE(bp) != DMU_OT_NONE);
150 	ASSERT(DMU_OT_IS_VALID(BP_GET_TYPE(bp)));
151 	if (ds == NULL) {
152 		dsl_pool_mos_diduse_space(tx->tx_pool,
153 		    used, compressed, uncompressed);
154 		return;
155 	}
156 
157 	ASSERT3U(bp->blk_birth, >, dsl_dataset_phys(ds)->ds_prev_snap_txg);
158 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
159 	mutex_enter(&ds->ds_lock);
160 	delta = parent_delta(ds, used);
161 	dsl_dataset_phys(ds)->ds_referenced_bytes += used;
162 	dsl_dataset_phys(ds)->ds_compressed_bytes += compressed;
163 	dsl_dataset_phys(ds)->ds_uncompressed_bytes += uncompressed;
164 	dsl_dataset_phys(ds)->ds_unique_bytes += used;
165 
166 	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE) {
167 		ds->ds_feature_activation[SPA_FEATURE_LARGE_BLOCKS] =
168 		    (void *)B_TRUE;
169 	}
170 
171 
172 	f = zio_checksum_to_feature(BP_GET_CHECKSUM(bp));
173 	if (f != SPA_FEATURE_NONE) {
174 		ASSERT3S(spa_feature_table[f].fi_type, ==,
175 		    ZFEATURE_TYPE_BOOLEAN);
176 		ds->ds_feature_activation[f] = (void *)B_TRUE;
177 	}
178 
179 	f = zio_compress_to_feature(BP_GET_COMPRESS(bp));
180 	if (f != SPA_FEATURE_NONE) {
181 		ASSERT3S(spa_feature_table[f].fi_type, ==,
182 		    ZFEATURE_TYPE_BOOLEAN);
183 		ds->ds_feature_activation[f] = (void *)B_TRUE;
184 	}
185 
186 	/*
187 	 * Track block for livelist, but ignore embedded blocks because
188 	 * they do not need to be freed.
189 	 */
190 	if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist) &&
191 	    bp->blk_birth > ds->ds_dir->dd_origin_txg &&
192 	    !(BP_IS_EMBEDDED(bp))) {
193 		ASSERT(dsl_dir_is_clone(ds->ds_dir));
194 		ASSERT(spa_feature_is_enabled(spa,
195 		    SPA_FEATURE_LIVELIST));
196 		bplist_append(&ds->ds_dir->dd_pending_allocs, bp);
197 	}
198 
199 	mutex_exit(&ds->ds_lock);
200 	dsl_dir_diduse_transfer_space(ds->ds_dir, delta,
201 	    compressed, uncompressed, used,
202 	    DD_USED_REFRSRV, DD_USED_HEAD, tx);
203 }
204 
205 /*
206  * Called when the specified segment has been remapped, and is thus no
207  * longer referenced in the head dataset.  The vdev must be indirect.
208  *
209  * If the segment is referenced by a snapshot, put it on the remap deadlist.
210  * Otherwise, add this segment to the obsolete spacemap.
211  */
212 void
213 dsl_dataset_block_remapped(dsl_dataset_t *ds, uint64_t vdev, uint64_t offset,
214     uint64_t size, uint64_t birth, dmu_tx_t *tx)
215 {
216 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
217 
218 	ASSERT(dmu_tx_is_syncing(tx));
219 	ASSERT(birth <= tx->tx_txg);
220 	ASSERT(!ds->ds_is_snapshot);
221 
222 	if (birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
223 		spa_vdev_indirect_mark_obsolete(spa, vdev, offset, size, tx);
224 	} else {
225 		blkptr_t fakebp;
226 		dva_t *dva = &fakebp.blk_dva[0];
227 
228 		ASSERT(ds != NULL);
229 
230 		mutex_enter(&ds->ds_remap_deadlist_lock);
231 		if (!dsl_dataset_remap_deadlist_exists(ds)) {
232 			dsl_dataset_create_remap_deadlist(ds, tx);
233 		}
234 		mutex_exit(&ds->ds_remap_deadlist_lock);
235 
236 		BP_ZERO(&fakebp);
237 		fakebp.blk_birth = birth;
238 		DVA_SET_VDEV(dva, vdev);
239 		DVA_SET_OFFSET(dva, offset);
240 		DVA_SET_ASIZE(dva, size);
241 		dsl_deadlist_insert(&ds->ds_remap_deadlist, &fakebp, B_FALSE,
242 		    tx);
243 	}
244 }
245 
246 int
247 dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
248     boolean_t async)
249 {
250 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
251 
252 	int used = bp_get_dsize_sync(spa, bp);
253 	int compressed = BP_GET_PSIZE(bp);
254 	int uncompressed = BP_GET_UCSIZE(bp);
255 
256 	if (BP_IS_HOLE(bp) || BP_IS_REDACTED(bp))
257 		return (0);
258 
259 	ASSERT(dmu_tx_is_syncing(tx));
260 	ASSERT(bp->blk_birth <= tx->tx_txg);
261 
262 	if (ds == NULL) {
263 		dsl_free(tx->tx_pool, tx->tx_txg, bp);
264 		dsl_pool_mos_diduse_space(tx->tx_pool,
265 		    -used, -compressed, -uncompressed);
266 		return (used);
267 	}
268 	ASSERT3P(tx->tx_pool, ==, ds->ds_dir->dd_pool);
269 
270 	ASSERT(!ds->ds_is_snapshot);
271 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
272 
273 	/*
274 	 * Track block for livelist, but ignore embedded blocks because
275 	 * they do not need to be freed.
276 	 */
277 	if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist) &&
278 	    bp->blk_birth > ds->ds_dir->dd_origin_txg &&
279 	    !(BP_IS_EMBEDDED(bp))) {
280 		ASSERT(dsl_dir_is_clone(ds->ds_dir));
281 		ASSERT(spa_feature_is_enabled(spa,
282 		    SPA_FEATURE_LIVELIST));
283 		bplist_append(&ds->ds_dir->dd_pending_frees, bp);
284 	}
285 
286 	if (bp->blk_birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
287 		int64_t delta;
288 
289 		dprintf_bp(bp, "freeing ds=%llu", (u_longlong_t)ds->ds_object);
290 		dsl_free(tx->tx_pool, tx->tx_txg, bp);
291 
292 		mutex_enter(&ds->ds_lock);
293 		ASSERT(dsl_dataset_phys(ds)->ds_unique_bytes >= used ||
294 		    !DS_UNIQUE_IS_ACCURATE(ds));
295 		delta = parent_delta(ds, -used);
296 		dsl_dataset_phys(ds)->ds_unique_bytes -= used;
297 		mutex_exit(&ds->ds_lock);
298 		dsl_dir_diduse_transfer_space(ds->ds_dir,
299 		    delta, -compressed, -uncompressed, -used,
300 		    DD_USED_REFRSRV, DD_USED_HEAD, tx);
301 	} else {
302 		dprintf_bp(bp, "putting on dead list: %s", "");
303 		if (async) {
304 			/*
305 			 * We are here as part of zio's write done callback,
306 			 * which means we're a zio interrupt thread.  We can't
307 			 * call dsl_deadlist_insert() now because it may block
308 			 * waiting for I/O.  Instead, put bp on the deferred
309 			 * queue and let dsl_pool_sync() finish the job.
310 			 */
311 			bplist_append(&ds->ds_pending_deadlist, bp);
312 		} else {
313 			dsl_deadlist_insert(&ds->ds_deadlist, bp, B_FALSE, tx);
314 		}
315 		ASSERT3U(ds->ds_prev->ds_object, ==,
316 		    dsl_dataset_phys(ds)->ds_prev_snap_obj);
317 		ASSERT(dsl_dataset_phys(ds->ds_prev)->ds_num_children > 0);
318 		/* if (bp->blk_birth > prev prev snap txg) prev unique += bs */
319 		if (dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
320 		    ds->ds_object && bp->blk_birth >
321 		    dsl_dataset_phys(ds->ds_prev)->ds_prev_snap_txg) {
322 			dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
323 			mutex_enter(&ds->ds_prev->ds_lock);
324 			dsl_dataset_phys(ds->ds_prev)->ds_unique_bytes += used;
325 			mutex_exit(&ds->ds_prev->ds_lock);
326 		}
327 		if (bp->blk_birth > ds->ds_dir->dd_origin_txg) {
328 			dsl_dir_transfer_space(ds->ds_dir, used,
329 			    DD_USED_HEAD, DD_USED_SNAP, tx);
330 		}
331 	}
332 
333 	dsl_bookmark_block_killed(ds, bp, tx);
334 
335 	mutex_enter(&ds->ds_lock);
336 	ASSERT3U(dsl_dataset_phys(ds)->ds_referenced_bytes, >=, used);
337 	dsl_dataset_phys(ds)->ds_referenced_bytes -= used;
338 	ASSERT3U(dsl_dataset_phys(ds)->ds_compressed_bytes, >=, compressed);
339 	dsl_dataset_phys(ds)->ds_compressed_bytes -= compressed;
340 	ASSERT3U(dsl_dataset_phys(ds)->ds_uncompressed_bytes, >=, uncompressed);
341 	dsl_dataset_phys(ds)->ds_uncompressed_bytes -= uncompressed;
342 	mutex_exit(&ds->ds_lock);
343 
344 	return (used);
345 }
346 
347 struct feature_type_uint64_array_arg {
348 	uint64_t length;
349 	uint64_t *array;
350 };
351 
352 static void
353 unload_zfeature(dsl_dataset_t *ds, spa_feature_t f)
354 {
355 	switch (spa_feature_table[f].fi_type) {
356 	case ZFEATURE_TYPE_BOOLEAN:
357 		break;
358 	case ZFEATURE_TYPE_UINT64_ARRAY:
359 	{
360 		struct feature_type_uint64_array_arg *ftuaa = ds->ds_feature[f];
361 		kmem_free(ftuaa->array, ftuaa->length * sizeof (uint64_t));
362 		kmem_free(ftuaa, sizeof (*ftuaa));
363 		break;
364 	}
365 	default:
366 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
367 	}
368 }
369 
370 static int
371 load_zfeature(objset_t *mos, dsl_dataset_t *ds, spa_feature_t f)
372 {
373 	int err = 0;
374 	switch (spa_feature_table[f].fi_type) {
375 	case ZFEATURE_TYPE_BOOLEAN:
376 		err = zap_contains(mos, ds->ds_object,
377 		    spa_feature_table[f].fi_guid);
378 		if (err == 0) {
379 			ds->ds_feature[f] = (void *)B_TRUE;
380 		} else {
381 			ASSERT3U(err, ==, ENOENT);
382 			err = 0;
383 		}
384 		break;
385 	case ZFEATURE_TYPE_UINT64_ARRAY:
386 	{
387 		uint64_t int_size, num_int;
388 		uint64_t *data;
389 		err = zap_length(mos, ds->ds_object,
390 		    spa_feature_table[f].fi_guid, &int_size, &num_int);
391 		if (err != 0) {
392 			ASSERT3U(err, ==, ENOENT);
393 			err = 0;
394 			break;
395 		}
396 		ASSERT3U(int_size, ==, sizeof (uint64_t));
397 		data = kmem_alloc(int_size * num_int, KM_SLEEP);
398 		VERIFY0(zap_lookup(mos, ds->ds_object,
399 		    spa_feature_table[f].fi_guid, int_size, num_int, data));
400 		struct feature_type_uint64_array_arg *ftuaa =
401 		    kmem_alloc(sizeof (*ftuaa), KM_SLEEP);
402 		ftuaa->length = num_int;
403 		ftuaa->array = data;
404 		ds->ds_feature[f] = ftuaa;
405 		break;
406 	}
407 	default:
408 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
409 	}
410 	return (err);
411 }
412 
413 /*
414  * We have to release the fsid synchronously or we risk that a subsequent
415  * mount of the same dataset will fail to unique_insert the fsid.  This
416  * failure would manifest itself as the fsid of this dataset changing
417  * between mounts which makes NFS clients quite unhappy.
418  */
419 static void
420 dsl_dataset_evict_sync(void *dbu)
421 {
422 	dsl_dataset_t *ds = dbu;
423 
424 	ASSERT(ds->ds_owner == NULL);
425 
426 	unique_remove(ds->ds_fsid_guid);
427 }
428 
429 static void
430 dsl_dataset_evict_async(void *dbu)
431 {
432 	dsl_dataset_t *ds = dbu;
433 
434 	ASSERT(ds->ds_owner == NULL);
435 
436 	ds->ds_dbuf = NULL;
437 
438 	if (ds->ds_objset != NULL)
439 		dmu_objset_evict(ds->ds_objset);
440 
441 	if (ds->ds_prev) {
442 		dsl_dataset_rele(ds->ds_prev, ds);
443 		ds->ds_prev = NULL;
444 	}
445 
446 	dsl_bookmark_fini_ds(ds);
447 
448 	bplist_destroy(&ds->ds_pending_deadlist);
449 	if (dsl_deadlist_is_open(&ds->ds_deadlist))
450 		dsl_deadlist_close(&ds->ds_deadlist);
451 	if (dsl_deadlist_is_open(&ds->ds_remap_deadlist))
452 		dsl_deadlist_close(&ds->ds_remap_deadlist);
453 	if (ds->ds_dir)
454 		dsl_dir_async_rele(ds->ds_dir, ds);
455 
456 	ASSERT(!list_link_active(&ds->ds_synced_link));
457 
458 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
459 		if (dsl_dataset_feature_is_active(ds, f))
460 			unload_zfeature(ds, f);
461 	}
462 
463 	list_destroy(&ds->ds_prop_cbs);
464 	mutex_destroy(&ds->ds_lock);
465 	mutex_destroy(&ds->ds_opening_lock);
466 	mutex_destroy(&ds->ds_sendstream_lock);
467 	mutex_destroy(&ds->ds_remap_deadlist_lock);
468 	zfs_refcount_destroy(&ds->ds_longholds);
469 	rrw_destroy(&ds->ds_bp_rwlock);
470 
471 	kmem_free(ds, sizeof (dsl_dataset_t));
472 }
473 
474 int
475 dsl_dataset_get_snapname(dsl_dataset_t *ds)
476 {
477 	dsl_dataset_phys_t *headphys;
478 	int err;
479 	dmu_buf_t *headdbuf;
480 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
481 	objset_t *mos = dp->dp_meta_objset;
482 
483 	if (ds->ds_snapname[0])
484 		return (0);
485 	if (dsl_dataset_phys(ds)->ds_next_snap_obj == 0)
486 		return (0);
487 
488 	err = dmu_bonus_hold(mos, dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj,
489 	    FTAG, &headdbuf);
490 	if (err != 0)
491 		return (err);
492 	headphys = headdbuf->db_data;
493 	err = zap_value_search(dp->dp_meta_objset,
494 	    headphys->ds_snapnames_zapobj, ds->ds_object, 0, ds->ds_snapname);
495 	if (err != 0 && zfs_recover == B_TRUE) {
496 		err = 0;
497 		(void) snprintf(ds->ds_snapname, sizeof (ds->ds_snapname),
498 		    "SNAPOBJ=%llu-ERR=%d",
499 		    (unsigned long long)ds->ds_object, err);
500 	}
501 	dmu_buf_rele(headdbuf, FTAG);
502 	return (err);
503 }
504 
505 int
506 dsl_dataset_snap_lookup(dsl_dataset_t *ds, const char *name, uint64_t *value)
507 {
508 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
509 	uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
510 	matchtype_t mt = 0;
511 	int err;
512 
513 	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
514 		mt = MT_NORMALIZE;
515 
516 	err = zap_lookup_norm(mos, snapobj, name, 8, 1,
517 	    value, mt, NULL, 0, NULL);
518 	if (err == ENOTSUP && (mt & MT_NORMALIZE))
519 		err = zap_lookup(mos, snapobj, name, 8, 1, value);
520 	return (err);
521 }
522 
523 int
524 dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx,
525     boolean_t adj_cnt)
526 {
527 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
528 	uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
529 	matchtype_t mt = 0;
530 	int err;
531 
532 	dsl_dir_snap_cmtime_update(ds->ds_dir);
533 
534 	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
535 		mt = MT_NORMALIZE;
536 
537 	err = zap_remove_norm(mos, snapobj, name, mt, tx);
538 	if (err == ENOTSUP && (mt & MT_NORMALIZE))
539 		err = zap_remove(mos, snapobj, name, tx);
540 
541 	if (err == 0 && adj_cnt)
542 		dsl_fs_ss_count_adjust(ds->ds_dir, -1,
543 		    DD_FIELD_SNAPSHOT_COUNT, tx);
544 
545 	return (err);
546 }
547 
548 boolean_t
549 dsl_dataset_try_add_ref(dsl_pool_t *dp, dsl_dataset_t *ds, void *tag)
550 {
551 	dmu_buf_t *dbuf = ds->ds_dbuf;
552 	boolean_t result = B_FALSE;
553 
554 	if (dbuf != NULL && dmu_buf_try_add_ref(dbuf, dp->dp_meta_objset,
555 	    ds->ds_object, DMU_BONUS_BLKID, tag)) {
556 
557 		if (ds == dmu_buf_get_user(dbuf))
558 			result = B_TRUE;
559 		else
560 			dmu_buf_rele(dbuf, tag);
561 	}
562 
563 	return (result);
564 }
565 
566 int
567 dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
568     dsl_dataset_t **dsp)
569 {
570 	objset_t *mos = dp->dp_meta_objset;
571 	dmu_buf_t *dbuf;
572 	dsl_dataset_t *ds;
573 	int err;
574 	dmu_object_info_t doi;
575 
576 	ASSERT(dsl_pool_config_held(dp));
577 
578 	err = dmu_bonus_hold(mos, dsobj, tag, &dbuf);
579 	if (err != 0)
580 		return (err);
581 
582 	/* Make sure dsobj has the correct object type. */
583 	dmu_object_info_from_db(dbuf, &doi);
584 	if (doi.doi_bonus_type != DMU_OT_DSL_DATASET) {
585 		dmu_buf_rele(dbuf, tag);
586 		return (SET_ERROR(EINVAL));
587 	}
588 
589 	ds = dmu_buf_get_user(dbuf);
590 	if (ds == NULL) {
591 		dsl_dataset_t *winner = NULL;
592 
593 		ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP);
594 		ds->ds_dbuf = dbuf;
595 		ds->ds_object = dsobj;
596 		ds->ds_is_snapshot = dsl_dataset_phys(ds)->ds_num_children != 0;
597 		list_link_init(&ds->ds_synced_link);
598 
599 		err = dsl_dir_hold_obj(dp, dsl_dataset_phys(ds)->ds_dir_obj,
600 		    NULL, ds, &ds->ds_dir);
601 		if (err != 0) {
602 			kmem_free(ds, sizeof (dsl_dataset_t));
603 			dmu_buf_rele(dbuf, tag);
604 			return (err);
605 		}
606 
607 		mutex_init(&ds->ds_lock, NULL, MUTEX_DEFAULT, NULL);
608 		mutex_init(&ds->ds_opening_lock, NULL, MUTEX_DEFAULT, NULL);
609 		mutex_init(&ds->ds_sendstream_lock, NULL, MUTEX_DEFAULT, NULL);
610 		mutex_init(&ds->ds_remap_deadlist_lock,
611 		    NULL, MUTEX_DEFAULT, NULL);
612 		rrw_init(&ds->ds_bp_rwlock, B_FALSE);
613 		zfs_refcount_create(&ds->ds_longholds);
614 
615 		bplist_create(&ds->ds_pending_deadlist);
616 
617 		list_create(&ds->ds_sendstreams, sizeof (dmu_sendstatus_t),
618 		    offsetof(dmu_sendstatus_t, dss_link));
619 
620 		list_create(&ds->ds_prop_cbs, sizeof (dsl_prop_cb_record_t),
621 		    offsetof(dsl_prop_cb_record_t, cbr_ds_node));
622 
623 		if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
624 			spa_feature_t f;
625 
626 			for (f = 0; f < SPA_FEATURES; f++) {
627 				if (!(spa_feature_table[f].fi_flags &
628 				    ZFEATURE_FLAG_PER_DATASET))
629 					continue;
630 				err = load_zfeature(mos, ds, f);
631 			}
632 		}
633 
634 		if (!ds->ds_is_snapshot) {
635 			ds->ds_snapname[0] = '\0';
636 			if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
637 				err = dsl_dataset_hold_obj(dp,
638 				    dsl_dataset_phys(ds)->ds_prev_snap_obj,
639 				    ds, &ds->ds_prev);
640 			}
641 			err = dsl_bookmark_init_ds(ds);
642 		} else {
643 			if (zfs_flags & ZFS_DEBUG_SNAPNAMES)
644 				err = dsl_dataset_get_snapname(ds);
645 			if (err == 0 &&
646 			    dsl_dataset_phys(ds)->ds_userrefs_obj != 0) {
647 				err = zap_count(
648 				    ds->ds_dir->dd_pool->dp_meta_objset,
649 				    dsl_dataset_phys(ds)->ds_userrefs_obj,
650 				    &ds->ds_userrefs);
651 			}
652 		}
653 
654 		if (err == 0 && !ds->ds_is_snapshot) {
655 			err = dsl_prop_get_int_ds(ds,
656 			    zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
657 			    &ds->ds_reserved);
658 			if (err == 0) {
659 				err = dsl_prop_get_int_ds(ds,
660 				    zfs_prop_to_name(ZFS_PROP_REFQUOTA),
661 				    &ds->ds_quota);
662 			}
663 		} else {
664 			ds->ds_reserved = ds->ds_quota = 0;
665 		}
666 
667 		if (err == 0 && ds->ds_dir->dd_crypto_obj != 0 &&
668 		    ds->ds_is_snapshot &&
669 		    zap_contains(mos, dsobj, DS_FIELD_IVSET_GUID) != 0) {
670 			dp->dp_spa->spa_errata =
671 			    ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
672 		}
673 
674 		dsl_deadlist_open(&ds->ds_deadlist,
675 		    mos, dsl_dataset_phys(ds)->ds_deadlist_obj);
676 		uint64_t remap_deadlist_obj =
677 		    dsl_dataset_get_remap_deadlist_object(ds);
678 		if (remap_deadlist_obj != 0) {
679 			dsl_deadlist_open(&ds->ds_remap_deadlist, mos,
680 			    remap_deadlist_obj);
681 		}
682 
683 		dmu_buf_init_user(&ds->ds_dbu, dsl_dataset_evict_sync,
684 		    dsl_dataset_evict_async, &ds->ds_dbuf);
685 		if (err == 0)
686 			winner = dmu_buf_set_user_ie(dbuf, &ds->ds_dbu);
687 
688 		if (err != 0 || winner != NULL) {
689 			bplist_destroy(&ds->ds_pending_deadlist);
690 			dsl_deadlist_close(&ds->ds_deadlist);
691 			if (dsl_deadlist_is_open(&ds->ds_remap_deadlist))
692 				dsl_deadlist_close(&ds->ds_remap_deadlist);
693 			dsl_bookmark_fini_ds(ds);
694 			if (ds->ds_prev)
695 				dsl_dataset_rele(ds->ds_prev, ds);
696 			dsl_dir_rele(ds->ds_dir, ds);
697 			for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
698 				if (dsl_dataset_feature_is_active(ds, f))
699 					unload_zfeature(ds, f);
700 			}
701 
702 			list_destroy(&ds->ds_prop_cbs);
703 			list_destroy(&ds->ds_sendstreams);
704 			mutex_destroy(&ds->ds_lock);
705 			mutex_destroy(&ds->ds_opening_lock);
706 			mutex_destroy(&ds->ds_sendstream_lock);
707 			mutex_destroy(&ds->ds_remap_deadlist_lock);
708 			zfs_refcount_destroy(&ds->ds_longholds);
709 			rrw_destroy(&ds->ds_bp_rwlock);
710 			kmem_free(ds, sizeof (dsl_dataset_t));
711 			if (err != 0) {
712 				dmu_buf_rele(dbuf, tag);
713 				return (err);
714 			}
715 			ds = winner;
716 		} else {
717 			ds->ds_fsid_guid =
718 			    unique_insert(dsl_dataset_phys(ds)->ds_fsid_guid);
719 			if (ds->ds_fsid_guid !=
720 			    dsl_dataset_phys(ds)->ds_fsid_guid) {
721 				zfs_dbgmsg("ds_fsid_guid changed from "
722 				    "%llx to %llx for pool %s dataset id %llu",
723 				    (long long)
724 				    dsl_dataset_phys(ds)->ds_fsid_guid,
725 				    (long long)ds->ds_fsid_guid,
726 				    spa_name(dp->dp_spa),
727 				    (u_longlong_t)dsobj);
728 			}
729 		}
730 	}
731 
732 	ASSERT3P(ds->ds_dbuf, ==, dbuf);
733 	ASSERT3P(dsl_dataset_phys(ds), ==, dbuf->db_data);
734 	ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0 ||
735 	    spa_version(dp->dp_spa) < SPA_VERSION_ORIGIN ||
736 	    dp->dp_origin_snap == NULL || ds == dp->dp_origin_snap);
737 	*dsp = ds;
738 
739 	return (0);
740 }
741 
742 int
743 dsl_dataset_create_key_mapping(dsl_dataset_t *ds)
744 {
745 	dsl_dir_t *dd = ds->ds_dir;
746 
747 	if (dd->dd_crypto_obj == 0)
748 		return (0);
749 
750 	return (spa_keystore_create_mapping(dd->dd_pool->dp_spa,
751 	    ds, ds, &ds->ds_key_mapping));
752 }
753 
754 int
755 dsl_dataset_hold_obj_flags(dsl_pool_t *dp, uint64_t dsobj,
756     ds_hold_flags_t flags, void *tag, dsl_dataset_t **dsp)
757 {
758 	int err;
759 
760 	err = dsl_dataset_hold_obj(dp, dsobj, tag, dsp);
761 	if (err != 0)
762 		return (err);
763 
764 	ASSERT3P(*dsp, !=, NULL);
765 
766 	if (flags & DS_HOLD_FLAG_DECRYPT) {
767 		err = dsl_dataset_create_key_mapping(*dsp);
768 		if (err != 0)
769 			dsl_dataset_rele(*dsp, tag);
770 	}
771 
772 	return (err);
773 }
774 
775 int
776 dsl_dataset_hold_flags(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
777     void *tag, dsl_dataset_t **dsp)
778 {
779 	dsl_dir_t *dd;
780 	const char *snapname;
781 	uint64_t obj;
782 	int err = 0;
783 	dsl_dataset_t *ds;
784 
785 	err = dsl_dir_hold(dp, name, FTAG, &dd, &snapname);
786 	if (err != 0)
787 		return (err);
788 
789 	ASSERT(dsl_pool_config_held(dp));
790 	obj = dsl_dir_phys(dd)->dd_head_dataset_obj;
791 	if (obj != 0)
792 		err = dsl_dataset_hold_obj_flags(dp, obj, flags, tag, &ds);
793 	else
794 		err = SET_ERROR(ENOENT);
795 
796 	/* we may be looking for a snapshot */
797 	if (err == 0 && snapname != NULL) {
798 		dsl_dataset_t *snap_ds;
799 
800 		if (*snapname++ != '@') {
801 			dsl_dataset_rele_flags(ds, flags, tag);
802 			dsl_dir_rele(dd, FTAG);
803 			return (SET_ERROR(ENOENT));
804 		}
805 
806 		dprintf("looking for snapshot '%s'\n", snapname);
807 		err = dsl_dataset_snap_lookup(ds, snapname, &obj);
808 		if (err == 0) {
809 			err = dsl_dataset_hold_obj_flags(dp, obj, flags, tag,
810 			    &snap_ds);
811 		}
812 		dsl_dataset_rele_flags(ds, flags, tag);
813 
814 		if (err == 0) {
815 			mutex_enter(&snap_ds->ds_lock);
816 			if (snap_ds->ds_snapname[0] == 0)
817 				(void) strlcpy(snap_ds->ds_snapname, snapname,
818 				    sizeof (snap_ds->ds_snapname));
819 			mutex_exit(&snap_ds->ds_lock);
820 			ds = snap_ds;
821 		}
822 	}
823 	if (err == 0)
824 		*dsp = ds;
825 	dsl_dir_rele(dd, FTAG);
826 	return (err);
827 }
828 
829 int
830 dsl_dataset_hold(dsl_pool_t *dp, const char *name, void *tag,
831     dsl_dataset_t **dsp)
832 {
833 	return (dsl_dataset_hold_flags(dp, name, 0, tag, dsp));
834 }
835 
836 static int
837 dsl_dataset_own_obj_impl(dsl_pool_t *dp, uint64_t dsobj, ds_hold_flags_t flags,
838     void *tag, boolean_t override, dsl_dataset_t **dsp)
839 {
840 	int err = dsl_dataset_hold_obj_flags(dp, dsobj, flags, tag, dsp);
841 	if (err != 0)
842 		return (err);
843 	if (!dsl_dataset_tryown(*dsp, tag, override)) {
844 		dsl_dataset_rele_flags(*dsp, flags, tag);
845 		*dsp = NULL;
846 		return (SET_ERROR(EBUSY));
847 	}
848 	return (0);
849 }
850 
851 
852 int
853 dsl_dataset_own_obj(dsl_pool_t *dp, uint64_t dsobj, ds_hold_flags_t flags,
854     void *tag, dsl_dataset_t **dsp)
855 {
856 	return (dsl_dataset_own_obj_impl(dp, dsobj, flags, tag, B_FALSE, dsp));
857 }
858 
859 int
860 dsl_dataset_own_obj_force(dsl_pool_t *dp, uint64_t dsobj,
861     ds_hold_flags_t flags, void *tag, dsl_dataset_t **dsp)
862 {
863 	return (dsl_dataset_own_obj_impl(dp, dsobj, flags, tag, B_TRUE, dsp));
864 }
865 
866 static int
867 dsl_dataset_own_impl(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
868     void *tag, boolean_t override, dsl_dataset_t **dsp)
869 {
870 	int err = dsl_dataset_hold_flags(dp, name, flags, tag, dsp);
871 	if (err != 0)
872 		return (err);
873 	if (!dsl_dataset_tryown(*dsp, tag, override)) {
874 		dsl_dataset_rele_flags(*dsp, flags, tag);
875 		return (SET_ERROR(EBUSY));
876 	}
877 	return (0);
878 }
879 
880 int
881 dsl_dataset_own_force(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
882     void *tag, dsl_dataset_t **dsp)
883 {
884 	return (dsl_dataset_own_impl(dp, name, flags, tag, B_TRUE, dsp));
885 }
886 
887 int
888 dsl_dataset_own(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
889     void *tag, dsl_dataset_t **dsp)
890 {
891 	return (dsl_dataset_own_impl(dp, name, flags, tag, B_FALSE, dsp));
892 }
893 
894 /*
895  * See the comment above dsl_pool_hold() for details.  In summary, a long
896  * hold is used to prevent destruction of a dataset while the pool hold
897  * is dropped, allowing other concurrent operations (e.g. spa_sync()).
898  *
899  * The dataset and pool must be held when this function is called.  After it
900  * is called, the pool hold may be released while the dataset is still held
901  * and accessed.
902  */
903 void
904 dsl_dataset_long_hold(dsl_dataset_t *ds, const void *tag)
905 {
906 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
907 	(void) zfs_refcount_add(&ds->ds_longholds, tag);
908 }
909 
910 void
911 dsl_dataset_long_rele(dsl_dataset_t *ds, const void *tag)
912 {
913 	(void) zfs_refcount_remove(&ds->ds_longholds, tag);
914 }
915 
916 /* Return B_TRUE if there are any long holds on this dataset. */
917 boolean_t
918 dsl_dataset_long_held(dsl_dataset_t *ds)
919 {
920 	return (!zfs_refcount_is_zero(&ds->ds_longholds));
921 }
922 
923 void
924 dsl_dataset_name(dsl_dataset_t *ds, char *name)
925 {
926 	if (ds == NULL) {
927 		(void) strlcpy(name, "mos", ZFS_MAX_DATASET_NAME_LEN);
928 	} else {
929 		dsl_dir_name(ds->ds_dir, name);
930 		VERIFY0(dsl_dataset_get_snapname(ds));
931 		if (ds->ds_snapname[0]) {
932 			VERIFY3U(strlcat(name, "@", ZFS_MAX_DATASET_NAME_LEN),
933 			    <, ZFS_MAX_DATASET_NAME_LEN);
934 			/*
935 			 * We use a "recursive" mutex so that we
936 			 * can call dprintf_ds() with ds_lock held.
937 			 */
938 			if (!MUTEX_HELD(&ds->ds_lock)) {
939 				mutex_enter(&ds->ds_lock);
940 				VERIFY3U(strlcat(name, ds->ds_snapname,
941 				    ZFS_MAX_DATASET_NAME_LEN), <,
942 				    ZFS_MAX_DATASET_NAME_LEN);
943 				mutex_exit(&ds->ds_lock);
944 			} else {
945 				VERIFY3U(strlcat(name, ds->ds_snapname,
946 				    ZFS_MAX_DATASET_NAME_LEN), <,
947 				    ZFS_MAX_DATASET_NAME_LEN);
948 			}
949 		}
950 	}
951 }
952 
953 int
954 dsl_dataset_namelen(dsl_dataset_t *ds)
955 {
956 	VERIFY0(dsl_dataset_get_snapname(ds));
957 	mutex_enter(&ds->ds_lock);
958 	int len = strlen(ds->ds_snapname);
959 	mutex_exit(&ds->ds_lock);
960 	/* add '@' if ds is a snap */
961 	if (len > 0)
962 		len++;
963 	len += dsl_dir_namelen(ds->ds_dir);
964 	return (len);
965 }
966 
967 void
968 dsl_dataset_rele(dsl_dataset_t *ds, void *tag)
969 {
970 	dmu_buf_rele(ds->ds_dbuf, tag);
971 }
972 
973 void
974 dsl_dataset_remove_key_mapping(dsl_dataset_t *ds)
975 {
976 	dsl_dir_t *dd = ds->ds_dir;
977 
978 	if (dd == NULL || dd->dd_crypto_obj == 0)
979 		return;
980 
981 	(void) spa_keystore_remove_mapping(dd->dd_pool->dp_spa,
982 	    ds->ds_object, ds);
983 }
984 
985 void
986 dsl_dataset_rele_flags(dsl_dataset_t *ds, ds_hold_flags_t flags, void *tag)
987 {
988 	if (flags & DS_HOLD_FLAG_DECRYPT)
989 		dsl_dataset_remove_key_mapping(ds);
990 
991 	dsl_dataset_rele(ds, tag);
992 }
993 
994 void
995 dsl_dataset_disown(dsl_dataset_t *ds, ds_hold_flags_t flags, void *tag)
996 {
997 	ASSERT3P(ds->ds_owner, ==, tag);
998 	ASSERT(ds->ds_dbuf != NULL);
999 
1000 	mutex_enter(&ds->ds_lock);
1001 	ds->ds_owner = NULL;
1002 	mutex_exit(&ds->ds_lock);
1003 	dsl_dataset_long_rele(ds, tag);
1004 	dsl_dataset_rele_flags(ds, flags, tag);
1005 }
1006 
1007 boolean_t
1008 dsl_dataset_tryown(dsl_dataset_t *ds, void *tag, boolean_t override)
1009 {
1010 	boolean_t gotit = FALSE;
1011 
1012 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
1013 	mutex_enter(&ds->ds_lock);
1014 	if (ds->ds_owner == NULL && (override || !(DS_IS_INCONSISTENT(ds) ||
1015 	    (dsl_dataset_feature_is_active(ds,
1016 	    SPA_FEATURE_REDACTED_DATASETS) &&
1017 	    !zfs_allow_redacted_dataset_mount)))) {
1018 		ds->ds_owner = tag;
1019 		dsl_dataset_long_hold(ds, tag);
1020 		gotit = TRUE;
1021 	}
1022 	mutex_exit(&ds->ds_lock);
1023 	return (gotit);
1024 }
1025 
1026 boolean_t
1027 dsl_dataset_has_owner(dsl_dataset_t *ds)
1028 {
1029 	boolean_t rv;
1030 	mutex_enter(&ds->ds_lock);
1031 	rv = (ds->ds_owner != NULL);
1032 	mutex_exit(&ds->ds_lock);
1033 	return (rv);
1034 }
1035 
1036 static boolean_t
1037 zfeature_active(spa_feature_t f, void *arg)
1038 {
1039 	switch (spa_feature_table[f].fi_type) {
1040 	case ZFEATURE_TYPE_BOOLEAN: {
1041 		boolean_t val = (boolean_t)(uintptr_t)arg;
1042 		ASSERT(val == B_FALSE || val == B_TRUE);
1043 		return (val);
1044 	}
1045 	case ZFEATURE_TYPE_UINT64_ARRAY:
1046 		/*
1047 		 * In this case, arg is a uint64_t array.  The feature is active
1048 		 * if the array is non-null.
1049 		 */
1050 		return (arg != NULL);
1051 	default:
1052 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
1053 		return (B_FALSE);
1054 	}
1055 }
1056 
1057 boolean_t
1058 dsl_dataset_feature_is_active(dsl_dataset_t *ds, spa_feature_t f)
1059 {
1060 	return (zfeature_active(f, ds->ds_feature[f]));
1061 }
1062 
1063 /*
1064  * The buffers passed out by this function are references to internal buffers;
1065  * they should not be freed by callers of this function, and they should not be
1066  * used after the dataset has been released.
1067  */
1068 boolean_t
1069 dsl_dataset_get_uint64_array_feature(dsl_dataset_t *ds, spa_feature_t f,
1070     uint64_t *outlength, uint64_t **outp)
1071 {
1072 	VERIFY(spa_feature_table[f].fi_type & ZFEATURE_TYPE_UINT64_ARRAY);
1073 	if (!dsl_dataset_feature_is_active(ds, f)) {
1074 		return (B_FALSE);
1075 	}
1076 	struct feature_type_uint64_array_arg *ftuaa = ds->ds_feature[f];
1077 	*outp = ftuaa->array;
1078 	*outlength = ftuaa->length;
1079 	return (B_TRUE);
1080 }
1081 
1082 void
1083 dsl_dataset_activate_feature(uint64_t dsobj, spa_feature_t f, void *arg,
1084     dmu_tx_t *tx)
1085 {
1086 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
1087 	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
1088 	uint64_t zero = 0;
1089 
1090 	VERIFY(spa_feature_table[f].fi_flags & ZFEATURE_FLAG_PER_DATASET);
1091 
1092 	spa_feature_incr(spa, f, tx);
1093 	dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
1094 
1095 	switch (spa_feature_table[f].fi_type) {
1096 	case ZFEATURE_TYPE_BOOLEAN:
1097 		ASSERT3S((boolean_t)(uintptr_t)arg, ==, B_TRUE);
1098 		VERIFY0(zap_add(mos, dsobj, spa_feature_table[f].fi_guid,
1099 		    sizeof (zero), 1, &zero, tx));
1100 		break;
1101 	case ZFEATURE_TYPE_UINT64_ARRAY:
1102 	{
1103 		struct feature_type_uint64_array_arg *ftuaa = arg;
1104 		VERIFY0(zap_add(mos, dsobj, spa_feature_table[f].fi_guid,
1105 		    sizeof (uint64_t), ftuaa->length, ftuaa->array, tx));
1106 		break;
1107 	}
1108 	default:
1109 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
1110 	}
1111 }
1112 
1113 static void
1114 dsl_dataset_deactivate_feature_impl(dsl_dataset_t *ds, spa_feature_t f,
1115     dmu_tx_t *tx)
1116 {
1117 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
1118 	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
1119 	uint64_t dsobj = ds->ds_object;
1120 
1121 	VERIFY(spa_feature_table[f].fi_flags & ZFEATURE_FLAG_PER_DATASET);
1122 
1123 	VERIFY0(zap_remove(mos, dsobj, spa_feature_table[f].fi_guid, tx));
1124 	spa_feature_decr(spa, f, tx);
1125 	ds->ds_feature[f] = NULL;
1126 }
1127 
1128 void
1129 dsl_dataset_deactivate_feature(dsl_dataset_t *ds, spa_feature_t f, dmu_tx_t *tx)
1130 {
1131 	unload_zfeature(ds, f);
1132 	dsl_dataset_deactivate_feature_impl(ds, f, tx);
1133 }
1134 
1135 uint64_t
1136 dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
1137     dsl_crypto_params_t *dcp, uint64_t flags, dmu_tx_t *tx)
1138 {
1139 	dsl_pool_t *dp = dd->dd_pool;
1140 	dmu_buf_t *dbuf;
1141 	dsl_dataset_phys_t *dsphys;
1142 	uint64_t dsobj;
1143 	objset_t *mos = dp->dp_meta_objset;
1144 
1145 	if (origin == NULL)
1146 		origin = dp->dp_origin_snap;
1147 
1148 	ASSERT(origin == NULL || origin->ds_dir->dd_pool == dp);
1149 	ASSERT(origin == NULL || dsl_dataset_phys(origin)->ds_num_children > 0);
1150 	ASSERT(dmu_tx_is_syncing(tx));
1151 	ASSERT(dsl_dir_phys(dd)->dd_head_dataset_obj == 0);
1152 
1153 	dsobj = dmu_object_alloc(mos, DMU_OT_DSL_DATASET, 0,
1154 	    DMU_OT_DSL_DATASET, sizeof (dsl_dataset_phys_t), tx);
1155 	VERIFY0(dmu_bonus_hold(mos, dsobj, FTAG, &dbuf));
1156 	dmu_buf_will_dirty(dbuf, tx);
1157 	dsphys = dbuf->db_data;
1158 	memset(dsphys, 0, sizeof (dsl_dataset_phys_t));
1159 	dsphys->ds_dir_obj = dd->dd_object;
1160 	dsphys->ds_flags = flags;
1161 	dsphys->ds_fsid_guid = unique_create();
1162 	(void) random_get_pseudo_bytes((void*)&dsphys->ds_guid,
1163 	    sizeof (dsphys->ds_guid));
1164 	dsphys->ds_snapnames_zapobj =
1165 	    zap_create_norm(mos, U8_TEXTPREP_TOUPPER, DMU_OT_DSL_DS_SNAP_MAP,
1166 	    DMU_OT_NONE, 0, tx);
1167 	dsphys->ds_creation_time = gethrestime_sec();
1168 	dsphys->ds_creation_txg = tx->tx_txg == TXG_INITIAL ? 1 : tx->tx_txg;
1169 
1170 	if (origin == NULL) {
1171 		dsphys->ds_deadlist_obj = dsl_deadlist_alloc(mos, tx);
1172 	} else {
1173 		dsl_dataset_t *ohds; /* head of the origin snapshot */
1174 
1175 		dsphys->ds_prev_snap_obj = origin->ds_object;
1176 		dsphys->ds_prev_snap_txg =
1177 		    dsl_dataset_phys(origin)->ds_creation_txg;
1178 		dsphys->ds_referenced_bytes =
1179 		    dsl_dataset_phys(origin)->ds_referenced_bytes;
1180 		dsphys->ds_compressed_bytes =
1181 		    dsl_dataset_phys(origin)->ds_compressed_bytes;
1182 		dsphys->ds_uncompressed_bytes =
1183 		    dsl_dataset_phys(origin)->ds_uncompressed_bytes;
1184 		rrw_enter(&origin->ds_bp_rwlock, RW_READER, FTAG);
1185 		dsphys->ds_bp = dsl_dataset_phys(origin)->ds_bp;
1186 		rrw_exit(&origin->ds_bp_rwlock, FTAG);
1187 
1188 		/*
1189 		 * Inherit flags that describe the dataset's contents
1190 		 * (INCONSISTENT) or properties (Case Insensitive).
1191 		 */
1192 		dsphys->ds_flags |= dsl_dataset_phys(origin)->ds_flags &
1193 		    (DS_FLAG_INCONSISTENT | DS_FLAG_CI_DATASET);
1194 
1195 		for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
1196 			if (zfeature_active(f, origin->ds_feature[f])) {
1197 				dsl_dataset_activate_feature(dsobj, f,
1198 				    origin->ds_feature[f], tx);
1199 			}
1200 		}
1201 
1202 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
1203 		dsl_dataset_phys(origin)->ds_num_children++;
1204 
1205 		VERIFY0(dsl_dataset_hold_obj(dp,
1206 		    dsl_dir_phys(origin->ds_dir)->dd_head_dataset_obj,
1207 		    FTAG, &ohds));
1208 		dsphys->ds_deadlist_obj = dsl_deadlist_clone(&ohds->ds_deadlist,
1209 		    dsphys->ds_prev_snap_txg, dsphys->ds_prev_snap_obj, tx);
1210 		dsl_dataset_rele(ohds, FTAG);
1211 
1212 		if (spa_version(dp->dp_spa) >= SPA_VERSION_NEXT_CLONES) {
1213 			if (dsl_dataset_phys(origin)->ds_next_clones_obj == 0) {
1214 				dsl_dataset_phys(origin)->ds_next_clones_obj =
1215 				    zap_create(mos,
1216 				    DMU_OT_NEXT_CLONES, DMU_OT_NONE, 0, tx);
1217 			}
1218 			VERIFY0(zap_add_int(mos,
1219 			    dsl_dataset_phys(origin)->ds_next_clones_obj,
1220 			    dsobj, tx));
1221 		}
1222 
1223 		dmu_buf_will_dirty(dd->dd_dbuf, tx);
1224 		dsl_dir_phys(dd)->dd_origin_obj = origin->ds_object;
1225 		if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
1226 			if (dsl_dir_phys(origin->ds_dir)->dd_clones == 0) {
1227 				dmu_buf_will_dirty(origin->ds_dir->dd_dbuf, tx);
1228 				dsl_dir_phys(origin->ds_dir)->dd_clones =
1229 				    zap_create(mos,
1230 				    DMU_OT_DSL_CLONES, DMU_OT_NONE, 0, tx);
1231 			}
1232 			VERIFY0(zap_add_int(mos,
1233 			    dsl_dir_phys(origin->ds_dir)->dd_clones,
1234 			    dsobj, tx));
1235 		}
1236 	}
1237 
1238 	/* handle encryption */
1239 	dsl_dataset_create_crypt_sync(dsobj, dd, origin, dcp, tx);
1240 
1241 	if (spa_version(dp->dp_spa) >= SPA_VERSION_UNIQUE_ACCURATE)
1242 		dsphys->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
1243 
1244 	dmu_buf_rele(dbuf, FTAG);
1245 
1246 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
1247 	dsl_dir_phys(dd)->dd_head_dataset_obj = dsobj;
1248 
1249 	return (dsobj);
1250 }
1251 
1252 static void
1253 dsl_dataset_zero_zil(dsl_dataset_t *ds, dmu_tx_t *tx)
1254 {
1255 	objset_t *os;
1256 
1257 	VERIFY0(dmu_objset_from_ds(ds, &os));
1258 	if (memcmp(&os->os_zil_header, &zero_zil, sizeof (zero_zil)) != 0) {
1259 		dsl_pool_t *dp = ds->ds_dir->dd_pool;
1260 		zio_t *zio;
1261 
1262 		memset(&os->os_zil_header, 0, sizeof (os->os_zil_header));
1263 		if (os->os_encrypted)
1264 			os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE;
1265 
1266 		zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
1267 		dsl_dataset_sync(ds, zio, tx);
1268 		VERIFY0(zio_wait(zio));
1269 
1270 		/* dsl_dataset_sync_done will drop this reference. */
1271 		dmu_buf_add_ref(ds->ds_dbuf, ds);
1272 		dsl_dataset_sync_done(ds, tx);
1273 	}
1274 }
1275 
1276 uint64_t
1277 dsl_dataset_create_sync(dsl_dir_t *pdd, const char *lastname,
1278     dsl_dataset_t *origin, uint64_t flags, cred_t *cr,
1279     dsl_crypto_params_t *dcp, dmu_tx_t *tx)
1280 {
1281 	dsl_pool_t *dp = pdd->dd_pool;
1282 	uint64_t dsobj, ddobj;
1283 	dsl_dir_t *dd;
1284 
1285 	ASSERT(dmu_tx_is_syncing(tx));
1286 	ASSERT(lastname[0] != '@');
1287 	/*
1288 	 * Filesystems will eventually have their origin set to dp_origin_snap,
1289 	 * but that's taken care of in dsl_dataset_create_sync_dd. When
1290 	 * creating a filesystem, this function is called with origin equal to
1291 	 * NULL.
1292 	 */
1293 	if (origin != NULL)
1294 		ASSERT3P(origin, !=, dp->dp_origin_snap);
1295 
1296 	ddobj = dsl_dir_create_sync(dp, pdd, lastname, tx);
1297 	VERIFY0(dsl_dir_hold_obj(dp, ddobj, lastname, FTAG, &dd));
1298 
1299 	dsobj = dsl_dataset_create_sync_dd(dd, origin, dcp,
1300 	    flags & ~DS_CREATE_FLAG_NODIRTY, tx);
1301 
1302 	dsl_deleg_set_create_perms(dd, tx, cr);
1303 
1304 	/*
1305 	 * If we are creating a clone and the livelist feature is enabled,
1306 	 * add the entry DD_FIELD_LIVELIST to ZAP.
1307 	 */
1308 	if (origin != NULL &&
1309 	    spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LIVELIST)) {
1310 		objset_t *mos = dd->dd_pool->dp_meta_objset;
1311 		dsl_dir_zapify(dd, tx);
1312 		uint64_t obj = dsl_deadlist_alloc(mos, tx);
1313 		VERIFY0(zap_add(mos, dd->dd_object, DD_FIELD_LIVELIST,
1314 		    sizeof (uint64_t), 1, &obj, tx));
1315 		spa_feature_incr(dp->dp_spa, SPA_FEATURE_LIVELIST, tx);
1316 	}
1317 
1318 	/*
1319 	 * Since we're creating a new node we know it's a leaf, so we can
1320 	 * initialize the counts if the limit feature is active.
1321 	 */
1322 	if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_FS_SS_LIMIT)) {
1323 		uint64_t cnt = 0;
1324 		objset_t *os = dd->dd_pool->dp_meta_objset;
1325 
1326 		dsl_dir_zapify(dd, tx);
1327 		VERIFY0(zap_add(os, dd->dd_object, DD_FIELD_FILESYSTEM_COUNT,
1328 		    sizeof (cnt), 1, &cnt, tx));
1329 		VERIFY0(zap_add(os, dd->dd_object, DD_FIELD_SNAPSHOT_COUNT,
1330 		    sizeof (cnt), 1, &cnt, tx));
1331 	}
1332 
1333 	dsl_dir_rele(dd, FTAG);
1334 
1335 	/*
1336 	 * If we are creating a clone, make sure we zero out any stale
1337 	 * data from the origin snapshots zil header.
1338 	 */
1339 	if (origin != NULL && !(flags & DS_CREATE_FLAG_NODIRTY)) {
1340 		dsl_dataset_t *ds;
1341 
1342 		VERIFY0(dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
1343 		dsl_dataset_zero_zil(ds, tx);
1344 		dsl_dataset_rele(ds, FTAG);
1345 	}
1346 
1347 	return (dsobj);
1348 }
1349 
1350 /*
1351  * The unique space in the head dataset can be calculated by subtracting
1352  * the space used in the most recent snapshot, that is still being used
1353  * in this file system, from the space currently in use.  To figure out
1354  * the space in the most recent snapshot still in use, we need to take
1355  * the total space used in the snapshot and subtract out the space that
1356  * has been freed up since the snapshot was taken.
1357  */
1358 void
1359 dsl_dataset_recalc_head_uniq(dsl_dataset_t *ds)
1360 {
1361 	uint64_t mrs_used;
1362 	uint64_t dlused, dlcomp, dluncomp;
1363 
1364 	ASSERT(!ds->ds_is_snapshot);
1365 
1366 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0)
1367 		mrs_used = dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes;
1368 	else
1369 		mrs_used = 0;
1370 
1371 	dsl_deadlist_space(&ds->ds_deadlist, &dlused, &dlcomp, &dluncomp);
1372 
1373 	ASSERT3U(dlused, <=, mrs_used);
1374 	dsl_dataset_phys(ds)->ds_unique_bytes =
1375 	    dsl_dataset_phys(ds)->ds_referenced_bytes - (mrs_used - dlused);
1376 
1377 	if (spa_version(ds->ds_dir->dd_pool->dp_spa) >=
1378 	    SPA_VERSION_UNIQUE_ACCURATE)
1379 		dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
1380 }
1381 
1382 void
1383 dsl_dataset_remove_from_next_clones(dsl_dataset_t *ds, uint64_t obj,
1384     dmu_tx_t *tx)
1385 {
1386 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
1387 	uint64_t count __maybe_unused;
1388 	int err;
1389 
1390 	ASSERT(dsl_dataset_phys(ds)->ds_num_children >= 2);
1391 	err = zap_remove_int(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
1392 	    obj, tx);
1393 	/*
1394 	 * The err should not be ENOENT, but a bug in a previous version
1395 	 * of the code could cause upgrade_clones_cb() to not set
1396 	 * ds_next_snap_obj when it should, leading to a missing entry.
1397 	 * If we knew that the pool was created after
1398 	 * SPA_VERSION_NEXT_CLONES, we could assert that it isn't
1399 	 * ENOENT.  However, at least we can check that we don't have
1400 	 * too many entries in the next_clones_obj even after failing to
1401 	 * remove this one.
1402 	 */
1403 	if (err != ENOENT)
1404 		VERIFY0(err);
1405 	ASSERT0(zap_count(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
1406 	    &count));
1407 	ASSERT3U(count, <=, dsl_dataset_phys(ds)->ds_num_children - 2);
1408 }
1409 
1410 
1411 blkptr_t *
1412 dsl_dataset_get_blkptr(dsl_dataset_t *ds)
1413 {
1414 	return (&dsl_dataset_phys(ds)->ds_bp);
1415 }
1416 
1417 spa_t *
1418 dsl_dataset_get_spa(dsl_dataset_t *ds)
1419 {
1420 	return (ds->ds_dir->dd_pool->dp_spa);
1421 }
1422 
1423 void
1424 dsl_dataset_dirty(dsl_dataset_t *ds, dmu_tx_t *tx)
1425 {
1426 	dsl_pool_t *dp;
1427 
1428 	if (ds == NULL) /* this is the meta-objset */
1429 		return;
1430 
1431 	ASSERT(ds->ds_objset != NULL);
1432 
1433 	if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0)
1434 		panic("dirtying snapshot!");
1435 
1436 	/* Must not dirty a dataset in the same txg where it got snapshotted. */
1437 	ASSERT3U(tx->tx_txg, >, dsl_dataset_phys(ds)->ds_prev_snap_txg);
1438 
1439 	dp = ds->ds_dir->dd_pool;
1440 	if (txg_list_add(&dp->dp_dirty_datasets, ds, tx->tx_txg)) {
1441 		objset_t *os = ds->ds_objset;
1442 
1443 		/* up the hold count until we can be written out */
1444 		dmu_buf_add_ref(ds->ds_dbuf, ds);
1445 
1446 		/* if this dataset is encrypted, grab a reference to the DCK */
1447 		if (ds->ds_dir->dd_crypto_obj != 0 &&
1448 		    !os->os_raw_receive &&
1449 		    !os->os_next_write_raw[tx->tx_txg & TXG_MASK]) {
1450 			ASSERT3P(ds->ds_key_mapping, !=, NULL);
1451 			key_mapping_add_ref(ds->ds_key_mapping, ds);
1452 		}
1453 	}
1454 }
1455 
1456 static int
1457 dsl_dataset_snapshot_reserve_space(dsl_dataset_t *ds, dmu_tx_t *tx)
1458 {
1459 	uint64_t asize;
1460 
1461 	if (!dmu_tx_is_syncing(tx))
1462 		return (0);
1463 
1464 	/*
1465 	 * If there's an fs-only reservation, any blocks that might become
1466 	 * owned by the snapshot dataset must be accommodated by space
1467 	 * outside of the reservation.
1468 	 */
1469 	ASSERT(ds->ds_reserved == 0 || DS_UNIQUE_IS_ACCURATE(ds));
1470 	asize = MIN(dsl_dataset_phys(ds)->ds_unique_bytes, ds->ds_reserved);
1471 	if (asize > dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE))
1472 		return (SET_ERROR(ENOSPC));
1473 
1474 	/*
1475 	 * Propagate any reserved space for this snapshot to other
1476 	 * snapshot checks in this sync group.
1477 	 */
1478 	if (asize > 0)
1479 		dsl_dir_willuse_space(ds->ds_dir, asize, tx);
1480 
1481 	return (0);
1482 }
1483 
1484 int
1485 dsl_dataset_snapshot_check_impl(dsl_dataset_t *ds, const char *snapname,
1486     dmu_tx_t *tx, boolean_t recv, uint64_t cnt, cred_t *cr, proc_t *proc)
1487 {
1488 	int error;
1489 	uint64_t value;
1490 
1491 	ds->ds_trysnap_txg = tx->tx_txg;
1492 
1493 	if (!dmu_tx_is_syncing(tx))
1494 		return (0);
1495 
1496 	/*
1497 	 * We don't allow multiple snapshots of the same txg.  If there
1498 	 * is already one, try again.
1499 	 */
1500 	if (dsl_dataset_phys(ds)->ds_prev_snap_txg >= tx->tx_txg)
1501 		return (SET_ERROR(EAGAIN));
1502 
1503 	/*
1504 	 * Check for conflicting snapshot name.
1505 	 */
1506 	error = dsl_dataset_snap_lookup(ds, snapname, &value);
1507 	if (error == 0)
1508 		return (SET_ERROR(EEXIST));
1509 	if (error != ENOENT)
1510 		return (error);
1511 
1512 	/*
1513 	 * We don't allow taking snapshots of inconsistent datasets, such as
1514 	 * those into which we are currently receiving.  However, if we are
1515 	 * creating this snapshot as part of a receive, this check will be
1516 	 * executed atomically with respect to the completion of the receive
1517 	 * itself but prior to the clearing of DS_FLAG_INCONSISTENT; in this
1518 	 * case we ignore this, knowing it will be fixed up for us shortly in
1519 	 * dmu_recv_end_sync().
1520 	 */
1521 	if (!recv && DS_IS_INCONSISTENT(ds))
1522 		return (SET_ERROR(EBUSY));
1523 
1524 	/*
1525 	 * Skip the check for temporary snapshots or if we have already checked
1526 	 * the counts in dsl_dataset_snapshot_check. This means we really only
1527 	 * check the count here when we're receiving a stream.
1528 	 */
1529 	if (cnt != 0 && cr != NULL) {
1530 		error = dsl_fs_ss_limit_check(ds->ds_dir, cnt,
1531 		    ZFS_PROP_SNAPSHOT_LIMIT, NULL, cr, proc);
1532 		if (error != 0)
1533 			return (error);
1534 	}
1535 
1536 	error = dsl_dataset_snapshot_reserve_space(ds, tx);
1537 	if (error != 0)
1538 		return (error);
1539 
1540 	return (0);
1541 }
1542 
1543 int
1544 dsl_dataset_snapshot_check(void *arg, dmu_tx_t *tx)
1545 {
1546 	dsl_dataset_snapshot_arg_t *ddsa = arg;
1547 	dsl_pool_t *dp = dmu_tx_pool(tx);
1548 	nvpair_t *pair;
1549 	int rv = 0;
1550 
1551 	/*
1552 	 * Pre-compute how many total new snapshots will be created for each
1553 	 * level in the tree and below. This is needed for validating the
1554 	 * snapshot limit when either taking a recursive snapshot or when
1555 	 * taking multiple snapshots.
1556 	 *
1557 	 * The problem is that the counts are not actually adjusted when
1558 	 * we are checking, only when we finally sync. For a single snapshot,
1559 	 * this is easy, the count will increase by 1 at each node up the tree,
1560 	 * but its more complicated for the recursive/multiple snapshot case.
1561 	 *
1562 	 * The dsl_fs_ss_limit_check function does recursively check the count
1563 	 * at each level up the tree but since it is validating each snapshot
1564 	 * independently we need to be sure that we are validating the complete
1565 	 * count for the entire set of snapshots. We do this by rolling up the
1566 	 * counts for each component of the name into an nvlist and then
1567 	 * checking each of those cases with the aggregated count.
1568 	 *
1569 	 * This approach properly handles not only the recursive snapshot
1570 	 * case (where we get all of those on the ddsa_snaps list) but also
1571 	 * the sibling case (e.g. snapshot a/b and a/c so that we will also
1572 	 * validate the limit on 'a' using a count of 2).
1573 	 *
1574 	 * We validate the snapshot names in the third loop and only report
1575 	 * name errors once.
1576 	 */
1577 	if (dmu_tx_is_syncing(tx)) {
1578 		char *nm;
1579 		nvlist_t *cnt_track = NULL;
1580 		cnt_track = fnvlist_alloc();
1581 
1582 		nm = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1583 
1584 		/* Rollup aggregated counts into the cnt_track list */
1585 		for (pair = nvlist_next_nvpair(ddsa->ddsa_snaps, NULL);
1586 		    pair != NULL;
1587 		    pair = nvlist_next_nvpair(ddsa->ddsa_snaps, pair)) {
1588 			char *pdelim;
1589 			uint64_t val;
1590 
1591 			(void) strlcpy(nm, nvpair_name(pair), MAXPATHLEN);
1592 			pdelim = strchr(nm, '@');
1593 			if (pdelim == NULL)
1594 				continue;
1595 			*pdelim = '\0';
1596 
1597 			do {
1598 				if (nvlist_lookup_uint64(cnt_track, nm,
1599 				    &val) == 0) {
1600 					/* update existing entry */
1601 					fnvlist_add_uint64(cnt_track, nm,
1602 					    val + 1);
1603 				} else {
1604 					/* add to list */
1605 					fnvlist_add_uint64(cnt_track, nm, 1);
1606 				}
1607 
1608 				pdelim = strrchr(nm, '/');
1609 				if (pdelim != NULL)
1610 					*pdelim = '\0';
1611 			} while (pdelim != NULL);
1612 		}
1613 
1614 		kmem_free(nm, MAXPATHLEN);
1615 
1616 		/* Check aggregated counts at each level */
1617 		for (pair = nvlist_next_nvpair(cnt_track, NULL);
1618 		    pair != NULL; pair = nvlist_next_nvpair(cnt_track, pair)) {
1619 			int error = 0;
1620 			char *name;
1621 			uint64_t cnt = 0;
1622 			dsl_dataset_t *ds;
1623 
1624 			name = nvpair_name(pair);
1625 			cnt = fnvpair_value_uint64(pair);
1626 			ASSERT(cnt > 0);
1627 
1628 			error = dsl_dataset_hold(dp, name, FTAG, &ds);
1629 			if (error == 0) {
1630 				error = dsl_fs_ss_limit_check(ds->ds_dir, cnt,
1631 				    ZFS_PROP_SNAPSHOT_LIMIT, NULL,
1632 				    ddsa->ddsa_cr, ddsa->ddsa_proc);
1633 				dsl_dataset_rele(ds, FTAG);
1634 			}
1635 
1636 			if (error != 0) {
1637 				if (ddsa->ddsa_errors != NULL)
1638 					fnvlist_add_int32(ddsa->ddsa_errors,
1639 					    name, error);
1640 				rv = error;
1641 				/* only report one error for this check */
1642 				break;
1643 			}
1644 		}
1645 		nvlist_free(cnt_track);
1646 	}
1647 
1648 	for (pair = nvlist_next_nvpair(ddsa->ddsa_snaps, NULL);
1649 	    pair != NULL; pair = nvlist_next_nvpair(ddsa->ddsa_snaps, pair)) {
1650 		int error = 0;
1651 		dsl_dataset_t *ds;
1652 		char *name, *atp = NULL;
1653 		char dsname[ZFS_MAX_DATASET_NAME_LEN];
1654 
1655 		name = nvpair_name(pair);
1656 		if (strlen(name) >= ZFS_MAX_DATASET_NAME_LEN)
1657 			error = SET_ERROR(ENAMETOOLONG);
1658 		if (error == 0) {
1659 			atp = strchr(name, '@');
1660 			if (atp == NULL)
1661 				error = SET_ERROR(EINVAL);
1662 			if (error == 0)
1663 				(void) strlcpy(dsname, name, atp - name + 1);
1664 		}
1665 		if (error == 0)
1666 			error = dsl_dataset_hold(dp, dsname, FTAG, &ds);
1667 		if (error == 0) {
1668 			/* passing 0/NULL skips dsl_fs_ss_limit_check */
1669 			error = dsl_dataset_snapshot_check_impl(ds,
1670 			    atp + 1, tx, B_FALSE, 0, NULL, NULL);
1671 			dsl_dataset_rele(ds, FTAG);
1672 		}
1673 
1674 		if (error != 0) {
1675 			if (ddsa->ddsa_errors != NULL) {
1676 				fnvlist_add_int32(ddsa->ddsa_errors,
1677 				    name, error);
1678 			}
1679 			rv = error;
1680 		}
1681 	}
1682 
1683 	return (rv);
1684 }
1685 
1686 void
1687 dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
1688     dmu_tx_t *tx)
1689 {
1690 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
1691 	dmu_buf_t *dbuf;
1692 	dsl_dataset_phys_t *dsphys;
1693 	uint64_t dsobj, crtxg;
1694 	objset_t *mos = dp->dp_meta_objset;
1695 	static zil_header_t zero_zil __maybe_unused;
1696 	objset_t *os __maybe_unused;
1697 
1698 	ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
1699 
1700 	/*
1701 	 * If we are on an old pool, the zil must not be active, in which
1702 	 * case it will be zeroed.  Usually zil_suspend() accomplishes this.
1703 	 */
1704 	ASSERT(spa_version(dmu_tx_pool(tx)->dp_spa) >= SPA_VERSION_FAST_SNAP ||
1705 	    dmu_objset_from_ds(ds, &os) != 0 ||
1706 	    memcmp(&os->os_phys->os_zil_header, &zero_zil,
1707 	    sizeof (zero_zil)) == 0);
1708 
1709 	/* Should not snapshot a dirty dataset. */
1710 	ASSERT(!txg_list_member(&ds->ds_dir->dd_pool->dp_dirty_datasets,
1711 	    ds, tx->tx_txg));
1712 
1713 	dsl_fs_ss_count_adjust(ds->ds_dir, 1, DD_FIELD_SNAPSHOT_COUNT, tx);
1714 
1715 	/*
1716 	 * The origin's ds_creation_txg has to be < TXG_INITIAL
1717 	 */
1718 	if (strcmp(snapname, ORIGIN_DIR_NAME) == 0)
1719 		crtxg = 1;
1720 	else
1721 		crtxg = tx->tx_txg;
1722 
1723 	dsobj = dmu_object_alloc(mos, DMU_OT_DSL_DATASET, 0,
1724 	    DMU_OT_DSL_DATASET, sizeof (dsl_dataset_phys_t), tx);
1725 	VERIFY0(dmu_bonus_hold(mos, dsobj, FTAG, &dbuf));
1726 	dmu_buf_will_dirty(dbuf, tx);
1727 	dsphys = dbuf->db_data;
1728 	memset(dsphys, 0, sizeof (dsl_dataset_phys_t));
1729 	dsphys->ds_dir_obj = ds->ds_dir->dd_object;
1730 	dsphys->ds_fsid_guid = unique_create();
1731 	(void) random_get_pseudo_bytes((void*)&dsphys->ds_guid,
1732 	    sizeof (dsphys->ds_guid));
1733 	dsphys->ds_prev_snap_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
1734 	dsphys->ds_prev_snap_txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
1735 	dsphys->ds_next_snap_obj = ds->ds_object;
1736 	dsphys->ds_num_children = 1;
1737 	dsphys->ds_creation_time = gethrestime_sec();
1738 	dsphys->ds_creation_txg = crtxg;
1739 	dsphys->ds_deadlist_obj = dsl_dataset_phys(ds)->ds_deadlist_obj;
1740 	dsphys->ds_referenced_bytes = dsl_dataset_phys(ds)->ds_referenced_bytes;
1741 	dsphys->ds_compressed_bytes = dsl_dataset_phys(ds)->ds_compressed_bytes;
1742 	dsphys->ds_uncompressed_bytes =
1743 	    dsl_dataset_phys(ds)->ds_uncompressed_bytes;
1744 	dsphys->ds_flags = dsl_dataset_phys(ds)->ds_flags;
1745 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
1746 	dsphys->ds_bp = dsl_dataset_phys(ds)->ds_bp;
1747 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
1748 	dmu_buf_rele(dbuf, FTAG);
1749 
1750 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
1751 		if (zfeature_active(f, ds->ds_feature[f])) {
1752 			dsl_dataset_activate_feature(dsobj, f,
1753 			    ds->ds_feature[f], tx);
1754 		}
1755 	}
1756 
1757 	ASSERT3U(ds->ds_prev != 0, ==,
1758 	    dsl_dataset_phys(ds)->ds_prev_snap_obj != 0);
1759 	if (ds->ds_prev) {
1760 		uint64_t next_clones_obj =
1761 		    dsl_dataset_phys(ds->ds_prev)->ds_next_clones_obj;
1762 		ASSERT(dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
1763 		    ds->ds_object ||
1764 		    dsl_dataset_phys(ds->ds_prev)->ds_num_children > 1);
1765 		if (dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
1766 		    ds->ds_object) {
1767 			dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
1768 			ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, ==,
1769 			    dsl_dataset_phys(ds->ds_prev)->ds_creation_txg);
1770 			dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj = dsobj;
1771 		} else if (next_clones_obj != 0) {
1772 			dsl_dataset_remove_from_next_clones(ds->ds_prev,
1773 			    dsphys->ds_next_snap_obj, tx);
1774 			VERIFY0(zap_add_int(mos,
1775 			    next_clones_obj, dsobj, tx));
1776 		}
1777 	}
1778 
1779 	/*
1780 	 * If we have a reference-reservation on this dataset, we will
1781 	 * need to increase the amount of refreservation being charged
1782 	 * since our unique space is going to zero.
1783 	 */
1784 	if (ds->ds_reserved) {
1785 		int64_t delta;
1786 		ASSERT(DS_UNIQUE_IS_ACCURATE(ds));
1787 		delta = MIN(dsl_dataset_phys(ds)->ds_unique_bytes,
1788 		    ds->ds_reserved);
1789 		dsl_dir_diduse_space(ds->ds_dir, DD_USED_REFRSRV,
1790 		    delta, 0, 0, tx);
1791 	}
1792 
1793 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
1794 	dsl_dataset_phys(ds)->ds_deadlist_obj =
1795 	    dsl_deadlist_clone(&ds->ds_deadlist, UINT64_MAX,
1796 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, tx);
1797 	dsl_deadlist_close(&ds->ds_deadlist);
1798 	dsl_deadlist_open(&ds->ds_deadlist, mos,
1799 	    dsl_dataset_phys(ds)->ds_deadlist_obj);
1800 	dsl_deadlist_add_key(&ds->ds_deadlist,
1801 	    dsl_dataset_phys(ds)->ds_prev_snap_txg, tx);
1802 	dsl_bookmark_snapshotted(ds, tx);
1803 
1804 	if (dsl_dataset_remap_deadlist_exists(ds)) {
1805 		uint64_t remap_deadlist_obj =
1806 		    dsl_dataset_get_remap_deadlist_object(ds);
1807 		/*
1808 		 * Move the remap_deadlist to the snapshot.  The head
1809 		 * will create a new remap deadlist on demand, from
1810 		 * dsl_dataset_block_remapped().
1811 		 */
1812 		dsl_dataset_unset_remap_deadlist_object(ds, tx);
1813 		dsl_deadlist_close(&ds->ds_remap_deadlist);
1814 
1815 		dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
1816 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_REMAP_DEADLIST,
1817 		    sizeof (remap_deadlist_obj), 1, &remap_deadlist_obj, tx));
1818 	}
1819 
1820 	/*
1821 	 * Create a ivset guid for this snapshot if the dataset is
1822 	 * encrypted. This may be overridden by a raw receive. A
1823 	 * previous implementation of this code did not have this
1824 	 * field as part of the on-disk format for ZFS encryption
1825 	 * (see errata #4). As part of the remediation for this
1826 	 * issue, we ask the user to enable the bookmark_v2 feature
1827 	 * which is now a dependency of the encryption feature. We
1828 	 * use this as a heuristic to determine when the user has
1829 	 * elected to correct any datasets created with the old code.
1830 	 * As a result, we only do this step if the bookmark_v2
1831 	 * feature is enabled, which limits the number of states a
1832 	 * given pool / dataset can be in with regards to terms of
1833 	 * correcting the issue.
1834 	 */
1835 	if (ds->ds_dir->dd_crypto_obj != 0 &&
1836 	    spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_BOOKMARK_V2)) {
1837 		uint64_t ivset_guid = unique_create();
1838 
1839 		dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
1840 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_IVSET_GUID,
1841 		    sizeof (ivset_guid), 1, &ivset_guid, tx));
1842 	}
1843 
1844 	ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, <, tx->tx_txg);
1845 	dsl_dataset_phys(ds)->ds_prev_snap_obj = dsobj;
1846 	dsl_dataset_phys(ds)->ds_prev_snap_txg = crtxg;
1847 	dsl_dataset_phys(ds)->ds_unique_bytes = 0;
1848 
1849 	if (spa_version(dp->dp_spa) >= SPA_VERSION_UNIQUE_ACCURATE)
1850 		dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
1851 
1852 	VERIFY0(zap_add(mos, dsl_dataset_phys(ds)->ds_snapnames_zapobj,
1853 	    snapname, 8, 1, &dsobj, tx));
1854 
1855 	if (ds->ds_prev)
1856 		dsl_dataset_rele(ds->ds_prev, ds);
1857 	VERIFY0(dsl_dataset_hold_obj(dp,
1858 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, ds, &ds->ds_prev));
1859 
1860 	dsl_scan_ds_snapshotted(ds, tx);
1861 
1862 	dsl_dir_snap_cmtime_update(ds->ds_dir);
1863 
1864 	spa_history_log_internal_ds(ds->ds_prev, "snapshot", tx, " ");
1865 }
1866 
1867 void
1868 dsl_dataset_snapshot_sync(void *arg, dmu_tx_t *tx)
1869 {
1870 	dsl_dataset_snapshot_arg_t *ddsa = arg;
1871 	dsl_pool_t *dp = dmu_tx_pool(tx);
1872 	nvpair_t *pair;
1873 
1874 	for (pair = nvlist_next_nvpair(ddsa->ddsa_snaps, NULL);
1875 	    pair != NULL; pair = nvlist_next_nvpair(ddsa->ddsa_snaps, pair)) {
1876 		dsl_dataset_t *ds;
1877 		char *name, *atp;
1878 		char dsname[ZFS_MAX_DATASET_NAME_LEN];
1879 
1880 		name = nvpair_name(pair);
1881 		atp = strchr(name, '@');
1882 		(void) strlcpy(dsname, name, atp - name + 1);
1883 		VERIFY0(dsl_dataset_hold(dp, dsname, FTAG, &ds));
1884 
1885 		dsl_dataset_snapshot_sync_impl(ds, atp + 1, tx);
1886 		if (ddsa->ddsa_props != NULL) {
1887 			dsl_props_set_sync_impl(ds->ds_prev,
1888 			    ZPROP_SRC_LOCAL, ddsa->ddsa_props, tx);
1889 		}
1890 		dsl_dataset_rele(ds, FTAG);
1891 	}
1892 }
1893 
1894 /*
1895  * The snapshots must all be in the same pool.
1896  * All-or-nothing: if there are any failures, nothing will be modified.
1897  */
1898 int
1899 dsl_dataset_snapshot(nvlist_t *snaps, nvlist_t *props, nvlist_t *errors)
1900 {
1901 	dsl_dataset_snapshot_arg_t ddsa;
1902 	nvpair_t *pair;
1903 	boolean_t needsuspend;
1904 	int error;
1905 	spa_t *spa;
1906 	char *firstname;
1907 	nvlist_t *suspended = NULL;
1908 
1909 	pair = nvlist_next_nvpair(snaps, NULL);
1910 	if (pair == NULL)
1911 		return (0);
1912 	firstname = nvpair_name(pair);
1913 
1914 	error = spa_open(firstname, &spa, FTAG);
1915 	if (error != 0)
1916 		return (error);
1917 	needsuspend = (spa_version(spa) < SPA_VERSION_FAST_SNAP);
1918 	spa_close(spa, FTAG);
1919 
1920 	if (needsuspend) {
1921 		suspended = fnvlist_alloc();
1922 		for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
1923 		    pair = nvlist_next_nvpair(snaps, pair)) {
1924 			char fsname[ZFS_MAX_DATASET_NAME_LEN];
1925 			char *snapname = nvpair_name(pair);
1926 			char *atp;
1927 			void *cookie;
1928 
1929 			atp = strchr(snapname, '@');
1930 			if (atp == NULL) {
1931 				error = SET_ERROR(EINVAL);
1932 				break;
1933 			}
1934 			(void) strlcpy(fsname, snapname, atp - snapname + 1);
1935 
1936 			error = zil_suspend(fsname, &cookie);
1937 			if (error != 0)
1938 				break;
1939 			fnvlist_add_uint64(suspended, fsname,
1940 			    (uintptr_t)cookie);
1941 		}
1942 	}
1943 
1944 	ddsa.ddsa_snaps = snaps;
1945 	ddsa.ddsa_props = props;
1946 	ddsa.ddsa_errors = errors;
1947 	ddsa.ddsa_cr = CRED();
1948 	ddsa.ddsa_proc = curproc;
1949 
1950 	if (error == 0) {
1951 		error = dsl_sync_task(firstname, dsl_dataset_snapshot_check,
1952 		    dsl_dataset_snapshot_sync, &ddsa,
1953 		    fnvlist_num_pairs(snaps) * 3, ZFS_SPACE_CHECK_NORMAL);
1954 	}
1955 
1956 	if (suspended != NULL) {
1957 		for (pair = nvlist_next_nvpair(suspended, NULL); pair != NULL;
1958 		    pair = nvlist_next_nvpair(suspended, pair)) {
1959 			zil_resume((void *)(uintptr_t)
1960 			    fnvpair_value_uint64(pair));
1961 		}
1962 		fnvlist_free(suspended);
1963 	}
1964 
1965 	if (error == 0) {
1966 		for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
1967 		    pair = nvlist_next_nvpair(snaps, pair)) {
1968 			zvol_create_minor(nvpair_name(pair));
1969 		}
1970 	}
1971 
1972 	return (error);
1973 }
1974 
1975 typedef struct dsl_dataset_snapshot_tmp_arg {
1976 	const char *ddsta_fsname;
1977 	const char *ddsta_snapname;
1978 	minor_t ddsta_cleanup_minor;
1979 	const char *ddsta_htag;
1980 } dsl_dataset_snapshot_tmp_arg_t;
1981 
1982 static int
1983 dsl_dataset_snapshot_tmp_check(void *arg, dmu_tx_t *tx)
1984 {
1985 	dsl_dataset_snapshot_tmp_arg_t *ddsta = arg;
1986 	dsl_pool_t *dp = dmu_tx_pool(tx);
1987 	dsl_dataset_t *ds;
1988 	int error;
1989 
1990 	error = dsl_dataset_hold(dp, ddsta->ddsta_fsname, FTAG, &ds);
1991 	if (error != 0)
1992 		return (error);
1993 
1994 	/* NULL cred means no limit check for tmp snapshot */
1995 	error = dsl_dataset_snapshot_check_impl(ds, ddsta->ddsta_snapname,
1996 	    tx, B_FALSE, 0, NULL, NULL);
1997 	if (error != 0) {
1998 		dsl_dataset_rele(ds, FTAG);
1999 		return (error);
2000 	}
2001 
2002 	if (spa_version(dp->dp_spa) < SPA_VERSION_USERREFS) {
2003 		dsl_dataset_rele(ds, FTAG);
2004 		return (SET_ERROR(ENOTSUP));
2005 	}
2006 	error = dsl_dataset_user_hold_check_one(NULL, ddsta->ddsta_htag,
2007 	    B_TRUE, tx);
2008 	if (error != 0) {
2009 		dsl_dataset_rele(ds, FTAG);
2010 		return (error);
2011 	}
2012 
2013 	dsl_dataset_rele(ds, FTAG);
2014 	return (0);
2015 }
2016 
2017 static void
2018 dsl_dataset_snapshot_tmp_sync(void *arg, dmu_tx_t *tx)
2019 {
2020 	dsl_dataset_snapshot_tmp_arg_t *ddsta = arg;
2021 	dsl_pool_t *dp = dmu_tx_pool(tx);
2022 	dsl_dataset_t *ds = NULL;
2023 
2024 	VERIFY0(dsl_dataset_hold(dp, ddsta->ddsta_fsname, FTAG, &ds));
2025 
2026 	dsl_dataset_snapshot_sync_impl(ds, ddsta->ddsta_snapname, tx);
2027 	dsl_dataset_user_hold_sync_one(ds->ds_prev, ddsta->ddsta_htag,
2028 	    ddsta->ddsta_cleanup_minor, gethrestime_sec(), tx);
2029 	dsl_destroy_snapshot_sync_impl(ds->ds_prev, B_TRUE, tx);
2030 
2031 	dsl_dataset_rele(ds, FTAG);
2032 }
2033 
2034 int
2035 dsl_dataset_snapshot_tmp(const char *fsname, const char *snapname,
2036     minor_t cleanup_minor, const char *htag)
2037 {
2038 	dsl_dataset_snapshot_tmp_arg_t ddsta;
2039 	int error;
2040 	spa_t *spa;
2041 	boolean_t needsuspend;
2042 	void *cookie;
2043 
2044 	ddsta.ddsta_fsname = fsname;
2045 	ddsta.ddsta_snapname = snapname;
2046 	ddsta.ddsta_cleanup_minor = cleanup_minor;
2047 	ddsta.ddsta_htag = htag;
2048 
2049 	error = spa_open(fsname, &spa, FTAG);
2050 	if (error != 0)
2051 		return (error);
2052 	needsuspend = (spa_version(spa) < SPA_VERSION_FAST_SNAP);
2053 	spa_close(spa, FTAG);
2054 
2055 	if (needsuspend) {
2056 		error = zil_suspend(fsname, &cookie);
2057 		if (error != 0)
2058 			return (error);
2059 	}
2060 
2061 	error = dsl_sync_task(fsname, dsl_dataset_snapshot_tmp_check,
2062 	    dsl_dataset_snapshot_tmp_sync, &ddsta, 3, ZFS_SPACE_CHECK_RESERVED);
2063 
2064 	if (needsuspend)
2065 		zil_resume(cookie);
2066 	return (error);
2067 }
2068 
2069 void
2070 dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx)
2071 {
2072 	ASSERT(dmu_tx_is_syncing(tx));
2073 	ASSERT(ds->ds_objset != NULL);
2074 	ASSERT(dsl_dataset_phys(ds)->ds_next_snap_obj == 0);
2075 
2076 	/*
2077 	 * in case we had to change ds_fsid_guid when we opened it,
2078 	 * sync it out now.
2079 	 */
2080 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
2081 	dsl_dataset_phys(ds)->ds_fsid_guid = ds->ds_fsid_guid;
2082 
2083 	if (ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] != 0) {
2084 		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
2085 		    ds->ds_object, DS_FIELD_RESUME_OBJECT, 8, 1,
2086 		    &ds->ds_resume_object[tx->tx_txg & TXG_MASK], tx));
2087 		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
2088 		    ds->ds_object, DS_FIELD_RESUME_OFFSET, 8, 1,
2089 		    &ds->ds_resume_offset[tx->tx_txg & TXG_MASK], tx));
2090 		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
2091 		    ds->ds_object, DS_FIELD_RESUME_BYTES, 8, 1,
2092 		    &ds->ds_resume_bytes[tx->tx_txg & TXG_MASK], tx));
2093 		ds->ds_resume_object[tx->tx_txg & TXG_MASK] = 0;
2094 		ds->ds_resume_offset[tx->tx_txg & TXG_MASK] = 0;
2095 		ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] = 0;
2096 	}
2097 
2098 	dmu_objset_sync(ds->ds_objset, zio, tx);
2099 
2100 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
2101 		if (zfeature_active(f, ds->ds_feature_activation[f])) {
2102 			if (zfeature_active(f, ds->ds_feature[f]))
2103 				continue;
2104 			dsl_dataset_activate_feature(ds->ds_object, f,
2105 			    ds->ds_feature_activation[f], tx);
2106 			ds->ds_feature[f] = ds->ds_feature_activation[f];
2107 		}
2108 	}
2109 }
2110 
2111 /*
2112  * Check if the percentage of blocks shared between the clone and the
2113  * snapshot (as opposed to those that are clone only) is below a certain
2114  * threshold
2115  */
2116 static boolean_t
2117 dsl_livelist_should_disable(dsl_dataset_t *ds)
2118 {
2119 	uint64_t used, referenced;
2120 	int percent_shared;
2121 
2122 	used = dsl_dir_get_usedds(ds->ds_dir);
2123 	referenced = dsl_get_referenced(ds);
2124 	ASSERT3U(referenced, >=, 0);
2125 	ASSERT3U(used, >=, 0);
2126 	if (referenced == 0)
2127 		return (B_FALSE);
2128 	percent_shared = (100 * (referenced - used)) / referenced;
2129 	if (percent_shared <= zfs_livelist_min_percent_shared)
2130 		return (B_TRUE);
2131 	return (B_FALSE);
2132 }
2133 
2134 /*
2135  *  Check if it is possible to combine two livelist entries into one.
2136  *  This is the case if the combined number of 'live' blkptrs (ALLOCs that
2137  *  don't have a matching FREE) is under the maximum sublist size.
2138  *  We check this by subtracting twice the total number of frees from the total
2139  *  number of blkptrs. FREEs are counted twice because each FREE blkptr
2140  *  will cancel out an ALLOC blkptr when the livelist is processed.
2141  */
2142 static boolean_t
2143 dsl_livelist_should_condense(dsl_deadlist_entry_t *first,
2144     dsl_deadlist_entry_t *next)
2145 {
2146 	uint64_t total_free = first->dle_bpobj.bpo_phys->bpo_num_freed +
2147 	    next->dle_bpobj.bpo_phys->bpo_num_freed;
2148 	uint64_t total_entries = first->dle_bpobj.bpo_phys->bpo_num_blkptrs +
2149 	    next->dle_bpobj.bpo_phys->bpo_num_blkptrs;
2150 	if ((total_entries - (2 * total_free)) < zfs_livelist_max_entries)
2151 		return (B_TRUE);
2152 	return (B_FALSE);
2153 }
2154 
2155 typedef struct try_condense_arg {
2156 	spa_t *spa;
2157 	dsl_dataset_t *ds;
2158 } try_condense_arg_t;
2159 
2160 /*
2161  * Iterate over the livelist entries, searching for a pair to condense.
2162  * A nonzero return value means stop, 0 means keep looking.
2163  */
2164 static int
2165 dsl_livelist_try_condense(void *arg, dsl_deadlist_entry_t *first)
2166 {
2167 	try_condense_arg_t *tca = arg;
2168 	spa_t *spa = tca->spa;
2169 	dsl_dataset_t *ds = tca->ds;
2170 	dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist;
2171 	dsl_deadlist_entry_t *next;
2172 
2173 	/* The condense thread has not yet been created at import */
2174 	if (spa->spa_livelist_condense_zthr == NULL)
2175 		return (1);
2176 
2177 	/* A condense is already in progress */
2178 	if (spa->spa_to_condense.ds != NULL)
2179 		return (1);
2180 
2181 	next = AVL_NEXT(&ll->dl_tree, &first->dle_node);
2182 	/* The livelist has only one entry - don't condense it */
2183 	if (next == NULL)
2184 		return (1);
2185 
2186 	/* Next is the newest entry - don't condense it */
2187 	if (AVL_NEXT(&ll->dl_tree, &next->dle_node) == NULL)
2188 		return (1);
2189 
2190 	/* This pair is not ready to condense but keep looking */
2191 	if (!dsl_livelist_should_condense(first, next))
2192 		return (0);
2193 
2194 	/*
2195 	 * Add a ref to prevent the dataset from being evicted while
2196 	 * the condense zthr or synctask are running. Ref will be
2197 	 * released at the end of the condense synctask
2198 	 */
2199 	dmu_buf_add_ref(ds->ds_dbuf, spa);
2200 
2201 	spa->spa_to_condense.ds = ds;
2202 	spa->spa_to_condense.first = first;
2203 	spa->spa_to_condense.next = next;
2204 	spa->spa_to_condense.syncing = B_FALSE;
2205 	spa->spa_to_condense.cancelled = B_FALSE;
2206 
2207 	zthr_wakeup(spa->spa_livelist_condense_zthr);
2208 	return (1);
2209 }
2210 
2211 static void
2212 dsl_flush_pending_livelist(dsl_dataset_t *ds, dmu_tx_t *tx)
2213 {
2214 	dsl_dir_t *dd = ds->ds_dir;
2215 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
2216 	dsl_deadlist_entry_t *last = dsl_deadlist_last(&dd->dd_livelist);
2217 
2218 	/* Check if we need to add a new sub-livelist */
2219 	if (last == NULL) {
2220 		/* The livelist is empty */
2221 		dsl_deadlist_add_key(&dd->dd_livelist,
2222 		    tx->tx_txg - 1, tx);
2223 	} else if (spa_sync_pass(spa) == 1) {
2224 		/*
2225 		 * Check if the newest entry is full. If it is, make a new one.
2226 		 * We only do this once per sync because we could overfill a
2227 		 * sublist in one sync pass and don't want to add another entry
2228 		 * for a txg that is already represented. This ensures that
2229 		 * blkptrs born in the same txg are stored in the same sublist.
2230 		 */
2231 		bpobj_t bpobj = last->dle_bpobj;
2232 		uint64_t all = bpobj.bpo_phys->bpo_num_blkptrs;
2233 		uint64_t free = bpobj.bpo_phys->bpo_num_freed;
2234 		uint64_t alloc = all - free;
2235 		if (alloc > zfs_livelist_max_entries) {
2236 			dsl_deadlist_add_key(&dd->dd_livelist,
2237 			    tx->tx_txg - 1, tx);
2238 		}
2239 	}
2240 
2241 	/* Insert each entry into the on-disk livelist */
2242 	bplist_iterate(&dd->dd_pending_allocs,
2243 	    dsl_deadlist_insert_alloc_cb, &dd->dd_livelist, tx);
2244 	bplist_iterate(&dd->dd_pending_frees,
2245 	    dsl_deadlist_insert_free_cb, &dd->dd_livelist, tx);
2246 
2247 	/* Attempt to condense every pair of adjacent entries */
2248 	try_condense_arg_t arg = {
2249 	    .spa = spa,
2250 	    .ds = ds
2251 	};
2252 	dsl_deadlist_iterate(&dd->dd_livelist, dsl_livelist_try_condense,
2253 	    &arg);
2254 }
2255 
2256 void
2257 dsl_dataset_sync_done(dsl_dataset_t *ds, dmu_tx_t *tx)
2258 {
2259 	objset_t *os = ds->ds_objset;
2260 
2261 	bplist_iterate(&ds->ds_pending_deadlist,
2262 	    dsl_deadlist_insert_alloc_cb, &ds->ds_deadlist, tx);
2263 
2264 	if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist)) {
2265 		dsl_flush_pending_livelist(ds, tx);
2266 		if (dsl_livelist_should_disable(ds)) {
2267 			dsl_dir_remove_livelist(ds->ds_dir, tx, B_TRUE);
2268 		}
2269 	}
2270 
2271 	dsl_bookmark_sync_done(ds, tx);
2272 
2273 	multilist_destroy(&os->os_synced_dnodes);
2274 
2275 	if (os->os_encrypted)
2276 		os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_FALSE;
2277 	else
2278 		ASSERT0(os->os_next_write_raw[tx->tx_txg & TXG_MASK]);
2279 
2280 	ASSERT(!dmu_objset_is_dirty(os, dmu_tx_get_txg(tx)));
2281 
2282 	dmu_buf_rele(ds->ds_dbuf, ds);
2283 }
2284 
2285 int
2286 get_clones_stat_impl(dsl_dataset_t *ds, nvlist_t *val)
2287 {
2288 	uint64_t count = 0;
2289 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
2290 	zap_cursor_t zc;
2291 	zap_attribute_t za;
2292 
2293 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
2294 
2295 	/*
2296 	 * There may be missing entries in ds_next_clones_obj
2297 	 * due to a bug in a previous version of the code.
2298 	 * Only trust it if it has the right number of entries.
2299 	 */
2300 	if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) {
2301 		VERIFY0(zap_count(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
2302 		    &count));
2303 	}
2304 	if (count != dsl_dataset_phys(ds)->ds_num_children - 1) {
2305 		return (SET_ERROR(ENOENT));
2306 	}
2307 	for (zap_cursor_init(&zc, mos,
2308 	    dsl_dataset_phys(ds)->ds_next_clones_obj);
2309 	    zap_cursor_retrieve(&zc, &za) == 0;
2310 	    zap_cursor_advance(&zc)) {
2311 		dsl_dataset_t *clone;
2312 		char buf[ZFS_MAX_DATASET_NAME_LEN];
2313 		VERIFY0(dsl_dataset_hold_obj(ds->ds_dir->dd_pool,
2314 		    za.za_first_integer, FTAG, &clone));
2315 		dsl_dir_name(clone->ds_dir, buf);
2316 		fnvlist_add_boolean(val, buf);
2317 		dsl_dataset_rele(clone, FTAG);
2318 	}
2319 	zap_cursor_fini(&zc);
2320 	return (0);
2321 }
2322 
2323 void
2324 get_clones_stat(dsl_dataset_t *ds, nvlist_t *nv)
2325 {
2326 	nvlist_t *propval = fnvlist_alloc();
2327 	nvlist_t *val = fnvlist_alloc();
2328 
2329 	if (get_clones_stat_impl(ds, val) == 0) {
2330 		fnvlist_add_nvlist(propval, ZPROP_VALUE, val);
2331 		fnvlist_add_nvlist(nv, zfs_prop_to_name(ZFS_PROP_CLONES),
2332 		    propval);
2333 	}
2334 
2335 	nvlist_free(val);
2336 	nvlist_free(propval);
2337 }
2338 
2339 static char *
2340 get_receive_resume_token_impl(dsl_dataset_t *ds)
2341 {
2342 	if (!dsl_dataset_has_resume_receive_state(ds))
2343 		return (NULL);
2344 
2345 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
2346 	char *str;
2347 	void *packed;
2348 	uint8_t *compressed;
2349 	uint64_t val;
2350 	nvlist_t *token_nv = fnvlist_alloc();
2351 	size_t packed_size, compressed_size;
2352 
2353 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
2354 	    DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val) == 0) {
2355 		fnvlist_add_uint64(token_nv, "fromguid", val);
2356 	}
2357 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
2358 	    DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val) == 0) {
2359 		fnvlist_add_uint64(token_nv, "object", val);
2360 	}
2361 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
2362 	    DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val) == 0) {
2363 		fnvlist_add_uint64(token_nv, "offset", val);
2364 	}
2365 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
2366 	    DS_FIELD_RESUME_BYTES, sizeof (val), 1, &val) == 0) {
2367 		fnvlist_add_uint64(token_nv, "bytes", val);
2368 	}
2369 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
2370 	    DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val) == 0) {
2371 		fnvlist_add_uint64(token_nv, "toguid", val);
2372 	}
2373 	char buf[MAXNAMELEN];
2374 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
2375 	    DS_FIELD_RESUME_TONAME, 1, sizeof (buf), buf) == 0) {
2376 		fnvlist_add_string(token_nv, "toname", buf);
2377 	}
2378 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
2379 	    DS_FIELD_RESUME_LARGEBLOCK) == 0) {
2380 		fnvlist_add_boolean(token_nv, "largeblockok");
2381 	}
2382 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
2383 	    DS_FIELD_RESUME_EMBEDOK) == 0) {
2384 		fnvlist_add_boolean(token_nv, "embedok");
2385 	}
2386 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
2387 	    DS_FIELD_RESUME_COMPRESSOK) == 0) {
2388 		fnvlist_add_boolean(token_nv, "compressok");
2389 	}
2390 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
2391 	    DS_FIELD_RESUME_RAWOK) == 0) {
2392 		fnvlist_add_boolean(token_nv, "rawok");
2393 	}
2394 	if (dsl_dataset_feature_is_active(ds,
2395 	    SPA_FEATURE_REDACTED_DATASETS)) {
2396 		uint64_t num_redact_snaps = 0;
2397 		uint64_t *redact_snaps = NULL;
2398 		VERIFY3B(dsl_dataset_get_uint64_array_feature(ds,
2399 		    SPA_FEATURE_REDACTED_DATASETS, &num_redact_snaps,
2400 		    &redact_snaps), ==, B_TRUE);
2401 		fnvlist_add_uint64_array(token_nv, "redact_snaps",
2402 		    redact_snaps, num_redact_snaps);
2403 	}
2404 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
2405 	    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS) == 0) {
2406 		uint64_t num_redact_snaps = 0, int_size = 0;
2407 		uint64_t *redact_snaps = NULL;
2408 		VERIFY0(zap_length(dp->dp_meta_objset, ds->ds_object,
2409 		    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, &int_size,
2410 		    &num_redact_snaps));
2411 		ASSERT3U(int_size, ==, sizeof (uint64_t));
2412 
2413 		redact_snaps = kmem_alloc(int_size * num_redact_snaps,
2414 		    KM_SLEEP);
2415 		VERIFY0(zap_lookup(dp->dp_meta_objset, ds->ds_object,
2416 		    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, int_size,
2417 		    num_redact_snaps, redact_snaps));
2418 		fnvlist_add_uint64_array(token_nv, "book_redact_snaps",
2419 		    redact_snaps, num_redact_snaps);
2420 		kmem_free(redact_snaps, int_size * num_redact_snaps);
2421 	}
2422 	packed = fnvlist_pack(token_nv, &packed_size);
2423 	fnvlist_free(token_nv);
2424 	compressed = kmem_alloc(packed_size, KM_SLEEP);
2425 
2426 	compressed_size = gzip_compress(packed, compressed,
2427 	    packed_size, packed_size, 6);
2428 
2429 	zio_cksum_t cksum;
2430 	fletcher_4_native_varsize(compressed, compressed_size, &cksum);
2431 
2432 	size_t alloc_size = compressed_size * 2 + 1;
2433 	str = kmem_alloc(alloc_size, KM_SLEEP);
2434 	for (int i = 0; i < compressed_size; i++) {
2435 		size_t offset = i * 2;
2436 		(void) snprintf(str + offset, alloc_size - offset,
2437 	    "%02x", compressed[i]);
2438 	}
2439 	str[compressed_size * 2] = '\0';
2440 	char *propval = kmem_asprintf("%u-%llx-%llx-%s",
2441 	    ZFS_SEND_RESUME_TOKEN_VERSION,
2442 	    (longlong_t)cksum.zc_word[0],
2443 	    (longlong_t)packed_size, str);
2444 	kmem_free(packed, packed_size);
2445 	kmem_free(str, alloc_size);
2446 	kmem_free(compressed, packed_size);
2447 	return (propval);
2448 }
2449 
2450 /*
2451  * Returns a string that represents the receive resume state token. It should
2452  * be freed with strfree(). NULL is returned if no resume state is present.
2453  */
2454 char *
2455 get_receive_resume_token(dsl_dataset_t *ds)
2456 {
2457 	/*
2458 	 * A failed "newfs" (e.g. full) resumable receive leaves
2459 	 * the stats set on this dataset.  Check here for the prop.
2460 	 */
2461 	char *token = get_receive_resume_token_impl(ds);
2462 	if (token != NULL)
2463 		return (token);
2464 	/*
2465 	 * A failed incremental resumable receive leaves the
2466 	 * stats set on our child named "%recv".  Check the child
2467 	 * for the prop.
2468 	 */
2469 	/* 6 extra bytes for /%recv */
2470 	char name[ZFS_MAX_DATASET_NAME_LEN + 6];
2471 	dsl_dataset_t *recv_ds;
2472 	dsl_dataset_name(ds, name);
2473 	if (strlcat(name, "/", sizeof (name)) < sizeof (name) &&
2474 	    strlcat(name, recv_clone_name, sizeof (name)) < sizeof (name) &&
2475 	    dsl_dataset_hold(ds->ds_dir->dd_pool, name, FTAG, &recv_ds) == 0) {
2476 		token = get_receive_resume_token_impl(recv_ds);
2477 		dsl_dataset_rele(recv_ds, FTAG);
2478 	}
2479 	return (token);
2480 }
2481 
2482 uint64_t
2483 dsl_get_refratio(dsl_dataset_t *ds)
2484 {
2485 	uint64_t ratio = dsl_dataset_phys(ds)->ds_compressed_bytes == 0 ? 100 :
2486 	    (dsl_dataset_phys(ds)->ds_uncompressed_bytes * 100 /
2487 	    dsl_dataset_phys(ds)->ds_compressed_bytes);
2488 	return (ratio);
2489 }
2490 
2491 uint64_t
2492 dsl_get_logicalreferenced(dsl_dataset_t *ds)
2493 {
2494 	return (dsl_dataset_phys(ds)->ds_uncompressed_bytes);
2495 }
2496 
2497 uint64_t
2498 dsl_get_compressratio(dsl_dataset_t *ds)
2499 {
2500 	if (ds->ds_is_snapshot) {
2501 		return (dsl_get_refratio(ds));
2502 	} else {
2503 		dsl_dir_t *dd = ds->ds_dir;
2504 		mutex_enter(&dd->dd_lock);
2505 		uint64_t val = dsl_dir_get_compressratio(dd);
2506 		mutex_exit(&dd->dd_lock);
2507 		return (val);
2508 	}
2509 }
2510 
2511 uint64_t
2512 dsl_get_used(dsl_dataset_t *ds)
2513 {
2514 	if (ds->ds_is_snapshot) {
2515 		return (dsl_dataset_phys(ds)->ds_unique_bytes);
2516 	} else {
2517 		dsl_dir_t *dd = ds->ds_dir;
2518 		mutex_enter(&dd->dd_lock);
2519 		uint64_t val = dsl_dir_get_used(dd);
2520 		mutex_exit(&dd->dd_lock);
2521 		return (val);
2522 	}
2523 }
2524 
2525 uint64_t
2526 dsl_get_creation(dsl_dataset_t *ds)
2527 {
2528 	return (dsl_dataset_phys(ds)->ds_creation_time);
2529 }
2530 
2531 uint64_t
2532 dsl_get_creationtxg(dsl_dataset_t *ds)
2533 {
2534 	return (dsl_dataset_phys(ds)->ds_creation_txg);
2535 }
2536 
2537 uint64_t
2538 dsl_get_refquota(dsl_dataset_t *ds)
2539 {
2540 	return (ds->ds_quota);
2541 }
2542 
2543 uint64_t
2544 dsl_get_refreservation(dsl_dataset_t *ds)
2545 {
2546 	return (ds->ds_reserved);
2547 }
2548 
2549 uint64_t
2550 dsl_get_guid(dsl_dataset_t *ds)
2551 {
2552 	return (dsl_dataset_phys(ds)->ds_guid);
2553 }
2554 
2555 uint64_t
2556 dsl_get_unique(dsl_dataset_t *ds)
2557 {
2558 	return (dsl_dataset_phys(ds)->ds_unique_bytes);
2559 }
2560 
2561 uint64_t
2562 dsl_get_objsetid(dsl_dataset_t *ds)
2563 {
2564 	return (ds->ds_object);
2565 }
2566 
2567 uint64_t
2568 dsl_get_userrefs(dsl_dataset_t *ds)
2569 {
2570 	return (ds->ds_userrefs);
2571 }
2572 
2573 uint64_t
2574 dsl_get_defer_destroy(dsl_dataset_t *ds)
2575 {
2576 	return (DS_IS_DEFER_DESTROY(ds) ? 1 : 0);
2577 }
2578 
2579 uint64_t
2580 dsl_get_referenced(dsl_dataset_t *ds)
2581 {
2582 	return (dsl_dataset_phys(ds)->ds_referenced_bytes);
2583 }
2584 
2585 uint64_t
2586 dsl_get_numclones(dsl_dataset_t *ds)
2587 {
2588 	ASSERT(ds->ds_is_snapshot);
2589 	return (dsl_dataset_phys(ds)->ds_num_children - 1);
2590 }
2591 
2592 uint64_t
2593 dsl_get_inconsistent(dsl_dataset_t *ds)
2594 {
2595 	return ((dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT) ?
2596 	    1 : 0);
2597 }
2598 
2599 uint64_t
2600 dsl_get_redacted(dsl_dataset_t *ds)
2601 {
2602 	return (dsl_dataset_feature_is_active(ds,
2603 	    SPA_FEATURE_REDACTED_DATASETS));
2604 }
2605 
2606 uint64_t
2607 dsl_get_available(dsl_dataset_t *ds)
2608 {
2609 	uint64_t refdbytes = dsl_get_referenced(ds);
2610 	uint64_t availbytes = dsl_dir_space_available(ds->ds_dir,
2611 	    NULL, 0, TRUE);
2612 	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes) {
2613 		availbytes +=
2614 		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes;
2615 	}
2616 	if (ds->ds_quota != 0) {
2617 		/*
2618 		 * Adjust available bytes according to refquota
2619 		 */
2620 		if (refdbytes < ds->ds_quota) {
2621 			availbytes = MIN(availbytes,
2622 			    ds->ds_quota - refdbytes);
2623 		} else {
2624 			availbytes = 0;
2625 		}
2626 	}
2627 	return (availbytes);
2628 }
2629 
2630 int
2631 dsl_get_written(dsl_dataset_t *ds, uint64_t *written)
2632 {
2633 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
2634 	dsl_dataset_t *prev;
2635 	int err = dsl_dataset_hold_obj(dp,
2636 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
2637 	if (err == 0) {
2638 		uint64_t comp, uncomp;
2639 		err = dsl_dataset_space_written(prev, ds, written,
2640 		    &comp, &uncomp);
2641 		dsl_dataset_rele(prev, FTAG);
2642 	}
2643 	return (err);
2644 }
2645 
2646 /*
2647  * 'snap' should be a buffer of size ZFS_MAX_DATASET_NAME_LEN.
2648  */
2649 int
2650 dsl_get_prev_snap(dsl_dataset_t *ds, char *snap)
2651 {
2652 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
2653 	if (ds->ds_prev != NULL && ds->ds_prev != dp->dp_origin_snap) {
2654 		dsl_dataset_name(ds->ds_prev, snap);
2655 		return (0);
2656 	} else {
2657 		return (SET_ERROR(ENOENT));
2658 	}
2659 }
2660 
2661 void
2662 dsl_get_redact_snaps(dsl_dataset_t *ds, nvlist_t *propval)
2663 {
2664 	uint64_t nsnaps;
2665 	uint64_t *snaps;
2666 	if (dsl_dataset_get_uint64_array_feature(ds,
2667 	    SPA_FEATURE_REDACTED_DATASETS, &nsnaps, &snaps)) {
2668 		fnvlist_add_uint64_array(propval, ZPROP_VALUE, snaps,
2669 		    nsnaps);
2670 	}
2671 }
2672 
2673 /*
2674  * Returns the mountpoint property and source for the given dataset in the value
2675  * and source buffers. The value buffer must be at least as large as MAXPATHLEN
2676  * and the source buffer as least as large a ZFS_MAX_DATASET_NAME_LEN.
2677  * Returns 0 on success and an error on failure.
2678  */
2679 int
2680 dsl_get_mountpoint(dsl_dataset_t *ds, const char *dsname, char *value,
2681     char *source)
2682 {
2683 	int error;
2684 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
2685 
2686 	/* Retrieve the mountpoint value stored in the zap object */
2687 	error = dsl_prop_get_ds(ds, zfs_prop_to_name(ZFS_PROP_MOUNTPOINT), 1,
2688 	    ZAP_MAXVALUELEN, value, source);
2689 	if (error != 0) {
2690 		return (error);
2691 	}
2692 
2693 	/*
2694 	 * Process the dsname and source to find the full mountpoint string.
2695 	 * Can be skipped for 'legacy' or 'none'.
2696 	 */
2697 	if (value[0] == '/') {
2698 		char *buf = kmem_alloc(ZAP_MAXVALUELEN, KM_SLEEP);
2699 		char *root = buf;
2700 		const char *relpath;
2701 
2702 		/*
2703 		 * If we inherit the mountpoint, even from a dataset
2704 		 * with a received value, the source will be the path of
2705 		 * the dataset we inherit from. If source is
2706 		 * ZPROP_SOURCE_VAL_RECVD, the received value is not
2707 		 * inherited.
2708 		 */
2709 		if (strcmp(source, ZPROP_SOURCE_VAL_RECVD) == 0) {
2710 			relpath = "";
2711 		} else {
2712 			ASSERT0(strncmp(dsname, source, strlen(source)));
2713 			relpath = dsname + strlen(source);
2714 			if (relpath[0] == '/')
2715 				relpath++;
2716 		}
2717 
2718 		spa_altroot(dp->dp_spa, root, ZAP_MAXVALUELEN);
2719 
2720 		/*
2721 		 * Special case an alternate root of '/'. This will
2722 		 * avoid having multiple leading slashes in the
2723 		 * mountpoint path.
2724 		 */
2725 		if (strcmp(root, "/") == 0)
2726 			root++;
2727 
2728 		/*
2729 		 * If the mountpoint is '/' then skip over this
2730 		 * if we are obtaining either an alternate root or
2731 		 * an inherited mountpoint.
2732 		 */
2733 		char *mnt = value;
2734 		if (value[1] == '\0' && (root[0] != '\0' ||
2735 		    relpath[0] != '\0'))
2736 			mnt = value + 1;
2737 
2738 		if (relpath[0] == '\0') {
2739 			(void) snprintf(value, ZAP_MAXVALUELEN, "%s%s",
2740 			    root, mnt);
2741 		} else {
2742 			(void) snprintf(value, ZAP_MAXVALUELEN, "%s%s%s%s",
2743 			    root, mnt, relpath[0] == '@' ? "" : "/",
2744 			    relpath);
2745 		}
2746 		kmem_free(buf, ZAP_MAXVALUELEN);
2747 	}
2748 
2749 	return (0);
2750 }
2751 
2752 void
2753 dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
2754 {
2755 	dsl_pool_t *dp __maybe_unused = ds->ds_dir->dd_pool;
2756 
2757 	ASSERT(dsl_pool_config_held(dp));
2758 
2759 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFRATIO,
2760 	    dsl_get_refratio(ds));
2761 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALREFERENCED,
2762 	    dsl_get_logicalreferenced(ds));
2763 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_COMPRESSRATIO,
2764 	    dsl_get_compressratio(ds));
2765 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USED,
2766 	    dsl_get_used(ds));
2767 
2768 	if (ds->ds_is_snapshot) {
2769 		get_clones_stat(ds, nv);
2770 	} else {
2771 		char buf[ZFS_MAX_DATASET_NAME_LEN];
2772 		if (dsl_get_prev_snap(ds, buf) == 0)
2773 			dsl_prop_nvlist_add_string(nv, ZFS_PROP_PREV_SNAP,
2774 			    buf);
2775 		dsl_dir_stats(ds->ds_dir, nv);
2776 	}
2777 
2778 	nvlist_t *propval = fnvlist_alloc();
2779 	dsl_get_redact_snaps(ds, propval);
2780 	fnvlist_add_nvlist(nv, zfs_prop_to_name(ZFS_PROP_REDACT_SNAPS),
2781 	    propval);
2782 	nvlist_free(propval);
2783 
2784 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_AVAILABLE,
2785 	    dsl_get_available(ds));
2786 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFERENCED,
2787 	    dsl_get_referenced(ds));
2788 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_CREATION,
2789 	    dsl_get_creation(ds));
2790 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_CREATETXG,
2791 	    dsl_get_creationtxg(ds));
2792 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFQUOTA,
2793 	    dsl_get_refquota(ds));
2794 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFRESERVATION,
2795 	    dsl_get_refreservation(ds));
2796 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_GUID,
2797 	    dsl_get_guid(ds));
2798 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_UNIQUE,
2799 	    dsl_get_unique(ds));
2800 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_OBJSETID,
2801 	    dsl_get_objsetid(ds));
2802 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USERREFS,
2803 	    dsl_get_userrefs(ds));
2804 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_DEFER_DESTROY,
2805 	    dsl_get_defer_destroy(ds));
2806 	dsl_dataset_crypt_stats(ds, nv);
2807 
2808 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
2809 		uint64_t written;
2810 		if (dsl_get_written(ds, &written) == 0) {
2811 			dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_WRITTEN,
2812 			    written);
2813 		}
2814 	}
2815 
2816 	if (!dsl_dataset_is_snapshot(ds)) {
2817 		char *token = get_receive_resume_token(ds);
2818 		if (token != NULL) {
2819 			dsl_prop_nvlist_add_string(nv,
2820 			    ZFS_PROP_RECEIVE_RESUME_TOKEN, token);
2821 			kmem_strfree(token);
2822 		}
2823 	}
2824 }
2825 
2826 void
2827 dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat)
2828 {
2829 	dsl_pool_t *dp __maybe_unused = ds->ds_dir->dd_pool;
2830 	ASSERT(dsl_pool_config_held(dp));
2831 
2832 	stat->dds_creation_txg = dsl_get_creationtxg(ds);
2833 	stat->dds_inconsistent = dsl_get_inconsistent(ds);
2834 	stat->dds_guid = dsl_get_guid(ds);
2835 	stat->dds_redacted = dsl_get_redacted(ds);
2836 	stat->dds_origin[0] = '\0';
2837 	if (ds->ds_is_snapshot) {
2838 		stat->dds_is_snapshot = B_TRUE;
2839 		stat->dds_num_clones = dsl_get_numclones(ds);
2840 	} else {
2841 		stat->dds_is_snapshot = B_FALSE;
2842 		stat->dds_num_clones = 0;
2843 
2844 		if (dsl_dir_is_clone(ds->ds_dir)) {
2845 			dsl_dir_get_origin(ds->ds_dir, stat->dds_origin);
2846 		}
2847 	}
2848 }
2849 
2850 uint64_t
2851 dsl_dataset_fsid_guid(dsl_dataset_t *ds)
2852 {
2853 	return (ds->ds_fsid_guid);
2854 }
2855 
2856 void
2857 dsl_dataset_space(dsl_dataset_t *ds,
2858     uint64_t *refdbytesp, uint64_t *availbytesp,
2859     uint64_t *usedobjsp, uint64_t *availobjsp)
2860 {
2861 	*refdbytesp = dsl_dataset_phys(ds)->ds_referenced_bytes;
2862 	*availbytesp = dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE);
2863 	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes)
2864 		*availbytesp +=
2865 		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes;
2866 	if (ds->ds_quota != 0) {
2867 		/*
2868 		 * Adjust available bytes according to refquota
2869 		 */
2870 		if (*refdbytesp < ds->ds_quota)
2871 			*availbytesp = MIN(*availbytesp,
2872 			    ds->ds_quota - *refdbytesp);
2873 		else
2874 			*availbytesp = 0;
2875 	}
2876 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
2877 	*usedobjsp = BP_GET_FILL(&dsl_dataset_phys(ds)->ds_bp);
2878 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
2879 	*availobjsp = DN_MAX_OBJECT - *usedobjsp;
2880 }
2881 
2882 boolean_t
2883 dsl_dataset_modified_since_snap(dsl_dataset_t *ds, dsl_dataset_t *snap)
2884 {
2885 	dsl_pool_t *dp __maybe_unused = ds->ds_dir->dd_pool;
2886 	uint64_t birth;
2887 
2888 	ASSERT(dsl_pool_config_held(dp));
2889 	if (snap == NULL)
2890 		return (B_FALSE);
2891 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
2892 	birth = dsl_dataset_get_blkptr(ds)->blk_birth;
2893 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
2894 	if (birth > dsl_dataset_phys(snap)->ds_creation_txg) {
2895 		objset_t *os, *os_snap;
2896 		/*
2897 		 * It may be that only the ZIL differs, because it was
2898 		 * reset in the head.  Don't count that as being
2899 		 * modified.
2900 		 */
2901 		if (dmu_objset_from_ds(ds, &os) != 0)
2902 			return (B_TRUE);
2903 		if (dmu_objset_from_ds(snap, &os_snap) != 0)
2904 			return (B_TRUE);
2905 		return (memcmp(&os->os_phys->os_meta_dnode,
2906 		    &os_snap->os_phys->os_meta_dnode,
2907 		    sizeof (os->os_phys->os_meta_dnode)) != 0);
2908 	}
2909 	return (B_FALSE);
2910 }
2911 
2912 typedef struct dsl_dataset_rename_snapshot_arg {
2913 	const char *ddrsa_fsname;
2914 	const char *ddrsa_oldsnapname;
2915 	const char *ddrsa_newsnapname;
2916 	boolean_t ddrsa_recursive;
2917 	dmu_tx_t *ddrsa_tx;
2918 } dsl_dataset_rename_snapshot_arg_t;
2919 
2920 static int
2921 dsl_dataset_rename_snapshot_check_impl(dsl_pool_t *dp,
2922     dsl_dataset_t *hds, void *arg)
2923 {
2924 	(void) dp;
2925 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
2926 	int error;
2927 	uint64_t val;
2928 
2929 	error = dsl_dataset_snap_lookup(hds, ddrsa->ddrsa_oldsnapname, &val);
2930 	if (error != 0) {
2931 		/* ignore nonexistent snapshots */
2932 		return (error == ENOENT ? 0 : error);
2933 	}
2934 
2935 	/* new name should not exist */
2936 	error = dsl_dataset_snap_lookup(hds, ddrsa->ddrsa_newsnapname, &val);
2937 	if (error == 0)
2938 		error = SET_ERROR(EEXIST);
2939 	else if (error == ENOENT)
2940 		error = 0;
2941 
2942 	/* dataset name + 1 for the "@" + the new snapshot name must fit */
2943 	if (dsl_dir_namelen(hds->ds_dir) + 1 +
2944 	    strlen(ddrsa->ddrsa_newsnapname) >= ZFS_MAX_DATASET_NAME_LEN)
2945 		error = SET_ERROR(ENAMETOOLONG);
2946 
2947 	return (error);
2948 }
2949 
2950 static int
2951 dsl_dataset_rename_snapshot_check(void *arg, dmu_tx_t *tx)
2952 {
2953 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
2954 	dsl_pool_t *dp = dmu_tx_pool(tx);
2955 	dsl_dataset_t *hds;
2956 	int error;
2957 
2958 	error = dsl_dataset_hold(dp, ddrsa->ddrsa_fsname, FTAG, &hds);
2959 	if (error != 0)
2960 		return (error);
2961 
2962 	if (ddrsa->ddrsa_recursive) {
2963 		error = dmu_objset_find_dp(dp, hds->ds_dir->dd_object,
2964 		    dsl_dataset_rename_snapshot_check_impl, ddrsa,
2965 		    DS_FIND_CHILDREN);
2966 	} else {
2967 		error = dsl_dataset_rename_snapshot_check_impl(dp, hds, ddrsa);
2968 	}
2969 	dsl_dataset_rele(hds, FTAG);
2970 	return (error);
2971 }
2972 
2973 static int
2974 dsl_dataset_rename_snapshot_sync_impl(dsl_pool_t *dp,
2975     dsl_dataset_t *hds, void *arg)
2976 {
2977 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
2978 	dsl_dataset_t *ds;
2979 	uint64_t val;
2980 	dmu_tx_t *tx = ddrsa->ddrsa_tx;
2981 	int error;
2982 
2983 	error = dsl_dataset_snap_lookup(hds, ddrsa->ddrsa_oldsnapname, &val);
2984 	ASSERT(error == 0 || error == ENOENT);
2985 	if (error == ENOENT) {
2986 		/* ignore nonexistent snapshots */
2987 		return (0);
2988 	}
2989 
2990 	VERIFY0(dsl_dataset_hold_obj(dp, val, FTAG, &ds));
2991 
2992 	/* log before we change the name */
2993 	spa_history_log_internal_ds(ds, "rename", tx,
2994 	    "-> @%s", ddrsa->ddrsa_newsnapname);
2995 
2996 	VERIFY0(dsl_dataset_snap_remove(hds, ddrsa->ddrsa_oldsnapname, tx,
2997 	    B_FALSE));
2998 	mutex_enter(&ds->ds_lock);
2999 	(void) strlcpy(ds->ds_snapname, ddrsa->ddrsa_newsnapname,
3000 	    sizeof (ds->ds_snapname));
3001 	mutex_exit(&ds->ds_lock);
3002 	VERIFY0(zap_add(dp->dp_meta_objset,
3003 	    dsl_dataset_phys(hds)->ds_snapnames_zapobj,
3004 	    ds->ds_snapname, 8, 1, &ds->ds_object, tx));
3005 	zvol_rename_minors(dp->dp_spa, ddrsa->ddrsa_oldsnapname,
3006 	    ddrsa->ddrsa_newsnapname, B_TRUE);
3007 
3008 	dsl_dataset_rele(ds, FTAG);
3009 	return (0);
3010 }
3011 
3012 static void
3013 dsl_dataset_rename_snapshot_sync(void *arg, dmu_tx_t *tx)
3014 {
3015 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
3016 	dsl_pool_t *dp = dmu_tx_pool(tx);
3017 	dsl_dataset_t *hds = NULL;
3018 
3019 	VERIFY0(dsl_dataset_hold(dp, ddrsa->ddrsa_fsname, FTAG, &hds));
3020 	ddrsa->ddrsa_tx = tx;
3021 	if (ddrsa->ddrsa_recursive) {
3022 		VERIFY0(dmu_objset_find_dp(dp, hds->ds_dir->dd_object,
3023 		    dsl_dataset_rename_snapshot_sync_impl, ddrsa,
3024 		    DS_FIND_CHILDREN));
3025 	} else {
3026 		VERIFY0(dsl_dataset_rename_snapshot_sync_impl(dp, hds, ddrsa));
3027 	}
3028 	dsl_dataset_rele(hds, FTAG);
3029 }
3030 
3031 int
3032 dsl_dataset_rename_snapshot(const char *fsname,
3033     const char *oldsnapname, const char *newsnapname, boolean_t recursive)
3034 {
3035 	dsl_dataset_rename_snapshot_arg_t ddrsa;
3036 
3037 	ddrsa.ddrsa_fsname = fsname;
3038 	ddrsa.ddrsa_oldsnapname = oldsnapname;
3039 	ddrsa.ddrsa_newsnapname = newsnapname;
3040 	ddrsa.ddrsa_recursive = recursive;
3041 
3042 	return (dsl_sync_task(fsname, dsl_dataset_rename_snapshot_check,
3043 	    dsl_dataset_rename_snapshot_sync, &ddrsa,
3044 	    1, ZFS_SPACE_CHECK_RESERVED));
3045 }
3046 
3047 /*
3048  * If we're doing an ownership handoff, we need to make sure that there is
3049  * only one long hold on the dataset.  We're not allowed to change anything here
3050  * so we don't permanently release the long hold or regular hold here.  We want
3051  * to do this only when syncing to avoid the dataset unexpectedly going away
3052  * when we release the long hold.
3053  */
3054 static int
3055 dsl_dataset_handoff_check(dsl_dataset_t *ds, void *owner, dmu_tx_t *tx)
3056 {
3057 	boolean_t held = B_FALSE;
3058 
3059 	if (!dmu_tx_is_syncing(tx))
3060 		return (0);
3061 
3062 	dsl_dir_t *dd = ds->ds_dir;
3063 	mutex_enter(&dd->dd_activity_lock);
3064 	uint64_t holds = zfs_refcount_count(&ds->ds_longholds) -
3065 	    (owner != NULL ? 1 : 0);
3066 	/*
3067 	 * The value of dd_activity_waiters can chance as soon as we drop the
3068 	 * lock, but we're fine with that; new waiters coming in or old
3069 	 * waiters leaving doesn't cause problems, since we're going to cancel
3070 	 * waiters later anyway. The goal of this check is to verify that no
3071 	 * non-waiters have long-holds, and all new long-holds will be
3072 	 * prevented because we're holding the pool config as writer.
3073 	 */
3074 	if (holds != dd->dd_activity_waiters)
3075 		held = B_TRUE;
3076 	mutex_exit(&dd->dd_activity_lock);
3077 
3078 	if (held)
3079 		return (SET_ERROR(EBUSY));
3080 
3081 	return (0);
3082 }
3083 
3084 int
3085 dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
3086 {
3087 	dsl_dataset_rollback_arg_t *ddra = arg;
3088 	dsl_pool_t *dp = dmu_tx_pool(tx);
3089 	dsl_dataset_t *ds;
3090 	int64_t unused_refres_delta;
3091 	int error;
3092 
3093 	error = dsl_dataset_hold(dp, ddra->ddra_fsname, FTAG, &ds);
3094 	if (error != 0)
3095 		return (error);
3096 
3097 	/* must not be a snapshot */
3098 	if (ds->ds_is_snapshot) {
3099 		dsl_dataset_rele(ds, FTAG);
3100 		return (SET_ERROR(EINVAL));
3101 	}
3102 
3103 	/* must have a most recent snapshot */
3104 	if (dsl_dataset_phys(ds)->ds_prev_snap_txg < TXG_INITIAL) {
3105 		dsl_dataset_rele(ds, FTAG);
3106 		return (SET_ERROR(ESRCH));
3107 	}
3108 
3109 	/*
3110 	 * No rollback to a snapshot created in the current txg, because
3111 	 * the rollback may dirty the dataset and create blocks that are
3112 	 * not reachable from the rootbp while having a birth txg that
3113 	 * falls into the snapshot's range.
3114 	 */
3115 	if (dmu_tx_is_syncing(tx) &&
3116 	    dsl_dataset_phys(ds)->ds_prev_snap_txg >= tx->tx_txg) {
3117 		dsl_dataset_rele(ds, FTAG);
3118 		return (SET_ERROR(EAGAIN));
3119 	}
3120 
3121 	/*
3122 	 * If the expected target snapshot is specified, then check that
3123 	 * the latest snapshot is it.
3124 	 */
3125 	if (ddra->ddra_tosnap != NULL) {
3126 		dsl_dataset_t *snapds;
3127 
3128 		/* Check if the target snapshot exists at all. */
3129 		error = dsl_dataset_hold(dp, ddra->ddra_tosnap, FTAG, &snapds);
3130 		if (error != 0) {
3131 			/*
3132 			 * ESRCH is used to signal that the target snapshot does
3133 			 * not exist, while ENOENT is used to report that
3134 			 * the rolled back dataset does not exist.
3135 			 * ESRCH is also used to cover other cases where the
3136 			 * target snapshot is not related to the dataset being
3137 			 * rolled back such as being in a different pool.
3138 			 */
3139 			if (error == ENOENT || error == EXDEV)
3140 				error = SET_ERROR(ESRCH);
3141 			dsl_dataset_rele(ds, FTAG);
3142 			return (error);
3143 		}
3144 		ASSERT(snapds->ds_is_snapshot);
3145 
3146 		/* Check if the snapshot is the latest snapshot indeed. */
3147 		if (snapds != ds->ds_prev) {
3148 			/*
3149 			 * Distinguish between the case where the only problem
3150 			 * is intervening snapshots (EEXIST) vs the snapshot
3151 			 * not being a valid target for rollback (ESRCH).
3152 			 */
3153 			if (snapds->ds_dir == ds->ds_dir ||
3154 			    (dsl_dir_is_clone(ds->ds_dir) &&
3155 			    dsl_dir_phys(ds->ds_dir)->dd_origin_obj ==
3156 			    snapds->ds_object)) {
3157 				error = SET_ERROR(EEXIST);
3158 			} else {
3159 				error = SET_ERROR(ESRCH);
3160 			}
3161 			dsl_dataset_rele(snapds, FTAG);
3162 			dsl_dataset_rele(ds, FTAG);
3163 			return (error);
3164 		}
3165 		dsl_dataset_rele(snapds, FTAG);
3166 	}
3167 
3168 	/* must not have any bookmarks after the most recent snapshot */
3169 	if (dsl_bookmark_latest_txg(ds) >
3170 	    dsl_dataset_phys(ds)->ds_prev_snap_txg) {
3171 		dsl_dataset_rele(ds, FTAG);
3172 		return (SET_ERROR(EEXIST));
3173 	}
3174 
3175 	error = dsl_dataset_handoff_check(ds, ddra->ddra_owner, tx);
3176 	if (error != 0) {
3177 		dsl_dataset_rele(ds, FTAG);
3178 		return (error);
3179 	}
3180 
3181 	/*
3182 	 * Check if the snap we are rolling back to uses more than
3183 	 * the refquota.
3184 	 */
3185 	if (ds->ds_quota != 0 &&
3186 	    dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes > ds->ds_quota) {
3187 		dsl_dataset_rele(ds, FTAG);
3188 		return (SET_ERROR(EDQUOT));
3189 	}
3190 
3191 	/*
3192 	 * When we do the clone swap, we will temporarily use more space
3193 	 * due to the refreservation (the head will no longer have any
3194 	 * unique space, so the entire amount of the refreservation will need
3195 	 * to be free).  We will immediately destroy the clone, freeing
3196 	 * this space, but the freeing happens over many txg's.
3197 	 */
3198 	unused_refres_delta = (int64_t)MIN(ds->ds_reserved,
3199 	    dsl_dataset_phys(ds)->ds_unique_bytes);
3200 
3201 	if (unused_refres_delta > 0 &&
3202 	    unused_refres_delta >
3203 	    dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE)) {
3204 		dsl_dataset_rele(ds, FTAG);
3205 		return (SET_ERROR(ENOSPC));
3206 	}
3207 
3208 	dsl_dataset_rele(ds, FTAG);
3209 	return (0);
3210 }
3211 
3212 void
3213 dsl_dataset_rollback_sync(void *arg, dmu_tx_t *tx)
3214 {
3215 	dsl_dataset_rollback_arg_t *ddra = arg;
3216 	dsl_pool_t *dp = dmu_tx_pool(tx);
3217 	dsl_dataset_t *ds, *clone;
3218 	uint64_t cloneobj;
3219 	char namebuf[ZFS_MAX_DATASET_NAME_LEN];
3220 
3221 	VERIFY0(dsl_dataset_hold(dp, ddra->ddra_fsname, FTAG, &ds));
3222 
3223 	dsl_dataset_name(ds->ds_prev, namebuf);
3224 	fnvlist_add_string(ddra->ddra_result, "target", namebuf);
3225 
3226 	cloneobj = dsl_dataset_create_sync(ds->ds_dir, "%rollback",
3227 	    ds->ds_prev, DS_CREATE_FLAG_NODIRTY, kcred, NULL, tx);
3228 
3229 	VERIFY0(dsl_dataset_hold_obj(dp, cloneobj, FTAG, &clone));
3230 
3231 	dsl_dataset_clone_swap_sync_impl(clone, ds, tx);
3232 	dsl_dataset_zero_zil(ds, tx);
3233 
3234 	dsl_destroy_head_sync_impl(clone, tx);
3235 
3236 	dsl_dataset_rele(clone, FTAG);
3237 	dsl_dataset_rele(ds, FTAG);
3238 }
3239 
3240 /*
3241  * Rolls back the given filesystem or volume to the most recent snapshot.
3242  * The name of the most recent snapshot will be returned under key "target"
3243  * in the result nvlist.
3244  *
3245  * If owner != NULL:
3246  * - The existing dataset MUST be owned by the specified owner at entry
3247  * - Upon return, dataset will still be held by the same owner, whether we
3248  *   succeed or not.
3249  *
3250  * This mode is required any time the existing filesystem is mounted.  See
3251  * notes above zfs_suspend_fs() for further details.
3252  */
3253 int
3254 dsl_dataset_rollback(const char *fsname, const char *tosnap, void *owner,
3255     nvlist_t *result)
3256 {
3257 	dsl_dataset_rollback_arg_t ddra;
3258 
3259 	ddra.ddra_fsname = fsname;
3260 	ddra.ddra_tosnap = tosnap;
3261 	ddra.ddra_owner = owner;
3262 	ddra.ddra_result = result;
3263 
3264 	return (dsl_sync_task(fsname, dsl_dataset_rollback_check,
3265 	    dsl_dataset_rollback_sync, &ddra,
3266 	    1, ZFS_SPACE_CHECK_RESERVED));
3267 }
3268 
3269 struct promotenode {
3270 	list_node_t link;
3271 	dsl_dataset_t *ds;
3272 };
3273 
3274 static int snaplist_space(list_t *l, uint64_t mintxg, uint64_t *spacep);
3275 static int promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp,
3276     void *tag);
3277 static void promote_rele(dsl_dataset_promote_arg_t *ddpa, void *tag);
3278 
3279 int
3280 dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
3281 {
3282 	dsl_dataset_promote_arg_t *ddpa = arg;
3283 	dsl_pool_t *dp = dmu_tx_pool(tx);
3284 	dsl_dataset_t *hds;
3285 	struct promotenode *snap;
3286 	dsl_dataset_t *origin_ds, *origin_head;
3287 	int err;
3288 	uint64_t unused;
3289 	uint64_t ss_mv_cnt;
3290 	size_t max_snap_len;
3291 	boolean_t conflicting_snaps;
3292 
3293 	err = promote_hold(ddpa, dp, FTAG);
3294 	if (err != 0)
3295 		return (err);
3296 
3297 	hds = ddpa->ddpa_clone;
3298 	max_snap_len = MAXNAMELEN - strlen(ddpa->ddpa_clonename) - 1;
3299 
3300 	if (dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE) {
3301 		promote_rele(ddpa, FTAG);
3302 		return (SET_ERROR(EXDEV));
3303 	}
3304 
3305 	snap = list_head(&ddpa->shared_snaps);
3306 	origin_head = snap->ds;
3307 	if (snap == NULL) {
3308 		err = SET_ERROR(ENOENT);
3309 		goto out;
3310 	}
3311 	origin_ds = snap->ds;
3312 
3313 	/*
3314 	 * Encrypted clones share a DSL Crypto Key with their origin's dsl dir.
3315 	 * When doing a promote we must make sure the encryption root for
3316 	 * both the target and the target's origin does not change to avoid
3317 	 * needing to rewrap encryption keys
3318 	 */
3319 	err = dsl_dataset_promote_crypt_check(hds->ds_dir, origin_ds->ds_dir);
3320 	if (err != 0)
3321 		goto out;
3322 
3323 	/*
3324 	 * Compute and check the amount of space to transfer.  Since this is
3325 	 * so expensive, don't do the preliminary check.
3326 	 */
3327 	if (!dmu_tx_is_syncing(tx)) {
3328 		promote_rele(ddpa, FTAG);
3329 		return (0);
3330 	}
3331 
3332 	/* compute origin's new unique space */
3333 	snap = list_tail(&ddpa->clone_snaps);
3334 	ASSERT(snap != NULL);
3335 	ASSERT3U(dsl_dataset_phys(snap->ds)->ds_prev_snap_obj, ==,
3336 	    origin_ds->ds_object);
3337 	dsl_deadlist_space_range(&snap->ds->ds_deadlist,
3338 	    dsl_dataset_phys(origin_ds)->ds_prev_snap_txg, UINT64_MAX,
3339 	    &ddpa->unique, &unused, &unused);
3340 
3341 	/*
3342 	 * Walk the snapshots that we are moving
3343 	 *
3344 	 * Compute space to transfer.  Consider the incremental changes
3345 	 * to used by each snapshot:
3346 	 * (my used) = (prev's used) + (blocks born) - (blocks killed)
3347 	 * So each snapshot gave birth to:
3348 	 * (blocks born) = (my used) - (prev's used) + (blocks killed)
3349 	 * So a sequence would look like:
3350 	 * (uN - u(N-1) + kN) + ... + (u1 - u0 + k1) + (u0 - 0 + k0)
3351 	 * Which simplifies to:
3352 	 * uN + kN + kN-1 + ... + k1 + k0
3353 	 * Note however, if we stop before we reach the ORIGIN we get:
3354 	 * uN + kN + kN-1 + ... + kM - uM-1
3355 	 */
3356 	conflicting_snaps = B_FALSE;
3357 	ss_mv_cnt = 0;
3358 	ddpa->used = dsl_dataset_phys(origin_ds)->ds_referenced_bytes;
3359 	ddpa->comp = dsl_dataset_phys(origin_ds)->ds_compressed_bytes;
3360 	ddpa->uncomp = dsl_dataset_phys(origin_ds)->ds_uncompressed_bytes;
3361 	for (snap = list_head(&ddpa->shared_snaps); snap;
3362 	    snap = list_next(&ddpa->shared_snaps, snap)) {
3363 		uint64_t val, dlused, dlcomp, dluncomp;
3364 		dsl_dataset_t *ds = snap->ds;
3365 
3366 		ss_mv_cnt++;
3367 
3368 		/*
3369 		 * If there are long holds, we won't be able to evict
3370 		 * the objset.
3371 		 */
3372 		if (dsl_dataset_long_held(ds)) {
3373 			err = SET_ERROR(EBUSY);
3374 			goto out;
3375 		}
3376 
3377 		/* Check that the snapshot name does not conflict */
3378 		VERIFY0(dsl_dataset_get_snapname(ds));
3379 		if (strlen(ds->ds_snapname) >= max_snap_len) {
3380 			err = SET_ERROR(ENAMETOOLONG);
3381 			goto out;
3382 		}
3383 		err = dsl_dataset_snap_lookup(hds, ds->ds_snapname, &val);
3384 		if (err == 0) {
3385 			fnvlist_add_boolean(ddpa->err_ds,
3386 			    snap->ds->ds_snapname);
3387 			conflicting_snaps = B_TRUE;
3388 		} else if (err != ENOENT) {
3389 			goto out;
3390 		}
3391 
3392 		/* The very first snapshot does not have a deadlist */
3393 		if (dsl_dataset_phys(ds)->ds_prev_snap_obj == 0)
3394 			continue;
3395 
3396 		dsl_deadlist_space(&ds->ds_deadlist,
3397 		    &dlused, &dlcomp, &dluncomp);
3398 		ddpa->used += dlused;
3399 		ddpa->comp += dlcomp;
3400 		ddpa->uncomp += dluncomp;
3401 	}
3402 
3403 	/*
3404 	 * Check that bookmarks that are being transferred don't have
3405 	 * name conflicts.
3406 	 */
3407 	for (dsl_bookmark_node_t *dbn = avl_first(&origin_head->ds_bookmarks);
3408 	    dbn != NULL && dbn->dbn_phys.zbm_creation_txg <=
3409 	    dsl_dataset_phys(origin_ds)->ds_creation_txg;
3410 	    dbn = AVL_NEXT(&origin_head->ds_bookmarks, dbn)) {
3411 		if (strlen(dbn->dbn_name) >= max_snap_len) {
3412 			err = SET_ERROR(ENAMETOOLONG);
3413 			goto out;
3414 		}
3415 		zfs_bookmark_phys_t bm;
3416 		err = dsl_bookmark_lookup_impl(ddpa->ddpa_clone,
3417 		    dbn->dbn_name, &bm);
3418 
3419 		if (err == 0) {
3420 			fnvlist_add_boolean(ddpa->err_ds, dbn->dbn_name);
3421 			conflicting_snaps = B_TRUE;
3422 		} else if (err == ESRCH) {
3423 			err = 0;
3424 		} else if (err != 0) {
3425 			goto out;
3426 		}
3427 	}
3428 
3429 	/*
3430 	 * In order to return the full list of conflicting snapshots, we check
3431 	 * whether there was a conflict after traversing all of them.
3432 	 */
3433 	if (conflicting_snaps) {
3434 		err = SET_ERROR(EEXIST);
3435 		goto out;
3436 	}
3437 
3438 	/*
3439 	 * If we are a clone of a clone then we never reached ORIGIN,
3440 	 * so we need to subtract out the clone origin's used space.
3441 	 */
3442 	if (ddpa->origin_origin) {
3443 		ddpa->used -=
3444 		    dsl_dataset_phys(ddpa->origin_origin)->ds_referenced_bytes;
3445 		ddpa->comp -=
3446 		    dsl_dataset_phys(ddpa->origin_origin)->ds_compressed_bytes;
3447 		ddpa->uncomp -=
3448 		    dsl_dataset_phys(ddpa->origin_origin)->
3449 		    ds_uncompressed_bytes;
3450 	}
3451 
3452 	/* Check that there is enough space and limit headroom here */
3453 	err = dsl_dir_transfer_possible(origin_ds->ds_dir, hds->ds_dir,
3454 	    0, ss_mv_cnt, ddpa->used, ddpa->cr, ddpa->proc);
3455 	if (err != 0)
3456 		goto out;
3457 
3458 	/*
3459 	 * Compute the amounts of space that will be used by snapshots
3460 	 * after the promotion (for both origin and clone).  For each,
3461 	 * it is the amount of space that will be on all of their
3462 	 * deadlists (that was not born before their new origin).
3463 	 */
3464 	if (dsl_dir_phys(hds->ds_dir)->dd_flags & DD_FLAG_USED_BREAKDOWN) {
3465 		uint64_t space;
3466 
3467 		/*
3468 		 * Note, typically this will not be a clone of a clone,
3469 		 * so dd_origin_txg will be < TXG_INITIAL, so
3470 		 * these snaplist_space() -> dsl_deadlist_space_range()
3471 		 * calls will be fast because they do not have to
3472 		 * iterate over all bps.
3473 		 */
3474 		snap = list_head(&ddpa->origin_snaps);
3475 		if (snap == NULL) {
3476 			err = SET_ERROR(ENOENT);
3477 			goto out;
3478 		}
3479 		err = snaplist_space(&ddpa->shared_snaps,
3480 		    snap->ds->ds_dir->dd_origin_txg, &ddpa->cloneusedsnap);
3481 		if (err != 0)
3482 			goto out;
3483 
3484 		err = snaplist_space(&ddpa->clone_snaps,
3485 		    snap->ds->ds_dir->dd_origin_txg, &space);
3486 		if (err != 0)
3487 			goto out;
3488 		ddpa->cloneusedsnap += space;
3489 	}
3490 	if (dsl_dir_phys(origin_ds->ds_dir)->dd_flags &
3491 	    DD_FLAG_USED_BREAKDOWN) {
3492 		err = snaplist_space(&ddpa->origin_snaps,
3493 		    dsl_dataset_phys(origin_ds)->ds_creation_txg,
3494 		    &ddpa->originusedsnap);
3495 		if (err != 0)
3496 			goto out;
3497 	}
3498 
3499 out:
3500 	promote_rele(ddpa, FTAG);
3501 	return (err);
3502 }
3503 
3504 void
3505 dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
3506 {
3507 	dsl_dataset_promote_arg_t *ddpa = arg;
3508 	dsl_pool_t *dp = dmu_tx_pool(tx);
3509 	dsl_dataset_t *hds;
3510 	struct promotenode *snap;
3511 	dsl_dataset_t *origin_ds;
3512 	dsl_dataset_t *origin_head;
3513 	dsl_dir_t *dd;
3514 	dsl_dir_t *odd = NULL;
3515 	uint64_t oldnext_obj;
3516 	int64_t delta;
3517 
3518 	ASSERT(nvlist_empty(ddpa->err_ds));
3519 
3520 	VERIFY0(promote_hold(ddpa, dp, FTAG));
3521 	hds = ddpa->ddpa_clone;
3522 
3523 	ASSERT0(dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE);
3524 
3525 	snap = list_head(&ddpa->shared_snaps);
3526 	origin_ds = snap->ds;
3527 	dd = hds->ds_dir;
3528 
3529 	snap = list_head(&ddpa->origin_snaps);
3530 	origin_head = snap->ds;
3531 
3532 	/*
3533 	 * We need to explicitly open odd, since origin_ds's dd will be
3534 	 * changing.
3535 	 */
3536 	VERIFY0(dsl_dir_hold_obj(dp, origin_ds->ds_dir->dd_object,
3537 	    NULL, FTAG, &odd));
3538 
3539 	dsl_dataset_promote_crypt_sync(hds->ds_dir, odd, tx);
3540 
3541 	/* change origin's next snap */
3542 	dmu_buf_will_dirty(origin_ds->ds_dbuf, tx);
3543 	oldnext_obj = dsl_dataset_phys(origin_ds)->ds_next_snap_obj;
3544 	snap = list_tail(&ddpa->clone_snaps);
3545 	ASSERT3U(dsl_dataset_phys(snap->ds)->ds_prev_snap_obj, ==,
3546 	    origin_ds->ds_object);
3547 	dsl_dataset_phys(origin_ds)->ds_next_snap_obj = snap->ds->ds_object;
3548 
3549 	/* change the origin's next clone */
3550 	if (dsl_dataset_phys(origin_ds)->ds_next_clones_obj) {
3551 		dsl_dataset_remove_from_next_clones(origin_ds,
3552 		    snap->ds->ds_object, tx);
3553 		VERIFY0(zap_add_int(dp->dp_meta_objset,
3554 		    dsl_dataset_phys(origin_ds)->ds_next_clones_obj,
3555 		    oldnext_obj, tx));
3556 	}
3557 
3558 	/* change origin */
3559 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
3560 	ASSERT3U(dsl_dir_phys(dd)->dd_origin_obj, ==, origin_ds->ds_object);
3561 	dsl_dir_phys(dd)->dd_origin_obj = dsl_dir_phys(odd)->dd_origin_obj;
3562 	dd->dd_origin_txg = origin_head->ds_dir->dd_origin_txg;
3563 	dmu_buf_will_dirty(odd->dd_dbuf, tx);
3564 	dsl_dir_phys(odd)->dd_origin_obj = origin_ds->ds_object;
3565 	origin_head->ds_dir->dd_origin_txg =
3566 	    dsl_dataset_phys(origin_ds)->ds_creation_txg;
3567 
3568 	/* change dd_clone entries */
3569 	if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
3570 		VERIFY0(zap_remove_int(dp->dp_meta_objset,
3571 		    dsl_dir_phys(odd)->dd_clones, hds->ds_object, tx));
3572 		VERIFY0(zap_add_int(dp->dp_meta_objset,
3573 		    dsl_dir_phys(ddpa->origin_origin->ds_dir)->dd_clones,
3574 		    hds->ds_object, tx));
3575 
3576 		VERIFY0(zap_remove_int(dp->dp_meta_objset,
3577 		    dsl_dir_phys(ddpa->origin_origin->ds_dir)->dd_clones,
3578 		    origin_head->ds_object, tx));
3579 		if (dsl_dir_phys(dd)->dd_clones == 0) {
3580 			dsl_dir_phys(dd)->dd_clones =
3581 			    zap_create(dp->dp_meta_objset, DMU_OT_DSL_CLONES,
3582 			    DMU_OT_NONE, 0, tx);
3583 		}
3584 		VERIFY0(zap_add_int(dp->dp_meta_objset,
3585 		    dsl_dir_phys(dd)->dd_clones, origin_head->ds_object, tx));
3586 	}
3587 
3588 	/*
3589 	 * Move bookmarks to this dir.
3590 	 */
3591 	dsl_bookmark_node_t *dbn_next;
3592 	for (dsl_bookmark_node_t *dbn = avl_first(&origin_head->ds_bookmarks);
3593 	    dbn != NULL && dbn->dbn_phys.zbm_creation_txg <=
3594 	    dsl_dataset_phys(origin_ds)->ds_creation_txg;
3595 	    dbn = dbn_next) {
3596 		dbn_next = AVL_NEXT(&origin_head->ds_bookmarks, dbn);
3597 
3598 		avl_remove(&origin_head->ds_bookmarks, dbn);
3599 		VERIFY0(zap_remove(dp->dp_meta_objset,
3600 		    origin_head->ds_bookmarks_obj, dbn->dbn_name, tx));
3601 
3602 		dsl_bookmark_node_add(hds, dbn, tx);
3603 	}
3604 
3605 	dsl_bookmark_next_changed(hds, origin_ds, tx);
3606 
3607 	/* move snapshots to this dir */
3608 	for (snap = list_head(&ddpa->shared_snaps); snap;
3609 	    snap = list_next(&ddpa->shared_snaps, snap)) {
3610 		dsl_dataset_t *ds = snap->ds;
3611 
3612 		/*
3613 		 * Property callbacks are registered to a particular
3614 		 * dsl_dir.  Since ours is changing, evict the objset
3615 		 * so that they will be unregistered from the old dsl_dir.
3616 		 */
3617 		if (ds->ds_objset) {
3618 			dmu_objset_evict(ds->ds_objset);
3619 			ds->ds_objset = NULL;
3620 		}
3621 
3622 		/* move snap name entry */
3623 		VERIFY0(dsl_dataset_get_snapname(ds));
3624 		VERIFY0(dsl_dataset_snap_remove(origin_head,
3625 		    ds->ds_snapname, tx, B_TRUE));
3626 		VERIFY0(zap_add(dp->dp_meta_objset,
3627 		    dsl_dataset_phys(hds)->ds_snapnames_zapobj, ds->ds_snapname,
3628 		    8, 1, &ds->ds_object, tx));
3629 		dsl_fs_ss_count_adjust(hds->ds_dir, 1,
3630 		    DD_FIELD_SNAPSHOT_COUNT, tx);
3631 
3632 		/* change containing dsl_dir */
3633 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
3634 		ASSERT3U(dsl_dataset_phys(ds)->ds_dir_obj, ==, odd->dd_object);
3635 		dsl_dataset_phys(ds)->ds_dir_obj = dd->dd_object;
3636 		ASSERT3P(ds->ds_dir, ==, odd);
3637 		dsl_dir_rele(ds->ds_dir, ds);
3638 		VERIFY0(dsl_dir_hold_obj(dp, dd->dd_object,
3639 		    NULL, ds, &ds->ds_dir));
3640 
3641 		/* move any clone references */
3642 		if (dsl_dataset_phys(ds)->ds_next_clones_obj &&
3643 		    spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
3644 			zap_cursor_t zc;
3645 			zap_attribute_t za;
3646 
3647 			for (zap_cursor_init(&zc, dp->dp_meta_objset,
3648 			    dsl_dataset_phys(ds)->ds_next_clones_obj);
3649 			    zap_cursor_retrieve(&zc, &za) == 0;
3650 			    zap_cursor_advance(&zc)) {
3651 				dsl_dataset_t *cnds;
3652 				uint64_t o;
3653 
3654 				if (za.za_first_integer == oldnext_obj) {
3655 					/*
3656 					 * We've already moved the
3657 					 * origin's reference.
3658 					 */
3659 					continue;
3660 				}
3661 
3662 				VERIFY0(dsl_dataset_hold_obj(dp,
3663 				    za.za_first_integer, FTAG, &cnds));
3664 				o = dsl_dir_phys(cnds->ds_dir)->
3665 				    dd_head_dataset_obj;
3666 
3667 				VERIFY0(zap_remove_int(dp->dp_meta_objset,
3668 				    dsl_dir_phys(odd)->dd_clones, o, tx));
3669 				VERIFY0(zap_add_int(dp->dp_meta_objset,
3670 				    dsl_dir_phys(dd)->dd_clones, o, tx));
3671 				dsl_dataset_rele(cnds, FTAG);
3672 			}
3673 			zap_cursor_fini(&zc);
3674 		}
3675 
3676 		ASSERT(!dsl_prop_hascb(ds));
3677 	}
3678 
3679 	/*
3680 	 * Change space accounting.
3681 	 * Note, pa->*usedsnap and dd_used_breakdown[SNAP] will either
3682 	 * both be valid, or both be 0 (resulting in delta == 0).  This
3683 	 * is true for each of {clone,origin} independently.
3684 	 */
3685 
3686 	delta = ddpa->cloneusedsnap -
3687 	    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_SNAP];
3688 	ASSERT3S(delta, >=, 0);
3689 	ASSERT3U(ddpa->used, >=, delta);
3690 	dsl_dir_diduse_space(dd, DD_USED_SNAP, delta, 0, 0, tx);
3691 	dsl_dir_diduse_space(dd, DD_USED_HEAD,
3692 	    ddpa->used - delta, ddpa->comp, ddpa->uncomp, tx);
3693 
3694 	delta = ddpa->originusedsnap -
3695 	    dsl_dir_phys(odd)->dd_used_breakdown[DD_USED_SNAP];
3696 	ASSERT3S(delta, <=, 0);
3697 	ASSERT3U(ddpa->used, >=, -delta);
3698 	dsl_dir_diduse_space(odd, DD_USED_SNAP, delta, 0, 0, tx);
3699 	dsl_dir_diduse_space(odd, DD_USED_HEAD,
3700 	    -ddpa->used - delta, -ddpa->comp, -ddpa->uncomp, tx);
3701 
3702 	dsl_dataset_phys(origin_ds)->ds_unique_bytes = ddpa->unique;
3703 
3704 	/*
3705 	 * Since livelists are specific to a clone's origin txg, they
3706 	 * are no longer accurate. Destroy the livelist from the clone being
3707 	 * promoted. If the origin dataset is a clone, destroy its livelist
3708 	 * as well.
3709 	 */
3710 	dsl_dir_remove_livelist(dd, tx, B_TRUE);
3711 	dsl_dir_remove_livelist(odd, tx, B_TRUE);
3712 
3713 	/* log history record */
3714 	spa_history_log_internal_ds(hds, "promote", tx, " ");
3715 
3716 	dsl_dir_rele(odd, FTAG);
3717 	promote_rele(ddpa, FTAG);
3718 
3719 	/*
3720 	 * Transfer common error blocks from old head to new head.
3721 	 */
3722 	if (spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_HEAD_ERRLOG)) {
3723 		uint64_t old_head = origin_head->ds_object;
3724 		uint64_t new_head = hds->ds_object;
3725 		spa_swap_errlog(dp->dp_spa, new_head, old_head, tx);
3726 	}
3727 }
3728 
3729 /*
3730  * Make a list of dsl_dataset_t's for the snapshots between first_obj
3731  * (exclusive) and last_obj (inclusive).  The list will be in reverse
3732  * order (last_obj will be the list_head()).  If first_obj == 0, do all
3733  * snapshots back to this dataset's origin.
3734  */
3735 static int
3736 snaplist_make(dsl_pool_t *dp,
3737     uint64_t first_obj, uint64_t last_obj, list_t *l, void *tag)
3738 {
3739 	uint64_t obj = last_obj;
3740 
3741 	list_create(l, sizeof (struct promotenode),
3742 	    offsetof(struct promotenode, link));
3743 
3744 	while (obj != first_obj) {
3745 		dsl_dataset_t *ds;
3746 		struct promotenode *snap;
3747 		int err;
3748 
3749 		err = dsl_dataset_hold_obj(dp, obj, tag, &ds);
3750 		ASSERT(err != ENOENT);
3751 		if (err != 0)
3752 			return (err);
3753 
3754 		if (first_obj == 0)
3755 			first_obj = dsl_dir_phys(ds->ds_dir)->dd_origin_obj;
3756 
3757 		snap = kmem_alloc(sizeof (*snap), KM_SLEEP);
3758 		snap->ds = ds;
3759 		list_insert_tail(l, snap);
3760 		obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
3761 	}
3762 
3763 	return (0);
3764 }
3765 
3766 static int
3767 snaplist_space(list_t *l, uint64_t mintxg, uint64_t *spacep)
3768 {
3769 	struct promotenode *snap;
3770 
3771 	*spacep = 0;
3772 	for (snap = list_head(l); snap; snap = list_next(l, snap)) {
3773 		uint64_t used, comp, uncomp;
3774 		dsl_deadlist_space_range(&snap->ds->ds_deadlist,
3775 		    mintxg, UINT64_MAX, &used, &comp, &uncomp);
3776 		*spacep += used;
3777 	}
3778 	return (0);
3779 }
3780 
3781 static void
3782 snaplist_destroy(list_t *l, void *tag)
3783 {
3784 	struct promotenode *snap;
3785 
3786 	if (l == NULL || !list_link_active(&l->list_head))
3787 		return;
3788 
3789 	while ((snap = list_tail(l)) != NULL) {
3790 		list_remove(l, snap);
3791 		dsl_dataset_rele(snap->ds, tag);
3792 		kmem_free(snap, sizeof (*snap));
3793 	}
3794 	list_destroy(l);
3795 }
3796 
3797 static int
3798 promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp, void *tag)
3799 {
3800 	int error;
3801 	dsl_dir_t *dd;
3802 	struct promotenode *snap;
3803 
3804 	error = dsl_dataset_hold(dp, ddpa->ddpa_clonename, tag,
3805 	    &ddpa->ddpa_clone);
3806 	if (error != 0)
3807 		return (error);
3808 	dd = ddpa->ddpa_clone->ds_dir;
3809 
3810 	if (ddpa->ddpa_clone->ds_is_snapshot ||
3811 	    !dsl_dir_is_clone(dd)) {
3812 		dsl_dataset_rele(ddpa->ddpa_clone, tag);
3813 		return (SET_ERROR(EINVAL));
3814 	}
3815 
3816 	error = snaplist_make(dp, 0, dsl_dir_phys(dd)->dd_origin_obj,
3817 	    &ddpa->shared_snaps, tag);
3818 	if (error != 0)
3819 		goto out;
3820 
3821 	error = snaplist_make(dp, 0, ddpa->ddpa_clone->ds_object,
3822 	    &ddpa->clone_snaps, tag);
3823 	if (error != 0)
3824 		goto out;
3825 
3826 	snap = list_head(&ddpa->shared_snaps);
3827 	ASSERT3U(snap->ds->ds_object, ==, dsl_dir_phys(dd)->dd_origin_obj);
3828 	error = snaplist_make(dp, dsl_dir_phys(dd)->dd_origin_obj,
3829 	    dsl_dir_phys(snap->ds->ds_dir)->dd_head_dataset_obj,
3830 	    &ddpa->origin_snaps, tag);
3831 	if (error != 0)
3832 		goto out;
3833 
3834 	if (dsl_dir_phys(snap->ds->ds_dir)->dd_origin_obj != 0) {
3835 		error = dsl_dataset_hold_obj(dp,
3836 		    dsl_dir_phys(snap->ds->ds_dir)->dd_origin_obj,
3837 		    tag, &ddpa->origin_origin);
3838 		if (error != 0)
3839 			goto out;
3840 	}
3841 out:
3842 	if (error != 0)
3843 		promote_rele(ddpa, tag);
3844 	return (error);
3845 }
3846 
3847 static void
3848 promote_rele(dsl_dataset_promote_arg_t *ddpa, void *tag)
3849 {
3850 	snaplist_destroy(&ddpa->shared_snaps, tag);
3851 	snaplist_destroy(&ddpa->clone_snaps, tag);
3852 	snaplist_destroy(&ddpa->origin_snaps, tag);
3853 	if (ddpa->origin_origin != NULL)
3854 		dsl_dataset_rele(ddpa->origin_origin, tag);
3855 	dsl_dataset_rele(ddpa->ddpa_clone, tag);
3856 }
3857 
3858 /*
3859  * Promote a clone.
3860  *
3861  * If it fails due to a conflicting snapshot name, "conflsnap" will be filled
3862  * in with the name.  (It must be at least ZFS_MAX_DATASET_NAME_LEN bytes long.)
3863  */
3864 int
3865 dsl_dataset_promote(const char *name, char *conflsnap)
3866 {
3867 	dsl_dataset_promote_arg_t ddpa = { 0 };
3868 	uint64_t numsnaps;
3869 	int error;
3870 	nvpair_t *snap_pair;
3871 	objset_t *os;
3872 
3873 	/*
3874 	 * We will modify space proportional to the number of
3875 	 * snapshots.  Compute numsnaps.
3876 	 */
3877 	error = dmu_objset_hold(name, FTAG, &os);
3878 	if (error != 0)
3879 		return (error);
3880 	error = zap_count(dmu_objset_pool(os)->dp_meta_objset,
3881 	    dsl_dataset_phys(dmu_objset_ds(os))->ds_snapnames_zapobj,
3882 	    &numsnaps);
3883 	dmu_objset_rele(os, FTAG);
3884 	if (error != 0)
3885 		return (error);
3886 
3887 	ddpa.ddpa_clonename = name;
3888 	ddpa.err_ds = fnvlist_alloc();
3889 	ddpa.cr = CRED();
3890 	ddpa.proc = curproc;
3891 
3892 	error = dsl_sync_task(name, dsl_dataset_promote_check,
3893 	    dsl_dataset_promote_sync, &ddpa,
3894 	    2 + numsnaps, ZFS_SPACE_CHECK_RESERVED);
3895 
3896 	/*
3897 	 * Return the first conflicting snapshot found.
3898 	 */
3899 	snap_pair = nvlist_next_nvpair(ddpa.err_ds, NULL);
3900 	if (snap_pair != NULL && conflsnap != NULL)
3901 		(void) strlcpy(conflsnap, nvpair_name(snap_pair),
3902 		    ZFS_MAX_DATASET_NAME_LEN);
3903 
3904 	fnvlist_free(ddpa.err_ds);
3905 	return (error);
3906 }
3907 
3908 int
3909 dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
3910     dsl_dataset_t *origin_head, boolean_t force, void *owner, dmu_tx_t *tx)
3911 {
3912 	/*
3913 	 * "slack" factor for received datasets with refquota set on them.
3914 	 * See the bottom of this function for details on its use.
3915 	 */
3916 	uint64_t refquota_slack = (uint64_t)DMU_MAX_ACCESS *
3917 	    spa_asize_inflation;
3918 	int64_t unused_refres_delta;
3919 
3920 	/* they should both be heads */
3921 	if (clone->ds_is_snapshot ||
3922 	    origin_head->ds_is_snapshot)
3923 		return (SET_ERROR(EINVAL));
3924 
3925 	/* if we are not forcing, the branch point should be just before them */
3926 	if (!force && clone->ds_prev != origin_head->ds_prev)
3927 		return (SET_ERROR(EINVAL));
3928 
3929 	/* clone should be the clone (unless they are unrelated) */
3930 	if (clone->ds_prev != NULL &&
3931 	    clone->ds_prev != clone->ds_dir->dd_pool->dp_origin_snap &&
3932 	    origin_head->ds_dir != clone->ds_prev->ds_dir)
3933 		return (SET_ERROR(EINVAL));
3934 
3935 	/* the clone should be a child of the origin */
3936 	if (clone->ds_dir->dd_parent != origin_head->ds_dir)
3937 		return (SET_ERROR(EINVAL));
3938 
3939 	/* origin_head shouldn't be modified unless 'force' */
3940 	if (!force &&
3941 	    dsl_dataset_modified_since_snap(origin_head, origin_head->ds_prev))
3942 		return (SET_ERROR(ETXTBSY));
3943 
3944 	/* origin_head should have no long holds (e.g. is not mounted) */
3945 	if (dsl_dataset_handoff_check(origin_head, owner, tx))
3946 		return (SET_ERROR(EBUSY));
3947 
3948 	/* check amount of any unconsumed refreservation */
3949 	unused_refres_delta =
3950 	    (int64_t)MIN(origin_head->ds_reserved,
3951 	    dsl_dataset_phys(origin_head)->ds_unique_bytes) -
3952 	    (int64_t)MIN(origin_head->ds_reserved,
3953 	    dsl_dataset_phys(clone)->ds_unique_bytes);
3954 
3955 	if (unused_refres_delta > 0 &&
3956 	    unused_refres_delta >
3957 	    dsl_dir_space_available(origin_head->ds_dir, NULL, 0, TRUE))
3958 		return (SET_ERROR(ENOSPC));
3959 
3960 	/*
3961 	 * The clone can't be too much over the head's refquota.
3962 	 *
3963 	 * To ensure that the entire refquota can be used, we allow one
3964 	 * transaction to exceed the refquota.  Therefore, this check
3965 	 * needs to also allow for the space referenced to be more than the
3966 	 * refquota.  The maximum amount of space that one transaction can use
3967 	 * on disk is DMU_MAX_ACCESS * spa_asize_inflation.  Allowing this
3968 	 * overage ensures that we are able to receive a filesystem that
3969 	 * exceeds the refquota on the source system.
3970 	 *
3971 	 * So that overage is the refquota_slack we use below.
3972 	 */
3973 	if (origin_head->ds_quota != 0 &&
3974 	    dsl_dataset_phys(clone)->ds_referenced_bytes >
3975 	    origin_head->ds_quota + refquota_slack)
3976 		return (SET_ERROR(EDQUOT));
3977 
3978 	return (0);
3979 }
3980 
3981 static void
3982 dsl_dataset_swap_remap_deadlists(dsl_dataset_t *clone,
3983     dsl_dataset_t *origin, dmu_tx_t *tx)
3984 {
3985 	uint64_t clone_remap_dl_obj, origin_remap_dl_obj;
3986 	dsl_pool_t *dp = dmu_tx_pool(tx);
3987 
3988 	ASSERT(dsl_pool_sync_context(dp));
3989 
3990 	clone_remap_dl_obj = dsl_dataset_get_remap_deadlist_object(clone);
3991 	origin_remap_dl_obj = dsl_dataset_get_remap_deadlist_object(origin);
3992 
3993 	if (clone_remap_dl_obj != 0) {
3994 		dsl_deadlist_close(&clone->ds_remap_deadlist);
3995 		dsl_dataset_unset_remap_deadlist_object(clone, tx);
3996 	}
3997 	if (origin_remap_dl_obj != 0) {
3998 		dsl_deadlist_close(&origin->ds_remap_deadlist);
3999 		dsl_dataset_unset_remap_deadlist_object(origin, tx);
4000 	}
4001 
4002 	if (clone_remap_dl_obj != 0) {
4003 		dsl_dataset_set_remap_deadlist_object(origin,
4004 		    clone_remap_dl_obj, tx);
4005 		dsl_deadlist_open(&origin->ds_remap_deadlist,
4006 		    dp->dp_meta_objset, clone_remap_dl_obj);
4007 	}
4008 	if (origin_remap_dl_obj != 0) {
4009 		dsl_dataset_set_remap_deadlist_object(clone,
4010 		    origin_remap_dl_obj, tx);
4011 		dsl_deadlist_open(&clone->ds_remap_deadlist,
4012 		    dp->dp_meta_objset, origin_remap_dl_obj);
4013 	}
4014 }
4015 
4016 void
4017 dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
4018     dsl_dataset_t *origin_head, dmu_tx_t *tx)
4019 {
4020 	dsl_pool_t *dp = dmu_tx_pool(tx);
4021 	int64_t unused_refres_delta;
4022 
4023 	ASSERT(clone->ds_reserved == 0);
4024 	/*
4025 	 * NOTE: On DEBUG kernels there could be a race between this and
4026 	 * the check function if spa_asize_inflation is adjusted...
4027 	 */
4028 	ASSERT(origin_head->ds_quota == 0 ||
4029 	    dsl_dataset_phys(clone)->ds_unique_bytes <= origin_head->ds_quota +
4030 	    DMU_MAX_ACCESS * spa_asize_inflation);
4031 	ASSERT3P(clone->ds_prev, ==, origin_head->ds_prev);
4032 
4033 	dsl_dir_cancel_waiters(origin_head->ds_dir);
4034 
4035 	/*
4036 	 * Swap per-dataset feature flags.
4037 	 */
4038 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
4039 		if (!(spa_feature_table[f].fi_flags &
4040 		    ZFEATURE_FLAG_PER_DATASET)) {
4041 			ASSERT(!dsl_dataset_feature_is_active(clone, f));
4042 			ASSERT(!dsl_dataset_feature_is_active(origin_head, f));
4043 			continue;
4044 		}
4045 
4046 		boolean_t clone_inuse = dsl_dataset_feature_is_active(clone, f);
4047 		void *clone_feature = clone->ds_feature[f];
4048 		boolean_t origin_head_inuse =
4049 		    dsl_dataset_feature_is_active(origin_head, f);
4050 		void *origin_head_feature = origin_head->ds_feature[f];
4051 
4052 		if (clone_inuse)
4053 			dsl_dataset_deactivate_feature_impl(clone, f, tx);
4054 		if (origin_head_inuse)
4055 			dsl_dataset_deactivate_feature_impl(origin_head, f, tx);
4056 
4057 		if (clone_inuse) {
4058 			dsl_dataset_activate_feature(origin_head->ds_object, f,
4059 			    clone_feature, tx);
4060 			origin_head->ds_feature[f] = clone_feature;
4061 		}
4062 		if (origin_head_inuse) {
4063 			dsl_dataset_activate_feature(clone->ds_object, f,
4064 			    origin_head_feature, tx);
4065 			clone->ds_feature[f] = origin_head_feature;
4066 		}
4067 	}
4068 
4069 	dmu_buf_will_dirty(clone->ds_dbuf, tx);
4070 	dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
4071 
4072 	if (clone->ds_objset != NULL) {
4073 		dmu_objset_evict(clone->ds_objset);
4074 		clone->ds_objset = NULL;
4075 	}
4076 
4077 	if (origin_head->ds_objset != NULL) {
4078 		dmu_objset_evict(origin_head->ds_objset);
4079 		origin_head->ds_objset = NULL;
4080 	}
4081 
4082 	unused_refres_delta =
4083 	    (int64_t)MIN(origin_head->ds_reserved,
4084 	    dsl_dataset_phys(origin_head)->ds_unique_bytes) -
4085 	    (int64_t)MIN(origin_head->ds_reserved,
4086 	    dsl_dataset_phys(clone)->ds_unique_bytes);
4087 
4088 	/*
4089 	 * Reset origin's unique bytes.
4090 	 */
4091 	{
4092 		dsl_dataset_t *origin = clone->ds_prev;
4093 		uint64_t comp, uncomp;
4094 
4095 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
4096 		dsl_deadlist_space_range(&clone->ds_deadlist,
4097 		    dsl_dataset_phys(origin)->ds_prev_snap_txg, UINT64_MAX,
4098 		    &dsl_dataset_phys(origin)->ds_unique_bytes, &comp, &uncomp);
4099 	}
4100 
4101 	/* swap blkptrs */
4102 	{
4103 		rrw_enter(&clone->ds_bp_rwlock, RW_WRITER, FTAG);
4104 		rrw_enter(&origin_head->ds_bp_rwlock, RW_WRITER, FTAG);
4105 		blkptr_t tmp;
4106 		tmp = dsl_dataset_phys(origin_head)->ds_bp;
4107 		dsl_dataset_phys(origin_head)->ds_bp =
4108 		    dsl_dataset_phys(clone)->ds_bp;
4109 		dsl_dataset_phys(clone)->ds_bp = tmp;
4110 		rrw_exit(&origin_head->ds_bp_rwlock, FTAG);
4111 		rrw_exit(&clone->ds_bp_rwlock, FTAG);
4112 	}
4113 
4114 	/* set dd_*_bytes */
4115 	{
4116 		int64_t dused, dcomp, duncomp;
4117 		uint64_t cdl_used, cdl_comp, cdl_uncomp;
4118 		uint64_t odl_used, odl_comp, odl_uncomp;
4119 
4120 		ASSERT3U(dsl_dir_phys(clone->ds_dir)->
4121 		    dd_used_breakdown[DD_USED_SNAP], ==, 0);
4122 
4123 		dsl_deadlist_space(&clone->ds_deadlist,
4124 		    &cdl_used, &cdl_comp, &cdl_uncomp);
4125 		dsl_deadlist_space(&origin_head->ds_deadlist,
4126 		    &odl_used, &odl_comp, &odl_uncomp);
4127 
4128 		dused = dsl_dataset_phys(clone)->ds_referenced_bytes +
4129 		    cdl_used -
4130 		    (dsl_dataset_phys(origin_head)->ds_referenced_bytes +
4131 		    odl_used);
4132 		dcomp = dsl_dataset_phys(clone)->ds_compressed_bytes +
4133 		    cdl_comp -
4134 		    (dsl_dataset_phys(origin_head)->ds_compressed_bytes +
4135 		    odl_comp);
4136 		duncomp = dsl_dataset_phys(clone)->ds_uncompressed_bytes +
4137 		    cdl_uncomp -
4138 		    (dsl_dataset_phys(origin_head)->ds_uncompressed_bytes +
4139 		    odl_uncomp);
4140 
4141 		dsl_dir_diduse_space(origin_head->ds_dir, DD_USED_HEAD,
4142 		    dused, dcomp, duncomp, tx);
4143 		dsl_dir_diduse_space(clone->ds_dir, DD_USED_HEAD,
4144 		    -dused, -dcomp, -duncomp, tx);
4145 
4146 		/*
4147 		 * The difference in the space used by snapshots is the
4148 		 * difference in snapshot space due to the head's
4149 		 * deadlist (since that's the only thing that's
4150 		 * changing that affects the snapused).
4151 		 */
4152 		dsl_deadlist_space_range(&clone->ds_deadlist,
4153 		    origin_head->ds_dir->dd_origin_txg, UINT64_MAX,
4154 		    &cdl_used, &cdl_comp, &cdl_uncomp);
4155 		dsl_deadlist_space_range(&origin_head->ds_deadlist,
4156 		    origin_head->ds_dir->dd_origin_txg, UINT64_MAX,
4157 		    &odl_used, &odl_comp, &odl_uncomp);
4158 		dsl_dir_transfer_space(origin_head->ds_dir, cdl_used - odl_used,
4159 		    DD_USED_HEAD, DD_USED_SNAP, tx);
4160 	}
4161 
4162 	/* swap ds_*_bytes */
4163 	SWITCH64(dsl_dataset_phys(origin_head)->ds_referenced_bytes,
4164 	    dsl_dataset_phys(clone)->ds_referenced_bytes);
4165 	SWITCH64(dsl_dataset_phys(origin_head)->ds_compressed_bytes,
4166 	    dsl_dataset_phys(clone)->ds_compressed_bytes);
4167 	SWITCH64(dsl_dataset_phys(origin_head)->ds_uncompressed_bytes,
4168 	    dsl_dataset_phys(clone)->ds_uncompressed_bytes);
4169 	SWITCH64(dsl_dataset_phys(origin_head)->ds_unique_bytes,
4170 	    dsl_dataset_phys(clone)->ds_unique_bytes);
4171 
4172 	/* apply any parent delta for change in unconsumed refreservation */
4173 	dsl_dir_diduse_space(origin_head->ds_dir, DD_USED_REFRSRV,
4174 	    unused_refres_delta, 0, 0, tx);
4175 
4176 	/*
4177 	 * Swap deadlists.
4178 	 */
4179 	dsl_deadlist_close(&clone->ds_deadlist);
4180 	dsl_deadlist_close(&origin_head->ds_deadlist);
4181 	SWITCH64(dsl_dataset_phys(origin_head)->ds_deadlist_obj,
4182 	    dsl_dataset_phys(clone)->ds_deadlist_obj);
4183 	dsl_deadlist_open(&clone->ds_deadlist, dp->dp_meta_objset,
4184 	    dsl_dataset_phys(clone)->ds_deadlist_obj);
4185 	dsl_deadlist_open(&origin_head->ds_deadlist, dp->dp_meta_objset,
4186 	    dsl_dataset_phys(origin_head)->ds_deadlist_obj);
4187 	dsl_dataset_swap_remap_deadlists(clone, origin_head, tx);
4188 
4189 	/*
4190 	 * If there is a bookmark at the origin, its "next dataset" is
4191 	 * changing, so we need to reset its FBN.
4192 	 */
4193 	dsl_bookmark_next_changed(origin_head, origin_head->ds_prev, tx);
4194 
4195 	dsl_scan_ds_clone_swapped(origin_head, clone, tx);
4196 
4197 	/*
4198 	 * Destroy any livelists associated with the clone or the origin,
4199 	 * since after the swap the corresponding livelists are no longer
4200 	 * valid.
4201 	 */
4202 	dsl_dir_remove_livelist(clone->ds_dir, tx, B_TRUE);
4203 	dsl_dir_remove_livelist(origin_head->ds_dir, tx, B_TRUE);
4204 
4205 	spa_history_log_internal_ds(clone, "clone swap", tx,
4206 	    "parent=%s", origin_head->ds_dir->dd_myname);
4207 }
4208 
4209 /*
4210  * Given a pool name and a dataset object number in that pool,
4211  * return the name of that dataset.
4212  */
4213 int
4214 dsl_dsobj_to_dsname(char *pname, uint64_t obj, char *buf)
4215 {
4216 	dsl_pool_t *dp;
4217 	dsl_dataset_t *ds;
4218 	int error;
4219 
4220 	error = dsl_pool_hold(pname, FTAG, &dp);
4221 	if (error != 0)
4222 		return (error);
4223 
4224 	error = dsl_dataset_hold_obj(dp, obj, FTAG, &ds);
4225 	if (error == 0) {
4226 		dsl_dataset_name(ds, buf);
4227 		dsl_dataset_rele(ds, FTAG);
4228 	}
4229 	dsl_pool_rele(dp, FTAG);
4230 
4231 	return (error);
4232 }
4233 
4234 int
4235 dsl_dataset_check_quota(dsl_dataset_t *ds, boolean_t check_quota,
4236     uint64_t asize, uint64_t inflight, uint64_t *used, uint64_t *ref_rsrv)
4237 {
4238 	int error = 0;
4239 
4240 	ASSERT3S(asize, >, 0);
4241 
4242 	/*
4243 	 * *ref_rsrv is the portion of asize that will come from any
4244 	 * unconsumed refreservation space.
4245 	 */
4246 	*ref_rsrv = 0;
4247 
4248 	mutex_enter(&ds->ds_lock);
4249 	/*
4250 	 * Make a space adjustment for reserved bytes.
4251 	 */
4252 	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes) {
4253 		ASSERT3U(*used, >=,
4254 		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes);
4255 		*used -=
4256 		    (ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes);
4257 		*ref_rsrv =
4258 		    asize - MIN(asize, parent_delta(ds, asize + inflight));
4259 	}
4260 
4261 	if (!check_quota || ds->ds_quota == 0) {
4262 		mutex_exit(&ds->ds_lock);
4263 		return (0);
4264 	}
4265 	/*
4266 	 * If they are requesting more space, and our current estimate
4267 	 * is over quota, they get to try again unless the actual
4268 	 * on-disk is over quota and there are no pending changes (which
4269 	 * may free up space for us).
4270 	 */
4271 	if (dsl_dataset_phys(ds)->ds_referenced_bytes + inflight >=
4272 	    ds->ds_quota) {
4273 		if (inflight > 0 ||
4274 		    dsl_dataset_phys(ds)->ds_referenced_bytes < ds->ds_quota)
4275 			error = SET_ERROR(ERESTART);
4276 		else
4277 			error = SET_ERROR(EDQUOT);
4278 	}
4279 	mutex_exit(&ds->ds_lock);
4280 
4281 	return (error);
4282 }
4283 
4284 typedef struct dsl_dataset_set_qr_arg {
4285 	const char *ddsqra_name;
4286 	zprop_source_t ddsqra_source;
4287 	uint64_t ddsqra_value;
4288 } dsl_dataset_set_qr_arg_t;
4289 
4290 
4291 static int
4292 dsl_dataset_set_refquota_check(void *arg, dmu_tx_t *tx)
4293 {
4294 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
4295 	dsl_pool_t *dp = dmu_tx_pool(tx);
4296 	dsl_dataset_t *ds;
4297 	int error;
4298 	uint64_t newval;
4299 
4300 	if (spa_version(dp->dp_spa) < SPA_VERSION_REFQUOTA)
4301 		return (SET_ERROR(ENOTSUP));
4302 
4303 	error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
4304 	if (error != 0)
4305 		return (error);
4306 
4307 	if (ds->ds_is_snapshot) {
4308 		dsl_dataset_rele(ds, FTAG);
4309 		return (SET_ERROR(EINVAL));
4310 	}
4311 
4312 	error = dsl_prop_predict(ds->ds_dir,
4313 	    zfs_prop_to_name(ZFS_PROP_REFQUOTA),
4314 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
4315 	if (error != 0) {
4316 		dsl_dataset_rele(ds, FTAG);
4317 		return (error);
4318 	}
4319 
4320 	if (newval == 0) {
4321 		dsl_dataset_rele(ds, FTAG);
4322 		return (0);
4323 	}
4324 
4325 	if (newval < dsl_dataset_phys(ds)->ds_referenced_bytes ||
4326 	    newval < ds->ds_reserved) {
4327 		dsl_dataset_rele(ds, FTAG);
4328 		return (SET_ERROR(ENOSPC));
4329 	}
4330 
4331 	dsl_dataset_rele(ds, FTAG);
4332 	return (0);
4333 }
4334 
4335 static void
4336 dsl_dataset_set_refquota_sync(void *arg, dmu_tx_t *tx)
4337 {
4338 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
4339 	dsl_pool_t *dp = dmu_tx_pool(tx);
4340 	dsl_dataset_t *ds = NULL;
4341 	uint64_t newval;
4342 
4343 	VERIFY0(dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds));
4344 
4345 	dsl_prop_set_sync_impl(ds,
4346 	    zfs_prop_to_name(ZFS_PROP_REFQUOTA),
4347 	    ddsqra->ddsqra_source, sizeof (ddsqra->ddsqra_value), 1,
4348 	    &ddsqra->ddsqra_value, tx);
4349 
4350 	VERIFY0(dsl_prop_get_int_ds(ds,
4351 	    zfs_prop_to_name(ZFS_PROP_REFQUOTA), &newval));
4352 
4353 	if (ds->ds_quota != newval) {
4354 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
4355 		ds->ds_quota = newval;
4356 	}
4357 	dsl_dataset_rele(ds, FTAG);
4358 }
4359 
4360 int
4361 dsl_dataset_set_refquota(const char *dsname, zprop_source_t source,
4362     uint64_t refquota)
4363 {
4364 	dsl_dataset_set_qr_arg_t ddsqra;
4365 
4366 	ddsqra.ddsqra_name = dsname;
4367 	ddsqra.ddsqra_source = source;
4368 	ddsqra.ddsqra_value = refquota;
4369 
4370 	return (dsl_sync_task(dsname, dsl_dataset_set_refquota_check,
4371 	    dsl_dataset_set_refquota_sync, &ddsqra, 0,
4372 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
4373 }
4374 
4375 static int
4376 dsl_dataset_set_refreservation_check(void *arg, dmu_tx_t *tx)
4377 {
4378 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
4379 	dsl_pool_t *dp = dmu_tx_pool(tx);
4380 	dsl_dataset_t *ds;
4381 	int error;
4382 	uint64_t newval, unique;
4383 
4384 	if (spa_version(dp->dp_spa) < SPA_VERSION_REFRESERVATION)
4385 		return (SET_ERROR(ENOTSUP));
4386 
4387 	error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
4388 	if (error != 0)
4389 		return (error);
4390 
4391 	if (ds->ds_is_snapshot) {
4392 		dsl_dataset_rele(ds, FTAG);
4393 		return (SET_ERROR(EINVAL));
4394 	}
4395 
4396 	error = dsl_prop_predict(ds->ds_dir,
4397 	    zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
4398 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
4399 	if (error != 0) {
4400 		dsl_dataset_rele(ds, FTAG);
4401 		return (error);
4402 	}
4403 
4404 	/*
4405 	 * If we are doing the preliminary check in open context, the
4406 	 * space estimates may be inaccurate.
4407 	 */
4408 	if (!dmu_tx_is_syncing(tx)) {
4409 		dsl_dataset_rele(ds, FTAG);
4410 		return (0);
4411 	}
4412 
4413 	mutex_enter(&ds->ds_lock);
4414 	if (!DS_UNIQUE_IS_ACCURATE(ds))
4415 		dsl_dataset_recalc_head_uniq(ds);
4416 	unique = dsl_dataset_phys(ds)->ds_unique_bytes;
4417 	mutex_exit(&ds->ds_lock);
4418 
4419 	if (MAX(unique, newval) > MAX(unique, ds->ds_reserved)) {
4420 		uint64_t delta = MAX(unique, newval) -
4421 		    MAX(unique, ds->ds_reserved);
4422 
4423 		if (delta >
4424 		    dsl_dir_space_available(ds->ds_dir, NULL, 0, B_TRUE) ||
4425 		    (ds->ds_quota > 0 && newval > ds->ds_quota)) {
4426 			dsl_dataset_rele(ds, FTAG);
4427 			return (SET_ERROR(ENOSPC));
4428 		}
4429 	}
4430 
4431 	dsl_dataset_rele(ds, FTAG);
4432 	return (0);
4433 }
4434 
4435 void
4436 dsl_dataset_set_refreservation_sync_impl(dsl_dataset_t *ds,
4437     zprop_source_t source, uint64_t value, dmu_tx_t *tx)
4438 {
4439 	uint64_t newval;
4440 	uint64_t unique;
4441 	int64_t delta;
4442 
4443 	dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
4444 	    source, sizeof (value), 1, &value, tx);
4445 
4446 	VERIFY0(dsl_prop_get_int_ds(ds,
4447 	    zfs_prop_to_name(ZFS_PROP_REFRESERVATION), &newval));
4448 
4449 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
4450 	mutex_enter(&ds->ds_dir->dd_lock);
4451 	mutex_enter(&ds->ds_lock);
4452 	ASSERT(DS_UNIQUE_IS_ACCURATE(ds));
4453 	unique = dsl_dataset_phys(ds)->ds_unique_bytes;
4454 	delta = MAX(0, (int64_t)(newval - unique)) -
4455 	    MAX(0, (int64_t)(ds->ds_reserved - unique));
4456 	ds->ds_reserved = newval;
4457 	mutex_exit(&ds->ds_lock);
4458 
4459 	dsl_dir_diduse_space(ds->ds_dir, DD_USED_REFRSRV, delta, 0, 0, tx);
4460 	mutex_exit(&ds->ds_dir->dd_lock);
4461 }
4462 
4463 static void
4464 dsl_dataset_set_refreservation_sync(void *arg, dmu_tx_t *tx)
4465 {
4466 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
4467 	dsl_pool_t *dp = dmu_tx_pool(tx);
4468 	dsl_dataset_t *ds = NULL;
4469 
4470 	VERIFY0(dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds));
4471 	dsl_dataset_set_refreservation_sync_impl(ds,
4472 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, tx);
4473 	dsl_dataset_rele(ds, FTAG);
4474 }
4475 
4476 int
4477 dsl_dataset_set_refreservation(const char *dsname, zprop_source_t source,
4478     uint64_t refreservation)
4479 {
4480 	dsl_dataset_set_qr_arg_t ddsqra;
4481 
4482 	ddsqra.ddsqra_name = dsname;
4483 	ddsqra.ddsqra_source = source;
4484 	ddsqra.ddsqra_value = refreservation;
4485 
4486 	return (dsl_sync_task(dsname, dsl_dataset_set_refreservation_check,
4487 	    dsl_dataset_set_refreservation_sync, &ddsqra, 0,
4488 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
4489 }
4490 
4491 typedef struct dsl_dataset_set_compression_arg {
4492 	const char *ddsca_name;
4493 	zprop_source_t ddsca_source;
4494 	uint64_t ddsca_value;
4495 } dsl_dataset_set_compression_arg_t;
4496 
4497 static int
4498 dsl_dataset_set_compression_check(void *arg, dmu_tx_t *tx)
4499 {
4500 	dsl_dataset_set_compression_arg_t *ddsca = arg;
4501 	dsl_pool_t *dp = dmu_tx_pool(tx);
4502 
4503 	uint64_t compval = ZIO_COMPRESS_ALGO(ddsca->ddsca_value);
4504 	spa_feature_t f = zio_compress_to_feature(compval);
4505 
4506 	if (f == SPA_FEATURE_NONE)
4507 		return (SET_ERROR(EINVAL));
4508 
4509 	if (!spa_feature_is_enabled(dp->dp_spa, f))
4510 		return (SET_ERROR(ENOTSUP));
4511 
4512 	return (0);
4513 }
4514 
4515 static void
4516 dsl_dataset_set_compression_sync(void *arg, dmu_tx_t *tx)
4517 {
4518 	dsl_dataset_set_compression_arg_t *ddsca = arg;
4519 	dsl_pool_t *dp = dmu_tx_pool(tx);
4520 	dsl_dataset_t *ds = NULL;
4521 
4522 	uint64_t compval = ZIO_COMPRESS_ALGO(ddsca->ddsca_value);
4523 	spa_feature_t f = zio_compress_to_feature(compval);
4524 	ASSERT3S(spa_feature_table[f].fi_type, ==, ZFEATURE_TYPE_BOOLEAN);
4525 
4526 	VERIFY0(dsl_dataset_hold(dp, ddsca->ddsca_name, FTAG, &ds));
4527 	if (zfeature_active(f, ds->ds_feature[f]) != B_TRUE) {
4528 		ds->ds_feature_activation[f] = (void *)B_TRUE;
4529 		dsl_dataset_activate_feature(ds->ds_object, f,
4530 		    ds->ds_feature_activation[f], tx);
4531 		ds->ds_feature[f] = ds->ds_feature_activation[f];
4532 	}
4533 	dsl_dataset_rele(ds, FTAG);
4534 }
4535 
4536 int
4537 dsl_dataset_set_compression(const char *dsname, zprop_source_t source,
4538     uint64_t compression)
4539 {
4540 	dsl_dataset_set_compression_arg_t ddsca;
4541 
4542 	/*
4543 	 * The sync task is only required for zstd in order to activate
4544 	 * the feature flag when the property is first set.
4545 	 */
4546 	if (ZIO_COMPRESS_ALGO(compression) != ZIO_COMPRESS_ZSTD)
4547 		return (0);
4548 
4549 	ddsca.ddsca_name = dsname;
4550 	ddsca.ddsca_source = source;
4551 	ddsca.ddsca_value = compression;
4552 
4553 	return (dsl_sync_task(dsname, dsl_dataset_set_compression_check,
4554 	    dsl_dataset_set_compression_sync, &ddsca, 0,
4555 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
4556 }
4557 
4558 /*
4559  * Return (in *usedp) the amount of space referenced by "new" that was not
4560  * referenced at the time the bookmark corresponds to.  "New" may be a
4561  * snapshot or a head.  The bookmark must be before new, in
4562  * new's filesystem (or its origin) -- caller verifies this.
4563  *
4564  * The written space is calculated by considering two components:  First, we
4565  * ignore any freed space, and calculate the written as new's used space
4566  * minus old's used space.  Next, we add in the amount of space that was freed
4567  * between the two time points, thus reducing new's used space relative to
4568  * old's. Specifically, this is the space that was born before
4569  * zbm_creation_txg, and freed before new (ie. on new's deadlist or a
4570  * previous deadlist).
4571  *
4572  * space freed                         [---------------------]
4573  * snapshots                       ---O-------O--------O-------O------
4574  *                                         bookmark           new
4575  *
4576  * Note, the bookmark's zbm_*_bytes_refd must be valid, but if the HAS_FBN
4577  * flag is not set, we will calculate the freed_before_next based on the
4578  * next snapshot's deadlist, rather than using zbm_*_freed_before_next_snap.
4579  */
4580 static int
4581 dsl_dataset_space_written_impl(zfs_bookmark_phys_t *bmp,
4582     dsl_dataset_t *new, uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
4583 {
4584 	int err = 0;
4585 	dsl_pool_t *dp = new->ds_dir->dd_pool;
4586 
4587 	ASSERT(dsl_pool_config_held(dp));
4588 	if (dsl_dataset_is_snapshot(new)) {
4589 		ASSERT3U(bmp->zbm_creation_txg, <,
4590 		    dsl_dataset_phys(new)->ds_creation_txg);
4591 	}
4592 
4593 	*usedp = 0;
4594 	*usedp += dsl_dataset_phys(new)->ds_referenced_bytes;
4595 	*usedp -= bmp->zbm_referenced_bytes_refd;
4596 
4597 	*compp = 0;
4598 	*compp += dsl_dataset_phys(new)->ds_compressed_bytes;
4599 	*compp -= bmp->zbm_compressed_bytes_refd;
4600 
4601 	*uncompp = 0;
4602 	*uncompp += dsl_dataset_phys(new)->ds_uncompressed_bytes;
4603 	*uncompp -= bmp->zbm_uncompressed_bytes_refd;
4604 
4605 	dsl_dataset_t *snap = new;
4606 
4607 	while (dsl_dataset_phys(snap)->ds_prev_snap_txg >
4608 	    bmp->zbm_creation_txg) {
4609 		uint64_t used, comp, uncomp;
4610 
4611 		dsl_deadlist_space_range(&snap->ds_deadlist,
4612 		    0, bmp->zbm_creation_txg,
4613 		    &used, &comp, &uncomp);
4614 		*usedp += used;
4615 		*compp += comp;
4616 		*uncompp += uncomp;
4617 
4618 		uint64_t snapobj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
4619 		if (snap != new)
4620 			dsl_dataset_rele(snap, FTAG);
4621 		err = dsl_dataset_hold_obj(dp, snapobj, FTAG, &snap);
4622 		if (err != 0)
4623 			break;
4624 	}
4625 
4626 	/*
4627 	 * We might not have the FBN if we are calculating written from
4628 	 * a snapshot (because we didn't know the correct "next" snapshot
4629 	 * until now).
4630 	 */
4631 	if (bmp->zbm_flags & ZBM_FLAG_HAS_FBN) {
4632 		*usedp += bmp->zbm_referenced_freed_before_next_snap;
4633 		*compp += bmp->zbm_compressed_freed_before_next_snap;
4634 		*uncompp += bmp->zbm_uncompressed_freed_before_next_snap;
4635 	} else {
4636 		ASSERT3U(dsl_dataset_phys(snap)->ds_prev_snap_txg, ==,
4637 		    bmp->zbm_creation_txg);
4638 		uint64_t used, comp, uncomp;
4639 		dsl_deadlist_space(&snap->ds_deadlist, &used, &comp, &uncomp);
4640 		*usedp += used;
4641 		*compp += comp;
4642 		*uncompp += uncomp;
4643 	}
4644 	if (snap != new)
4645 		dsl_dataset_rele(snap, FTAG);
4646 	return (err);
4647 }
4648 
4649 /*
4650  * Return (in *usedp) the amount of space written in new that was not
4651  * present at the time the bookmark corresponds to.  New may be a
4652  * snapshot or the head.  Old must be a bookmark before new, in
4653  * new's filesystem (or its origin) -- caller verifies this.
4654  */
4655 int
4656 dsl_dataset_space_written_bookmark(zfs_bookmark_phys_t *bmp,
4657     dsl_dataset_t *new, uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
4658 {
4659 	if (!(bmp->zbm_flags & ZBM_FLAG_HAS_FBN))
4660 		return (SET_ERROR(ENOTSUP));
4661 	return (dsl_dataset_space_written_impl(bmp, new,
4662 	    usedp, compp, uncompp));
4663 }
4664 
4665 /*
4666  * Return (in *usedp) the amount of space written in new that is not
4667  * present in oldsnap.  New may be a snapshot or the head.  Old must be
4668  * a snapshot before new, in new's filesystem (or its origin).  If not then
4669  * fail and return EINVAL.
4670  */
4671 int
4672 dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
4673     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
4674 {
4675 	if (!dsl_dataset_is_before(new, oldsnap, 0))
4676 		return (SET_ERROR(EINVAL));
4677 
4678 	zfs_bookmark_phys_t zbm = { 0 };
4679 	dsl_dataset_phys_t *dsp = dsl_dataset_phys(oldsnap);
4680 	zbm.zbm_guid = dsp->ds_guid;
4681 	zbm.zbm_creation_txg = dsp->ds_creation_txg;
4682 	zbm.zbm_creation_time = dsp->ds_creation_time;
4683 	zbm.zbm_referenced_bytes_refd = dsp->ds_referenced_bytes;
4684 	zbm.zbm_compressed_bytes_refd = dsp->ds_compressed_bytes;
4685 	zbm.zbm_uncompressed_bytes_refd = dsp->ds_uncompressed_bytes;
4686 
4687 	/*
4688 	 * If oldsnap is the origin (or origin's origin, ...) of new,
4689 	 * we can't easily calculate the effective FBN.  Therefore,
4690 	 * we do not set ZBM_FLAG_HAS_FBN, so that the _impl will calculate
4691 	 * it relative to the correct "next": the next snapshot towards "new",
4692 	 * rather than the next snapshot in oldsnap's dsl_dir.
4693 	 */
4694 	return (dsl_dataset_space_written_impl(&zbm, new,
4695 	    usedp, compp, uncompp));
4696 }
4697 
4698 /*
4699  * Return (in *usedp) the amount of space that will be reclaimed if firstsnap,
4700  * lastsnap, and all snapshots in between are deleted.
4701  *
4702  * blocks that would be freed            [---------------------------]
4703  * snapshots                       ---O-------O--------O-------O--------O
4704  *                                        firstsnap        lastsnap
4705  *
4706  * This is the set of blocks that were born after the snap before firstsnap,
4707  * (birth > firstsnap->prev_snap_txg) and died before the snap after the
4708  * last snap (ie, is on lastsnap->ds_next->ds_deadlist or an earlier deadlist).
4709  * We calculate this by iterating over the relevant deadlists (from the snap
4710  * after lastsnap, backward to the snap after firstsnap), summing up the
4711  * space on the deadlist that was born after the snap before firstsnap.
4712  */
4713 int
4714 dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
4715     dsl_dataset_t *lastsnap,
4716     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
4717 {
4718 	int err = 0;
4719 	uint64_t snapobj;
4720 	dsl_pool_t *dp = firstsnap->ds_dir->dd_pool;
4721 
4722 	ASSERT(firstsnap->ds_is_snapshot);
4723 	ASSERT(lastsnap->ds_is_snapshot);
4724 
4725 	/*
4726 	 * Check that the snapshots are in the same dsl_dir, and firstsnap
4727 	 * is before lastsnap.
4728 	 */
4729 	if (firstsnap->ds_dir != lastsnap->ds_dir ||
4730 	    dsl_dataset_phys(firstsnap)->ds_creation_txg >
4731 	    dsl_dataset_phys(lastsnap)->ds_creation_txg)
4732 		return (SET_ERROR(EINVAL));
4733 
4734 	*usedp = *compp = *uncompp = 0;
4735 
4736 	snapobj = dsl_dataset_phys(lastsnap)->ds_next_snap_obj;
4737 	while (snapobj != firstsnap->ds_object) {
4738 		dsl_dataset_t *ds;
4739 		uint64_t used, comp, uncomp;
4740 
4741 		err = dsl_dataset_hold_obj(dp, snapobj, FTAG, &ds);
4742 		if (err != 0)
4743 			break;
4744 
4745 		dsl_deadlist_space_range(&ds->ds_deadlist,
4746 		    dsl_dataset_phys(firstsnap)->ds_prev_snap_txg, UINT64_MAX,
4747 		    &used, &comp, &uncomp);
4748 		*usedp += used;
4749 		*compp += comp;
4750 		*uncompp += uncomp;
4751 
4752 		snapobj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
4753 		ASSERT3U(snapobj, !=, 0);
4754 		dsl_dataset_rele(ds, FTAG);
4755 	}
4756 	return (err);
4757 }
4758 
4759 /*
4760  * Return TRUE if 'earlier' is an earlier snapshot in 'later's timeline.
4761  * For example, they could both be snapshots of the same filesystem, and
4762  * 'earlier' is before 'later'.  Or 'earlier' could be the origin of
4763  * 'later's filesystem.  Or 'earlier' could be an older snapshot in the origin's
4764  * filesystem.  Or 'earlier' could be the origin's origin.
4765  *
4766  * If non-zero, earlier_txg is used instead of earlier's ds_creation_txg.
4767  */
4768 boolean_t
4769 dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
4770     uint64_t earlier_txg)
4771 {
4772 	dsl_pool_t *dp = later->ds_dir->dd_pool;
4773 	int error;
4774 	boolean_t ret;
4775 
4776 	ASSERT(dsl_pool_config_held(dp));
4777 	ASSERT(earlier->ds_is_snapshot || earlier_txg != 0);
4778 
4779 	if (earlier_txg == 0)
4780 		earlier_txg = dsl_dataset_phys(earlier)->ds_creation_txg;
4781 
4782 	if (later->ds_is_snapshot &&
4783 	    earlier_txg >= dsl_dataset_phys(later)->ds_creation_txg)
4784 		return (B_FALSE);
4785 
4786 	if (later->ds_dir == earlier->ds_dir)
4787 		return (B_TRUE);
4788 
4789 	/*
4790 	 * We check dd_origin_obj explicitly here rather than using
4791 	 * dsl_dir_is_clone() so that we will return TRUE if "earlier"
4792 	 * is $ORIGIN@$ORIGIN.  dsl_dataset_space_written() depends on
4793 	 * this behavior.
4794 	 */
4795 	if (dsl_dir_phys(later->ds_dir)->dd_origin_obj == 0)
4796 		return (B_FALSE);
4797 
4798 	dsl_dataset_t *origin;
4799 	error = dsl_dataset_hold_obj(dp,
4800 	    dsl_dir_phys(later->ds_dir)->dd_origin_obj, FTAG, &origin);
4801 	if (error != 0)
4802 		return (B_FALSE);
4803 	if (dsl_dataset_phys(origin)->ds_creation_txg == earlier_txg &&
4804 	    origin->ds_dir == earlier->ds_dir) {
4805 		dsl_dataset_rele(origin, FTAG);
4806 		return (B_TRUE);
4807 	}
4808 	ret = dsl_dataset_is_before(origin, earlier, earlier_txg);
4809 	dsl_dataset_rele(origin, FTAG);
4810 	return (ret);
4811 }
4812 
4813 void
4814 dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx)
4815 {
4816 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
4817 	dmu_object_zapify(mos, ds->ds_object, DMU_OT_DSL_DATASET, tx);
4818 }
4819 
4820 boolean_t
4821 dsl_dataset_is_zapified(dsl_dataset_t *ds)
4822 {
4823 	dmu_object_info_t doi;
4824 
4825 	dmu_object_info_from_db(ds->ds_dbuf, &doi);
4826 	return (doi.doi_type == DMU_OTN_ZAP_METADATA);
4827 }
4828 
4829 boolean_t
4830 dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds)
4831 {
4832 	return (dsl_dataset_is_zapified(ds) &&
4833 	    zap_contains(ds->ds_dir->dd_pool->dp_meta_objset,
4834 	    ds->ds_object, DS_FIELD_RESUME_TOGUID) == 0);
4835 }
4836 
4837 uint64_t
4838 dsl_dataset_get_remap_deadlist_object(dsl_dataset_t *ds)
4839 {
4840 	uint64_t remap_deadlist_obj;
4841 	int err;
4842 
4843 	if (!dsl_dataset_is_zapified(ds))
4844 		return (0);
4845 
4846 	err = zap_lookup(ds->ds_dir->dd_pool->dp_meta_objset, ds->ds_object,
4847 	    DS_FIELD_REMAP_DEADLIST, sizeof (remap_deadlist_obj), 1,
4848 	    &remap_deadlist_obj);
4849 
4850 	if (err != 0) {
4851 		VERIFY3S(err, ==, ENOENT);
4852 		return (0);
4853 	}
4854 
4855 	ASSERT(remap_deadlist_obj != 0);
4856 	return (remap_deadlist_obj);
4857 }
4858 
4859 boolean_t
4860 dsl_dataset_remap_deadlist_exists(dsl_dataset_t *ds)
4861 {
4862 	EQUIV(dsl_deadlist_is_open(&ds->ds_remap_deadlist),
4863 	    dsl_dataset_get_remap_deadlist_object(ds) != 0);
4864 	return (dsl_deadlist_is_open(&ds->ds_remap_deadlist));
4865 }
4866 
4867 static void
4868 dsl_dataset_set_remap_deadlist_object(dsl_dataset_t *ds, uint64_t obj,
4869     dmu_tx_t *tx)
4870 {
4871 	ASSERT(obj != 0);
4872 	dsl_dataset_zapify(ds, tx);
4873 	VERIFY0(zap_add(ds->ds_dir->dd_pool->dp_meta_objset, ds->ds_object,
4874 	    DS_FIELD_REMAP_DEADLIST, sizeof (obj), 1, &obj, tx));
4875 }
4876 
4877 static void
4878 dsl_dataset_unset_remap_deadlist_object(dsl_dataset_t *ds, dmu_tx_t *tx)
4879 {
4880 	VERIFY0(zap_remove(ds->ds_dir->dd_pool->dp_meta_objset,
4881 	    ds->ds_object, DS_FIELD_REMAP_DEADLIST, tx));
4882 }
4883 
4884 void
4885 dsl_dataset_destroy_remap_deadlist(dsl_dataset_t *ds, dmu_tx_t *tx)
4886 {
4887 	uint64_t remap_deadlist_object;
4888 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
4889 
4890 	ASSERT(dmu_tx_is_syncing(tx));
4891 	ASSERT(dsl_dataset_remap_deadlist_exists(ds));
4892 
4893 	remap_deadlist_object = ds->ds_remap_deadlist.dl_object;
4894 	dsl_deadlist_close(&ds->ds_remap_deadlist);
4895 	dsl_deadlist_free(spa_meta_objset(spa), remap_deadlist_object, tx);
4896 	dsl_dataset_unset_remap_deadlist_object(ds, tx);
4897 	spa_feature_decr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
4898 }
4899 
4900 void
4901 dsl_dataset_create_remap_deadlist(dsl_dataset_t *ds, dmu_tx_t *tx)
4902 {
4903 	uint64_t remap_deadlist_obj;
4904 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
4905 
4906 	ASSERT(dmu_tx_is_syncing(tx));
4907 	ASSERT(MUTEX_HELD(&ds->ds_remap_deadlist_lock));
4908 	/*
4909 	 * Currently we only create remap deadlists when there are indirect
4910 	 * vdevs with referenced mappings.
4911 	 */
4912 	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL));
4913 
4914 	remap_deadlist_obj = dsl_deadlist_clone(
4915 	    &ds->ds_deadlist, UINT64_MAX,
4916 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, tx);
4917 	dsl_dataset_set_remap_deadlist_object(ds,
4918 	    remap_deadlist_obj, tx);
4919 	dsl_deadlist_open(&ds->ds_remap_deadlist, spa_meta_objset(spa),
4920 	    remap_deadlist_obj);
4921 	spa_feature_incr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
4922 }
4923 
4924 void
4925 dsl_dataset_activate_redaction(dsl_dataset_t *ds, uint64_t *redact_snaps,
4926     uint64_t num_redact_snaps, dmu_tx_t *tx)
4927 {
4928 	uint64_t dsobj = ds->ds_object;
4929 	struct feature_type_uint64_array_arg *ftuaa =
4930 	    kmem_zalloc(sizeof (*ftuaa), KM_SLEEP);
4931 	ftuaa->length = (int64_t)num_redact_snaps;
4932 	if (num_redact_snaps > 0) {
4933 		ftuaa->array = kmem_alloc(num_redact_snaps * sizeof (uint64_t),
4934 		    KM_SLEEP);
4935 		memcpy(ftuaa->array, redact_snaps, num_redact_snaps *
4936 		    sizeof (uint64_t));
4937 	}
4938 	dsl_dataset_activate_feature(dsobj, SPA_FEATURE_REDACTED_DATASETS,
4939 	    ftuaa, tx);
4940 	ds->ds_feature[SPA_FEATURE_REDACTED_DATASETS] = ftuaa;
4941 }
4942 
4943 /*
4944  * Find and return (in *oldest_dsobj) the oldest snapshot of the dsobj
4945  * dataset whose birth time is >= min_txg.
4946  */
4947 int
4948 dsl_dataset_oldest_snapshot(spa_t *spa, uint64_t head_ds, uint64_t min_txg,
4949     uint64_t *oldest_dsobj)
4950 {
4951 	dsl_dataset_t *ds;
4952 	dsl_pool_t *dp = spa->spa_dsl_pool;
4953 
4954 	int error = dsl_dataset_hold_obj(dp, head_ds, FTAG, &ds);
4955 	if (error != 0)
4956 		return (error);
4957 
4958 	uint64_t prev_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
4959 	uint64_t prev_obj_txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
4960 
4961 	while (prev_obj != 0 && min_txg < prev_obj_txg) {
4962 		dsl_dataset_rele(ds, FTAG);
4963 		if ((error = dsl_dataset_hold_obj(dp, prev_obj,
4964 		    FTAG, &ds)) != 0)
4965 			return (error);
4966 		prev_obj_txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
4967 		prev_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
4968 	}
4969 	*oldest_dsobj = ds->ds_object;
4970 	dsl_dataset_rele(ds, FTAG);
4971 	return (0);
4972 }
4973 
4974 ZFS_MODULE_PARAM(zfs, zfs_, max_recordsize, INT, ZMOD_RW,
4975 	"Max allowed record size");
4976 
4977 ZFS_MODULE_PARAM(zfs, zfs_, allow_redacted_dataset_mount, INT, ZMOD_RW,
4978 	"Allow mounting of redacted datasets");
4979 
4980 EXPORT_SYMBOL(dsl_dataset_hold);
4981 EXPORT_SYMBOL(dsl_dataset_hold_flags);
4982 EXPORT_SYMBOL(dsl_dataset_hold_obj);
4983 EXPORT_SYMBOL(dsl_dataset_hold_obj_flags);
4984 EXPORT_SYMBOL(dsl_dataset_own);
4985 EXPORT_SYMBOL(dsl_dataset_own_obj);
4986 EXPORT_SYMBOL(dsl_dataset_name);
4987 EXPORT_SYMBOL(dsl_dataset_rele);
4988 EXPORT_SYMBOL(dsl_dataset_rele_flags);
4989 EXPORT_SYMBOL(dsl_dataset_disown);
4990 EXPORT_SYMBOL(dsl_dataset_tryown);
4991 EXPORT_SYMBOL(dsl_dataset_create_sync);
4992 EXPORT_SYMBOL(dsl_dataset_create_sync_dd);
4993 EXPORT_SYMBOL(dsl_dataset_snapshot_check);
4994 EXPORT_SYMBOL(dsl_dataset_snapshot_sync);
4995 EXPORT_SYMBOL(dsl_dataset_promote);
4996 EXPORT_SYMBOL(dsl_dataset_user_hold);
4997 EXPORT_SYMBOL(dsl_dataset_user_release);
4998 EXPORT_SYMBOL(dsl_dataset_get_holds);
4999 EXPORT_SYMBOL(dsl_dataset_get_blkptr);
5000 EXPORT_SYMBOL(dsl_dataset_get_spa);
5001 EXPORT_SYMBOL(dsl_dataset_modified_since_snap);
5002 EXPORT_SYMBOL(dsl_dataset_space_written);
5003 EXPORT_SYMBOL(dsl_dataset_space_wouldfree);
5004 EXPORT_SYMBOL(dsl_dataset_sync);
5005 EXPORT_SYMBOL(dsl_dataset_block_born);
5006 EXPORT_SYMBOL(dsl_dataset_block_kill);
5007 EXPORT_SYMBOL(dsl_dataset_dirty);
5008 EXPORT_SYMBOL(dsl_dataset_stats);
5009 EXPORT_SYMBOL(dsl_dataset_fast_stat);
5010 EXPORT_SYMBOL(dsl_dataset_space);
5011 EXPORT_SYMBOL(dsl_dataset_fsid_guid);
5012 EXPORT_SYMBOL(dsl_dsobj_to_dsname);
5013 EXPORT_SYMBOL(dsl_dataset_check_quota);
5014 EXPORT_SYMBOL(dsl_dataset_clone_swap_check_impl);
5015 EXPORT_SYMBOL(dsl_dataset_clone_swap_sync_impl);
5016