xref: /freebsd/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.kshlib (revision 61145dc2b94f12f6a47344fb9aac702321880e43)
1# SPDX-License-Identifier: CDDL-1.0
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or https://opensource.org/licenses/CDDL-1.0.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23#
24# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
25# Use is subject to license terms.
26#
27
28#
29# Copyright (c) 2013, 2018 by Delphix. All rights reserved.
30# Copyright (c) 2020 by Datto Inc. All rights reserved.
31#
32
33. $STF_SUITE/include/libtest.shlib
34. $STF_SUITE/include/math.shlib
35. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
36. $STF_SUITE/tests/functional/rsend/rsend.cfg
37
38#
39# Set up test model which includes various datasets
40#
41#               @final
42#               @snapB
43#               @init
44#               |
45#   ______ pclone
46#  |      /
47#  |@psnap
48#  ||                         @final
49#  ||@final       @final      @snapC
50#  ||@snapC       @snapC      @snapB
51#  ||@snapA       @snapB      @snapA
52#  ||@init        @init       @init
53#  |||            |           |
54# $pool -------- $FS ------- fs1 ------- fs2
55#    \             \\_____     \          |
56#     vol           vol   \____ \         @fsnap
57#      |              |        \ \              \
58#      @init          @vsnap   |  ------------ fclone
59#      @snapA         @init \  |                    |
60#      @final         @snapB \ |                    @init
61#                     @snapC  vclone                @snapA
62#                     @final       |                @final
63#                                 @init
64#                                 @snapC
65#                                 @final
66#
67# $1 pool name
68#
69function setup_test_model
70{
71	typeset pool=$1
72
73	log_must zfs create -p $pool/$FS/fs1/fs2
74
75	log_must zfs snapshot $pool@psnap
76	log_must zfs clone $pool@psnap $pool/pclone
77
78	if is_global_zone ; then
79		log_must zfs create -V 16M $pool/vol
80		log_must zfs create -V 16M $pool/$FS/vol
81		block_device_wait
82
83		log_must zfs snapshot $pool/$FS/vol@vsnap
84		log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
85		block_device_wait
86	fi
87
88	log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
89	log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
90	log_must zfs snapshot -r $pool@init
91
92	log_must snapshot_tree $pool@snapA
93	log_must snapshot_tree $pool@snapC
94	log_must snapshot_tree $pool/pclone@snapB
95	log_must snapshot_tree $pool/$FS@snapB
96	log_must snapshot_tree $pool/$FS@snapC
97	log_must snapshot_tree $pool/$FS/fs1@snapA
98	log_must snapshot_tree $pool/$FS/fs1@snapB
99	log_must snapshot_tree $pool/$FS/fs1@snapC
100	log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
101
102	if is_global_zone ; then
103		log_must zfs snapshot $pool/vol@snapA
104		log_must zfs snapshot $pool/$FS/vol@snapB
105		log_must zfs snapshot $pool/$FS/vol@snapC
106		log_must zfs snapshot $pool/$FS/vclone@snapC
107	fi
108
109	log_must zfs snapshot -r $pool@final
110
111	return 0
112}
113
114#
115# Cleanup the BACKDIR and given pool content and all the sub datasets
116#
117# $1 pool name
118#
119function cleanup_pool
120{
121	typeset pool=$1
122	log_must rm -rf $BACKDIR/*
123
124	if is_global_zone ; then
125		#
126		# Linux: Issuing a `df` seems to properly force any negative
127		# dcache entries to be invalidated preventing failures when
128		# accessing the mount point. Additional investigation required.
129		#
130		# https://github.com/openzfs/zfs/issues/6143
131		#
132		log_must eval "df >/dev/null"
133		log_must_busy zfs destroy -Rf $pool
134	else
135		typeset list=$(zfs list -H -r -t all -o name $pool)
136		for ds in $list ; do
137			if [[ $ds != $pool ]] ; then
138				if datasetexists $ds ; then
139					log_must_busy zfs destroy -Rf $ds
140				fi
141			fi
142		done
143	fi
144
145	typeset mntpnt=$(get_prop mountpoint $pool)
146	if ! ismounted $pool ; then
147		# Make sure mountpoint directory is empty
148		if [[ -d $mntpnt ]]; then
149			log_must rm -rf $mntpnt/*
150		fi
151
152		log_must zfs mount $pool
153	fi
154	if [[ -d $mntpnt ]]; then
155		rm -rf $mntpnt/*
156	fi
157}
158
159#
160# Detect if the given two filesystems have same sub-datasets
161#
162# $1 source filesystem
163# $2 destination filesystem
164#
165function cmp_ds_subs
166{
167	typeset src_fs=$1
168	typeset dst_fs=$2
169
170	diff \
171		<(zfs list -rHt all -o name $src_fs | sed "s:^$src_fs:PREFIX:g") \
172		<(zfs list -rHt all -o name $dst_fs | sed "s:^$dst_fs:PREFIX:g")
173}
174
175#
176# Compare all the directories and files in two filesystems
177#
178# $1 source filesystem
179# $2 destination filesystem
180#
181function cmp_ds_cont
182{
183	typeset src_fs=$1
184	typeset dst_fs=$2
185
186	typeset srcdir dstdir
187	srcdir=$(get_prop mountpoint $src_fs)
188	dstdir=$(get_prop mountpoint $dst_fs)
189
190	replay_directory_diff $srcdir $dstdir
191}
192
193#
194# Compare the given two dataset properties
195#
196# $1 dataset 1
197# $2 dataset 2
198# $3 -n == don't track property source
199# $4 -n == don't track the origin property
200#
201function cmp_ds_prop
202{
203	typeset dtst1=$1
204	typeset dtst2=$2
205	typeset nosource=$3
206	typeset noorigin=$4
207	typeset source=",source"; [ -n "$nosource" ] && source=
208	typeset origin=",origin"; [ -n "$noorigin" ] && origin=
209	typeset props="type$origin,volblocksize,acltype,dnodesize"
210	props+=",atime,canmount,checksum,compression,copies,devices"
211	props+=",exec,quota,readonly,recordsize,reservation,setuid"
212	props+=",snapdir,version,volsize,xattr,mountpoint"
213	if is_freebsd; then
214		props+=",jailed"
215	else
216		props+=",zoned"
217	fi
218
219	diff \
220		<(zfs get -Ho property,value$source $props $dtst1 | sed -e "s:$dtst1:PREFIX:g" -e 's/^origin	[^@]*/origin	POOL/' -e 's/	inherited from [^/]*/	inherited from POOL/') \
221		<(zfs get -Ho property,value$source $props $dtst2 | sed -e "s:$dtst2:PREFIX:g" -e 's/^origin	[^@]*/origin	POOL/' -e 's/	inherited from [^/]*/	inherited from POOL/')
222}
223
224#
225# Random create directories and files
226#
227# $1 directory
228#
229function random_tree
230{
231	typeset dir=$1
232
233	if [[ -d $dir ]]; then
234		rm -rf $dir
235	fi
236	mkdir -p $dir
237	typeset -i ret=$?
238
239	typeset -i nl nd nf
240	((nl = RANDOM % 6 + 1))
241	((nd = RANDOM % 3 ))
242	((nf = RANDOM % 5 ))
243	mktree -b $dir -l $nl -d $nd -f $nf
244	((ret |= $?))
245
246	return $ret
247}
248
249#
250# Put data in filesystem and take snapshot
251#
252# $1 snapshot name
253#
254function snapshot_tree
255{
256	typeset snap=$1
257	typeset ds=${snap%%@*}
258	typeset type=$(get_prop "type" $ds)
259
260	typeset -i ret=0
261	if [[ $type == "filesystem" ]]; then
262		typeset mntpnt=$(get_prop mountpoint $ds)
263
264		if ((ret == 0)) ; then
265			eval random_tree $mntpnt/${snap##$ds}
266			((ret |= $?))
267		fi
268	fi
269
270	if ((ret == 0)) ; then
271		zfs snapshot $snap
272		((ret |= $?))
273	fi
274
275	return $ret
276}
277
278#
279# Destroy the given snapshot and stuff
280#
281# $1 snapshot
282#
283function destroy_tree
284{
285	typeset -i ret=0
286	typeset snap
287	for snap in "$@" ; do
288		log_must_busy zfs destroy $snap
289
290		typeset ds=${snap%%@*}
291		typeset type=$(get_prop "type" $ds)
292		if [[ $type == "filesystem" ]]; then
293			typeset mntpnt=$(get_prop mountpoint $ds)
294			if [[ -n $mntpnt ]]; then
295				rm -rf $mntpnt/$snap
296			fi
297		fi
298	done
299
300	return 0
301}
302
303#
304# Get all the sub-datasets of give dataset with specific suffix
305#
306# $1 Given dataset
307# $2 Suffix
308#
309function getds_with_suffix
310{
311	typeset ds=$1
312	typeset suffix=$2
313
314	zfs list -rHt all -o name $ds | grep "$suffix$"
315}
316
317#
318# Output inherited properties which is edited for file system
319#
320function fs_inherit_prop
321{
322	typeset fs_prop
323	if is_global_zone ; then
324		fs_prop=$(zfs inherit 2>&1 | \
325		    awk '$2=="YES" && $3=="YES" {print $1}')
326		if ! is_te_enabled ; then
327		        fs_prop=$(echo $fs_prop | grep -v "mlslabel")
328		fi
329	else
330		fs_prop=$(zfs inherit 2>&1 | \
331		    awk '$2=="YES" && $3=="YES" && !/devices|mlslabel|sharenfs|sharesmb|zoned/ {print $1}')
332	fi
333
334	echo $fs_prop
335}
336
337#
338# Output inherited properties for volume
339#
340function vol_inherit_prop
341{
342	echo checksum readonly
343}
344
345#
346# Get the destination dataset to compare
347#
348function get_dst_ds
349{
350	typeset srcfs=$1
351	typeset dstfs=$2
352
353	#
354	# If the srcfs is not pool
355	#
356	if ! zpool list $srcfs > /dev/null 2>&1 ; then
357		eval dstfs="$dstfs/${srcfs#*/}"
358	fi
359
360	echo $dstfs
361}
362
363#
364# Make test files
365#
366# $1 Number of files to create
367# $2 Maximum file size
368# $3 File ID offset
369# $4 File system to create the files on
370#
371function mk_files
372{
373	nfiles=$1
374	maxsize=$2
375	file_id_offset=$3
376	fs=$4
377	bs=512
378
379	for ((i=0; i<$nfiles; i=i+1)); do
380		file_name="/$fs/file-$maxsize-$((i+$file_id_offset))"
381		file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
382
383		#
384		# Create an interesting mix of files which contain both
385		# data blocks and holes for more realistic test coverage.
386		# Half the files are created as sparse then partially filled,
387		# the other half is dense then a hole is punched in the file.
388		#
389		if [ $((RANDOM % 2)) -eq 0 ]; then
390			truncate -s $file_size $file_name || \
391			    log_fail "Failed to create $file_name"
392			dd if=/dev/urandom of=$file_name \
393			    bs=$bs count=$(($file_size / 2 / $bs)) \
394			    seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
395			    conv=notrunc >/dev/null 2>&1 || \
396			    log_fail "Failed to create $file_name"
397		else
398			dd if=/dev/urandom of=$file_name \
399			    bs=$file_size count=1 >/dev/null 2>&1 || \
400			    log_fail "Failed to create $file_name"
401			dd if=/dev/zero of=$file_name \
402			    bs=$bs count=$(($file_size / 2 / $bs)) \
403			    seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
404			    conv=notrunc >/dev/null 2>&1 || \
405			    log_fail "Failed to create $file_name"
406		fi
407	done
408	echo Created $nfiles files of random sizes up to $maxsize bytes
409}
410
411#
412# Remove test files
413#
414# $1 Number of files to remove
415# $2 Maximum file size
416# $3 File ID offset
417# $4 File system to remove the files from
418#
419function rm_files
420{
421	nfiles=$1
422	maxsize=$2
423	file_id_offset=$3
424	fs=$4
425
426	for ((i=0; i<$nfiles; i=i+1)); do
427		rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
428	done
429	echo Removed $nfiles files of random sizes up to $maxsize bytes
430}
431
432#
433# Simulate a random set of operations which could be reasonably expected
434# to occur on an average filesystem.
435#
436# $1 Number of files to modify
437# $2 Maximum file size
438# $3 File system to modify the file on
439# $4 Enabled xattrs (optional)
440#
441function churn_files
442{
443	nfiles=$1
444	maxsize=$2
445	fs=$3
446	xattrs=${4:-1}
447
448	#
449	# Remove roughly half of the files in order to make it more
450	# likely that a dnode will be reallocated.
451	#
452	for ((i=0; i<$nfiles; i=i+1)); do
453		file_name="/$fs/file-$i"
454
455		if [[ -e $file_name ]]; then
456			if [ $((RANDOM % 2)) -eq 0 ]; then
457				rm $file_name || \
458				    log_fail "Failed to remove $file_name"
459			fi
460		fi
461	done
462
463	#
464	# Remount the filesystem to simulate normal usage.  This resets
465	# the last allocated object id allowing for new objects to be
466	# reallocated in the locations of previously freed objects.
467	#
468	log_must zfs unmount $fs
469	log_must zfs mount $fs
470
471	for i in {0..$nfiles}; do
472		file_name="/$fs/file-$i"
473		file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
474
475		#
476		# When the file exists modify it in one of five ways to
477		# simulate normal usage:
478		# - (20%) Remove and set and extended attribute on the file
479		# - (20%) Overwrite the existing file
480		# - (20%) Truncate the existing file to a random length
481		# - (20%) Truncate the existing file to zero length
482		# - (20%) Remove the file
483		#
484		# Otherwise create the missing file.  20% of the created
485		# files will be small and use embedded block pointers, the
486		# remainder with have random sizes up to the maximum size.
487		# Three extended attributes are attached to all of the files.
488		#
489		if [[ -e $file_name ]]; then
490			value=$((RANDOM % 5))
491			if [ $value -eq 0 -a $xattrs -ne 0 ]; then
492				attrname="testattr$((RANDOM % 3))"
493				attrlen="$(((RANDOM % 1000) + 1))"
494				attrvalue="$(random_string VALID_NAME_CHAR \
495				    $attrlen)"
496				rm_xattr $attrname $file_name || \
497				    log_fail "Failed to remove $attrname"
498				set_xattr $attrname "$attrvalue" $file_name || \
499				    log_fail "Failed to set $attrname"
500			elif [ $value -eq 1 ]; then
501				dd if=/dev/urandom of=$file_name \
502				    bs=$file_size count=1 >/dev/null 2>&1 || \
503				    log_fail "Failed to overwrite $file_name"
504			elif [ $value -eq 2 ]; then
505				truncate -s $file_size $file_name || \
506				    log_fail "Failed to truncate $file_name"
507			elif [ $value -eq 3 ]; then
508				truncate -s 0 $file_name || \
509				    log_fail "Failed to truncate $file_name"
510			else
511				rm $file_name || \
512				    log_fail "Failed to remove $file_name"
513			fi
514		else
515			if [ $((RANDOM % 5)) -eq 0 ]; then
516				file_size=$((($RANDOM % 64) + 1))
517			fi
518
519			dd if=/dev/urandom of=$file_name \
520			    bs=$file_size count=1 >/dev/null 2>&1 || \
521			    log_fail "Failed to create $file_name"
522
523			if [ $xattrs -ne 0 ]; then
524				for j in {0..2}; do
525					attrname="testattr$j"
526					attrlen="$(((RANDOM % 1000) + 1))"
527					attrvalue="$(random_string \
528					    VALID_NAME_CHAR $attrlen)"
529					set_xattr $attrname \
530					    "$attrvalue" $file_name || \
531					    log_fail "Failed to set $attrname"
532				done
533			fi
534		fi
535	done
536
537	return 0
538}
539
540#
541# Mess up a send file's contents
542#
543# $1 The send file path
544#
545function mess_send_file
546{
547	typeset -i minsize=2072
548	file=$1
549
550	filesize=$(stat_size $file)
551	if [ $filesize -lt $minsize ]; then
552		log_fail "Send file too small: $filesize < $minsize"
553	fi
554
555	# Truncate the send stream at a random offset after the DRR_BEGIN
556	# record (beyond 2072 bytes), any smaller than this and the receiving
557	# system won't have enough info to create the partial dataset at all.
558	# We use zstream dump to verify there is an intact DRR_BEGIN record.
559	offset=$(((($RANDOM * $RANDOM) % ($filesize - $minsize)) + $minsize))
560	nr_begins=$(head -c $offset $file | zstream dump | \
561	    awk '/DRR_BEGIN/ { print $5 }')
562	log_must [ "$nr_begins" -eq 1 ]
563
564	if (($RANDOM % 7 <= 1)); then
565		#
566		# We corrupt 2 bytes to minimize the chance that we
567		# write the same value that's already there.
568		#
569		log_must eval "dd if=/dev/urandom of=$file conv=notrunc " \
570		    "bs=1 count=2 seek=$offset >/dev/null 2>&1"
571	else
572		log_must truncate -s $offset $file
573	fi
574}
575
576#
577# Diff the send/receive filesystems
578#
579# $1 The sent filesystem
580# $2 The received filesystem
581#
582function file_check
583{
584	sendfs=$1
585	recvfs=$2
586
587	if [[ -d /$recvfs/.zfs/snapshot/a && -d \
588	    /$sendfs/.zfs/snapshot/a ]]; then
589		log_must directory_diff /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
590	fi
591	if [[ -d /$recvfs/.zfs/snapshot/b && -d \
592	    /$sendfs/.zfs/snapshot/b ]]; then
593		log_must directory_diff /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
594	fi
595}
596
597#
598# Resume test helper
599#
600# $1 The ZFS send command
601# $2 The filesystem where the streams are sent
602# $3 The receive filesystem
603# $4 Test dry-run (optional)
604#
605function resume_test
606{
607	typeset sendcmd=$1
608	typeset streamfs=$2
609	typeset recvfs=$3
610	typeset dryrun=${4:-1}
611
612	stream_num=1
613	log_must eval "$sendcmd >/$streamfs/$stream_num"
614
615	for ((i=0; i<2; i=i+1)); do
616		mess_send_file /$streamfs/$stream_num
617		log_mustnot eval "zfs recv -suv $recvfs </$streamfs/$stream_num"
618		stream_num=$((stream_num+1))
619
620		token=$(get_prop receive_resume_token $recvfs)
621
622		# Do a dry-run
623		[ $dryrun -ne 0 ] && \
624			log_must eval "zfs send -nvt $token > /dev/null"
625
626		log_must eval "zfs send -t $token  >/$streamfs/$stream_num"
627	done
628	log_must eval "zfs recv -suv $recvfs </$streamfs/$stream_num"
629}
630
631function get_resume_token
632{
633	sendcmd=$1
634	streamfs=$2
635	recvfs=$3
636
637	log_must eval "$sendcmd > /$streamfs/1"
638	mess_send_file /$streamfs/1
639	log_mustnot eval "zfs recv -suv $recvfs < /$streamfs/1 2>&1"
640	get_prop receive_resume_token $recvfs > /$streamfs/resume_token
641}
642
643#
644# Setup filesystems for the resumable send/receive tests
645#
646# $1 The "send" filesystem
647# $2 The "recv" filesystem
648#
649function test_fs_setup
650{
651	typeset sendfs=$1
652	typeset recvfs=$2
653	typeset streamfs=$3
654	typeset sendpool=${sendfs%%/*}
655	typeset recvpool=${recvfs%%/*}
656
657	datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
658	datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
659	datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
660
661	if datasetexists $sendfs || zfs create -o compress=lz4 $sendfs; then
662		mk_files 1000 256 0 $sendfs &
663		mk_files 1000 131072 0 $sendfs &
664		mk_files 100 1048576 0 $sendfs &
665		mk_files 10 10485760 0 $sendfs &
666		mk_files 1 104857600 0 $sendfs &
667		log_must wait
668		log_must zfs snapshot $sendfs@a
669
670		rm_files 200 256 0 $sendfs &
671		rm_files 200 131072 0 $sendfs &
672		rm_files 20 1048576 0 $sendfs &
673		rm_files 2 10485760 0 $sendfs &
674		log_must wait
675
676		mk_files 400 256 0 $sendfs &
677		mk_files 400 131072 0 $sendfs &
678		mk_files 40 1048576 0 $sendfs &
679		mk_files 4 10485760 0 $sendfs &
680		log_must wait
681
682		log_must zfs snapshot $sendfs@b
683		log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
684		log_must eval "zfs send -v -i @a $sendfs@b " \
685		    ">/$sendpool/incremental.zsend"
686	fi
687
688	log_must zfs create -o compress=lz4 $streamfs
689}
690
691#
692# Check to see if the specified features are set in a send stream.
693# The values for these features are found in include/sys/zfs_ioctl.h
694#
695# $1 The stream file
696# $2-$n The flags expected in the stream
697#
698function stream_has_features
699{
700	typeset file=$1
701	shift
702
703	[[ -f $file ]] || log_fail "Couldn't find file: $file"
704	typeset flags=$(zstream dump $file | \
705	    awk '/features =/ {features = $3} END {print features}')
706	typeset -A feature
707	feature[dedup]="1"
708	feature[dedupprops]="2"
709	feature[sa_spill]="4"
710	feature[embed_data]="10000"
711	feature[lz4]="20000"
712	feature[mooch_byteswap]="40000"
713	feature[large_blocks]="80000"
714	feature[resuming]="100000"
715	feature[redacted]="200000"
716	feature[compressed]="400000"
717	feature[longname]="10000000"
718
719	typeset flag known derived=0
720	for flag in "$@"; do
721		known=${feature[$flag]}
722		[[ -z $known ]] && log_fail "Unknown feature: $flag"
723
724		derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]})))
725		[[ $derived = $known ]] || return 1
726	done
727
728	return 0
729}
730
731#
732# Given a send stream, verify that the size of the stream matches what's
733# expected based on the source or target dataset. If the stream is an
734# incremental stream, subtract the size of the source snapshot before
735# comparing. This function does not currently handle incremental streams
736# that remove data.
737#
738# $1 The zstream dump output file
739# $2 The dataset to compare against
740#    This can be a source of a send or recv target (fs, not snapshot)
741# $3 The percentage below which verification is deemed a failure
742# $4 The source snapshot of an incremental send
743#
744
745function verify_stream_size
746{
747	typeset stream=$1
748	typeset ds=$2
749	typeset percent=${3:-90}
750	typeset inc_src=$4
751
752	[[ -f $stream ]] || log_fail "No such file: $stream"
753	datasetexists $ds || log_fail "No such dataset: $ds"
754
755	typeset stream_size=$(zstream dump $stream | sed -n \
756	    's/	Total payload size = \(.*\) (0x.*)/\1/p')
757
758	typeset inc_size=0
759	if [[ -n $inc_src ]]; then
760		inc_size=$(get_prop lrefer $inc_src)
761		if stream_has_features $stream compressed; then
762			inc_size=$(get_prop refer $inc_src)
763		fi
764	fi
765
766	if stream_has_features $stream compressed; then
767		ds_size=$(get_prop refer $ds)
768	else
769		ds_size=$(get_prop lrefer $ds)
770	fi
771	ds_size=$((ds_size - inc_size))
772
773	log_must within_percent $stream_size $ds_size $percent
774}
775
776# Cleanup function for tests involving resumable send
777function resume_cleanup
778{
779	typeset sendfs=$1
780	typeset streamfs=$2
781	typeset sendpool=${sendfs%%/*}
782
783	datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
784	datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
785	cleanup_pool $POOL2
786	rm -f /$sendpool/initial.zsend /$sendpool/incremental.zsend
787}
788
789# Randomly set the property to one of the enumerated values.
790function rand_set_prop
791{
792	typeset dtst=$1
793	typeset prop=$2
794	shift 2
795	typeset value=$(random_get $@)
796
797	log_must eval "zfs set $prop='$value' $dtst"
798}
799
800# Generate a recursive checksum of a filesystem which includes the file
801# contents and any associated extended attributes.
802function recursive_cksum
803{
804	case "$(uname)" in
805	FreeBSD)
806		find $1 -type f -exec sh -c 'xxh128sum {}; \
807		    lsextattr -q system {} | xxh128sum; \
808		    lsextattr -q user {} | xxh128sum' \; \
809		    | sort -k 2 | awk '{ print $1 }' | xxh128digest
810		;;
811	*)
812		find $1 -type f -exec sh -c 'xxh128sum {}; getfattr \
813		    --absolute-names --only-values -d {} | xxh128sum' \
814		    \; | sort -k 2 | awk '{ print $1 }' | xxh128digest
815		;;
816	esac
817}
818