xref: /freebsd/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/rsend.kshlib (revision 7a7741af18d6c8a804cc643cb7ecda9d730c6aa6)
1#
2# CDDL HEADER START
3#
4# The contents of this file are subject to the terms of the
5# Common Development and Distribution License (the "License").
6# You may not use this file except in compliance with the License.
7#
8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9# or https://opensource.org/licenses/CDDL-1.0.
10# See the License for the specific language governing permissions
11# and limitations under the License.
12#
13# When distributing Covered Code, include this CDDL HEADER in each
14# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15# If applicable, add the following below this CDDL HEADER, with the
16# fields enclosed by brackets "[]" replaced with your own identifying
17# information: Portions Copyright [yyyy] [name of copyright owner]
18#
19# CDDL HEADER END
20#
21
22#
23# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24# Use is subject to license terms.
25#
26
27#
28# Copyright (c) 2013, 2018 by Delphix. All rights reserved.
29# Copyright (c) 2020 by Datto Inc. All rights reserved.
30#
31
32. $STF_SUITE/include/libtest.shlib
33. $STF_SUITE/include/math.shlib
34. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
35. $STF_SUITE/tests/functional/rsend/rsend.cfg
36
37#
38# Set up test model which includes various datasets
39#
40#               @final
41#               @snapB
42#               @init
43#               |
44#   ______ pclone
45#  |      /
46#  |@psnap
47#  ||                         @final
48#  ||@final       @final      @snapC
49#  ||@snapC       @snapC      @snapB
50#  ||@snapA       @snapB      @snapA
51#  ||@init        @init       @init
52#  |||            |           |
53# $pool -------- $FS ------- fs1 ------- fs2
54#    \             \\_____     \          |
55#     vol           vol   \____ \         @fsnap
56#      |              |        \ \              \
57#      @init          @vsnap   |  ------------ fclone
58#      @snapA         @init \  |                    |
59#      @final         @snapB \ |                    @init
60#                     @snapC  vclone                @snapA
61#                     @final       |                @final
62#                                 @init
63#                                 @snapC
64#                                 @final
65#
66# $1 pool name
67#
68function setup_test_model
69{
70	typeset pool=$1
71
72	log_must zfs create -p $pool/$FS/fs1/fs2
73
74	log_must zfs snapshot $pool@psnap
75	log_must zfs clone $pool@psnap $pool/pclone
76
77	if is_global_zone ; then
78		log_must zfs create -V 16M $pool/vol
79		log_must zfs create -V 16M $pool/$FS/vol
80		block_device_wait
81
82		log_must zfs snapshot $pool/$FS/vol@vsnap
83		log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
84		block_device_wait
85	fi
86
87	log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
88	log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
89	log_must zfs snapshot -r $pool@init
90
91	log_must snapshot_tree $pool@snapA
92	log_must snapshot_tree $pool@snapC
93	log_must snapshot_tree $pool/pclone@snapB
94	log_must snapshot_tree $pool/$FS@snapB
95	log_must snapshot_tree $pool/$FS@snapC
96	log_must snapshot_tree $pool/$FS/fs1@snapA
97	log_must snapshot_tree $pool/$FS/fs1@snapB
98	log_must snapshot_tree $pool/$FS/fs1@snapC
99	log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
100
101	if is_global_zone ; then
102		log_must zfs snapshot $pool/vol@snapA
103		log_must zfs snapshot $pool/$FS/vol@snapB
104		log_must zfs snapshot $pool/$FS/vol@snapC
105		log_must zfs snapshot $pool/$FS/vclone@snapC
106	fi
107
108	log_must zfs snapshot -r $pool@final
109
110	return 0
111}
112
113#
114# Cleanup the BACKDIR and given pool content and all the sub datasets
115#
116# $1 pool name
117#
118function cleanup_pool
119{
120	typeset pool=$1
121	log_must rm -rf $BACKDIR/*
122
123	if is_global_zone ; then
124		#
125		# Linux: Issuing a `df` seems to properly force any negative
126		# dcache entries to be invalidated preventing failures when
127		# accessing the mount point. Additional investigation required.
128		#
129		# https://github.com/openzfs/zfs/issues/6143
130		#
131		log_must eval "df >/dev/null"
132		log_must_busy zfs destroy -Rf $pool
133	else
134		typeset list=$(zfs list -H -r -t all -o name $pool)
135		for ds in $list ; do
136			if [[ $ds != $pool ]] ; then
137				if datasetexists $ds ; then
138					log_must_busy zfs destroy -Rf $ds
139				fi
140			fi
141		done
142	fi
143
144	typeset mntpnt=$(get_prop mountpoint $pool)
145	if ! ismounted $pool ; then
146		# Make sure mountpoint directory is empty
147		if [[ -d $mntpnt ]]; then
148			log_must rm -rf $mntpnt/*
149		fi
150
151		log_must zfs mount $pool
152	fi
153	if [[ -d $mntpnt ]]; then
154		rm -rf $mntpnt/*
155	fi
156}
157
158#
159# Detect if the given two filesystems have same sub-datasets
160#
161# $1 source filesystem
162# $2 destination filesystem
163#
164function cmp_ds_subs
165{
166	typeset src_fs=$1
167	typeset dst_fs=$2
168
169	diff \
170		<(zfs list -rHt all -o name $src_fs | sed "s:^$src_fs:PREFIX:g") \
171		<(zfs list -rHt all -o name $dst_fs | sed "s:^$dst_fs:PREFIX:g")
172}
173
174#
175# Compare all the directories and files in two filesystems
176#
177# $1 source filesystem
178# $2 destination filesystem
179#
180function cmp_ds_cont
181{
182	typeset src_fs=$1
183	typeset dst_fs=$2
184
185	typeset srcdir dstdir
186	srcdir=$(get_prop mountpoint $src_fs)
187	dstdir=$(get_prop mountpoint $dst_fs)
188
189	replay_directory_diff $srcdir $dstdir
190}
191
192#
193# Compare the given two dataset properties
194#
195# $1 dataset 1
196# $2 dataset 2
197# $3 -n == don't track property source
198# $4 -n == don't track the origin property
199#
200function cmp_ds_prop
201{
202	typeset dtst1=$1
203	typeset dtst2=$2
204	typeset nosource=$3
205	typeset noorigin=$4
206	typeset source=",source"; [ -n "$nosource" ] && source=
207	typeset origin=",origin"; [ -n "$noorigin" ] && origin=
208	typeset props="type$origin,volblocksize,acltype,dnodesize"
209	props+=",atime,canmount,checksum,compression,copies,devices"
210	props+=",exec,quota,readonly,recordsize,reservation,setuid"
211	props+=",snapdir,version,volsize,xattr,mountpoint"
212	if is_freebsd; then
213		props+=",jailed"
214	else
215		props+=",zoned"
216	fi
217
218	diff \
219		<(zfs get -Ho property,value$source $props $dtst1 | sed -e "s:$dtst1:PREFIX:g" -e 's/^origin	[^@]*/origin	POOL/' -e 's/	inherited from [^/]*/	inherited from POOL/') \
220		<(zfs get -Ho property,value$source $props $dtst2 | sed -e "s:$dtst2:PREFIX:g" -e 's/^origin	[^@]*/origin	POOL/' -e 's/	inherited from [^/]*/	inherited from POOL/')
221}
222
223#
224# Random create directories and files
225#
226# $1 directory
227#
228function random_tree
229{
230	typeset dir=$1
231
232	if [[ -d $dir ]]; then
233		rm -rf $dir
234	fi
235	mkdir -p $dir
236	typeset -i ret=$?
237
238	typeset -i nl nd nf
239	((nl = RANDOM % 6 + 1))
240	((nd = RANDOM % 3 ))
241	((nf = RANDOM % 5 ))
242	mktree -b $dir -l $nl -d $nd -f $nf
243	((ret |= $?))
244
245	return $ret
246}
247
248#
249# Put data in filesystem and take snapshot
250#
251# $1 snapshot name
252#
253function snapshot_tree
254{
255	typeset snap=$1
256	typeset ds=${snap%%@*}
257	typeset type=$(get_prop "type" $ds)
258
259	typeset -i ret=0
260	if [[ $type == "filesystem" ]]; then
261		typeset mntpnt=$(get_prop mountpoint $ds)
262
263		if ((ret == 0)) ; then
264			eval random_tree $mntpnt/${snap##$ds}
265			((ret |= $?))
266		fi
267	fi
268
269	if ((ret == 0)) ; then
270		zfs snapshot $snap
271		((ret |= $?))
272	fi
273
274	return $ret
275}
276
277#
278# Destroy the given snapshot and stuff
279#
280# $1 snapshot
281#
282function destroy_tree
283{
284	typeset -i ret=0
285	typeset snap
286	for snap in "$@" ; do
287		log_must_busy zfs destroy $snap
288
289		typeset ds=${snap%%@*}
290		typeset type=$(get_prop "type" $ds)
291		if [[ $type == "filesystem" ]]; then
292			typeset mntpnt=$(get_prop mountpoint $ds)
293			if [[ -n $mntpnt ]]; then
294				rm -rf $mntpnt/$snap
295			fi
296		fi
297	done
298
299	return 0
300}
301
302#
303# Get all the sub-datasets of give dataset with specific suffix
304#
305# $1 Given dataset
306# $2 Suffix
307#
308function getds_with_suffix
309{
310	typeset ds=$1
311	typeset suffix=$2
312
313	zfs list -rHt all -o name $ds | grep "$suffix$"
314}
315
316#
317# Output inherited properties which is edited for file system
318#
319function fs_inherit_prop
320{
321	typeset fs_prop
322	if is_global_zone ; then
323		fs_prop=$(zfs inherit 2>&1 | \
324		    awk '$2=="YES" && $3=="YES" {print $1}')
325		if ! is_te_enabled ; then
326		        fs_prop=$(echo $fs_prop | grep -v "mlslabel")
327		fi
328	else
329		fs_prop=$(zfs inherit 2>&1 | \
330		    awk '$2=="YES" && $3=="YES" && !/devices|mlslabel|sharenfs|sharesmb|zoned/ {print $1}')
331	fi
332
333	echo $fs_prop
334}
335
336#
337# Output inherited properties for volume
338#
339function vol_inherit_prop
340{
341	echo checksum readonly
342}
343
344#
345# Get the destination dataset to compare
346#
347function get_dst_ds
348{
349	typeset srcfs=$1
350	typeset dstfs=$2
351
352	#
353	# If the srcfs is not pool
354	#
355	if ! zpool list $srcfs > /dev/null 2>&1 ; then
356		eval dstfs="$dstfs/${srcfs#*/}"
357	fi
358
359	echo $dstfs
360}
361
362#
363# Make test files
364#
365# $1 Number of files to create
366# $2 Maximum file size
367# $3 File ID offset
368# $4 File system to create the files on
369#
370function mk_files
371{
372	nfiles=$1
373	maxsize=$2
374	file_id_offset=$3
375	fs=$4
376	bs=512
377
378	for ((i=0; i<$nfiles; i=i+1)); do
379		file_name="/$fs/file-$maxsize-$((i+$file_id_offset))"
380		file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
381
382		#
383		# Create an interesting mix of files which contain both
384		# data blocks and holes for more realistic test coverage.
385		# Half the files are created as sparse then partially filled,
386		# the other half is dense then a hole is punched in the file.
387		#
388		if [ $((RANDOM % 2)) -eq 0 ]; then
389			truncate -s $file_size $file_name || \
390			    log_fail "Failed to create $file_name"
391			dd if=/dev/urandom of=$file_name \
392			    bs=$bs count=$(($file_size / 2 / $bs)) \
393			    seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
394			    conv=notrunc >/dev/null 2>&1 || \
395			    log_fail "Failed to create $file_name"
396		else
397			dd if=/dev/urandom of=$file_name \
398			    bs=$file_size count=1 >/dev/null 2>&1 || \
399			    log_fail "Failed to create $file_name"
400			dd if=/dev/zero of=$file_name \
401			    bs=$bs count=$(($file_size / 2 / $bs)) \
402			    seek=$(($RANDOM % (($file_size / 2 / $bs) + 1))) \
403			    conv=notrunc >/dev/null 2>&1 || \
404			    log_fail "Failed to create $file_name"
405		fi
406	done
407	echo Created $nfiles files of random sizes up to $maxsize bytes
408}
409
410#
411# Remove test files
412#
413# $1 Number of files to remove
414# $2 Maximum file size
415# $3 File ID offset
416# $4 File system to remove the files from
417#
418function rm_files
419{
420	nfiles=$1
421	maxsize=$2
422	file_id_offset=$3
423	fs=$4
424
425	for ((i=0; i<$nfiles; i=i+1)); do
426		rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
427	done
428	echo Removed $nfiles files of random sizes up to $maxsize bytes
429}
430
431#
432# Simulate a random set of operations which could be reasonably expected
433# to occur on an average filesystem.
434#
435# $1 Number of files to modify
436# $2 Maximum file size
437# $3 File system to modify the file on
438# $4 Enabled xattrs (optional)
439#
440function churn_files
441{
442	nfiles=$1
443	maxsize=$2
444	fs=$3
445	xattrs=${4:-1}
446
447	#
448	# Remove roughly half of the files in order to make it more
449	# likely that a dnode will be reallocated.
450	#
451	for ((i=0; i<$nfiles; i=i+1)); do
452		file_name="/$fs/file-$i"
453
454		if [[ -e $file_name ]]; then
455			if [ $((RANDOM % 2)) -eq 0 ]; then
456				rm $file_name || \
457				    log_fail "Failed to remove $file_name"
458			fi
459		fi
460	done
461
462	#
463	# Remount the filesystem to simulate normal usage.  This resets
464	# the last allocated object id allowing for new objects to be
465	# reallocated in the locations of previously freed objects.
466	#
467	log_must zfs unmount $fs
468	log_must zfs mount $fs
469
470	for i in {0..$nfiles}; do
471		file_name="/$fs/file-$i"
472		file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
473
474		#
475		# When the file exists modify it in one of five ways to
476		# simulate normal usage:
477		# - (20%) Remove and set and extended attribute on the file
478		# - (20%) Overwrite the existing file
479		# - (20%) Truncate the existing file to a random length
480		# - (20%) Truncate the existing file to zero length
481		# - (20%) Remove the file
482		#
483		# Otherwise create the missing file.  20% of the created
484		# files will be small and use embedded block pointers, the
485		# remainder with have random sizes up to the maximum size.
486		# Three extended attributes are attached to all of the files.
487		#
488		if [[ -e $file_name ]]; then
489			value=$((RANDOM % 5))
490			if [ $value -eq 0 -a $xattrs -ne 0 ]; then
491				attrname="testattr$((RANDOM % 3))"
492				attrlen="$(((RANDOM % 1000) + 1))"
493				attrvalue="$(random_string VALID_NAME_CHAR \
494				    $attrlen)"
495				rm_xattr $attrname $file_name || \
496				    log_fail "Failed to remove $attrname"
497				set_xattr $attrname "$attrvalue" $file_name || \
498				    log_fail "Failed to set $attrname"
499			elif [ $value -eq 1 ]; then
500				dd if=/dev/urandom of=$file_name \
501				    bs=$file_size count=1 >/dev/null 2>&1 || \
502				    log_fail "Failed to overwrite $file_name"
503			elif [ $value -eq 2 ]; then
504				truncate -s $file_size $file_name || \
505				    log_fail "Failed to truncate $file_name"
506			elif [ $value -eq 3 ]; then
507				truncate -s 0 $file_name || \
508				    log_fail "Failed to truncate $file_name"
509			else
510				rm $file_name || \
511				    log_fail "Failed to remove $file_name"
512			fi
513		else
514			if [ $((RANDOM % 5)) -eq 0 ]; then
515				file_size=$((($RANDOM % 64) + 1))
516			fi
517
518			dd if=/dev/urandom of=$file_name \
519			    bs=$file_size count=1 >/dev/null 2>&1 || \
520			    log_fail "Failed to create $file_name"
521
522			if [ $xattrs -ne 0 ]; then
523				for j in {0..2}; do
524					attrname="testattr$j"
525					attrlen="$(((RANDOM % 1000) + 1))"
526					attrvalue="$(random_string \
527					    VALID_NAME_CHAR $attrlen)"
528					set_xattr $attrname \
529					    "$attrvalue" $file_name || \
530					    log_fail "Failed to set $attrname"
531				done
532			fi
533		fi
534	done
535
536	return 0
537}
538
539#
540# Mess up a send file's contents
541#
542# $1 The send file path
543#
544function mess_send_file
545{
546	typeset -i minsize=2072
547	file=$1
548
549	filesize=$(stat_size $file)
550	if [ $filesize -lt $minsize ]; then
551		log_fail "Send file too small: $filesize < $minsize"
552	fi
553
554	# Truncate the send stream at a random offset after the DRR_BEGIN
555	# record (beyond 2072 bytes), any smaller than this and the receiving
556	# system won't have enough info to create the partial dataset at all.
557	# We use zstream dump to verify there is an intact DRR_BEGIN record.
558	offset=$(((($RANDOM * $RANDOM) % ($filesize - $minsize)) + $minsize))
559	nr_begins=$(head -c $offset $file | zstream dump | \
560	    awk '/DRR_BEGIN/ { print $5 }')
561	log_must [ "$nr_begins" -eq 1 ]
562
563	if (($RANDOM % 7 <= 1)); then
564		#
565		# We corrupt 2 bytes to minimize the chance that we
566		# write the same value that's already there.
567		#
568		log_must eval "dd if=/dev/urandom of=$file conv=notrunc " \
569		    "bs=1 count=2 seek=$offset >/dev/null 2>&1"
570	else
571		log_must truncate -s $offset $file
572	fi
573}
574
575#
576# Diff the send/receive filesystems
577#
578# $1 The sent filesystem
579# $2 The received filesystem
580#
581function file_check
582{
583	sendfs=$1
584	recvfs=$2
585
586	if [[ -d /$recvfs/.zfs/snapshot/a && -d \
587	    /$sendfs/.zfs/snapshot/a ]]; then
588		log_must directory_diff /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
589	fi
590	if [[ -d /$recvfs/.zfs/snapshot/b && -d \
591	    /$sendfs/.zfs/snapshot/b ]]; then
592		log_must directory_diff /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
593	fi
594}
595
596#
597# Resume test helper
598#
599# $1 The ZFS send command
600# $2 The filesystem where the streams are sent
601# $3 The receive filesystem
602# $4 Test dry-run (optional)
603#
604function resume_test
605{
606	typeset sendcmd=$1
607	typeset streamfs=$2
608	typeset recvfs=$3
609	typeset dryrun=${4:-1}
610
611	stream_num=1
612	log_must eval "$sendcmd >/$streamfs/$stream_num"
613
614	for ((i=0; i<2; i=i+1)); do
615		mess_send_file /$streamfs/$stream_num
616		log_mustnot eval "zfs recv -suv $recvfs </$streamfs/$stream_num"
617		stream_num=$((stream_num+1))
618
619		token=$(get_prop receive_resume_token $recvfs)
620
621		# Do a dry-run
622		[ $dryrun -ne 0 ] && \
623			log_must eval "zfs send -nvt $token > /dev/null"
624
625		log_must eval "zfs send -t $token  >/$streamfs/$stream_num"
626	done
627	log_must eval "zfs recv -suv $recvfs </$streamfs/$stream_num"
628}
629
630function get_resume_token
631{
632	sendcmd=$1
633	streamfs=$2
634	recvfs=$3
635
636	log_must eval "$sendcmd > /$streamfs/1"
637	mess_send_file /$streamfs/1
638	log_mustnot eval "zfs recv -suv $recvfs < /$streamfs/1 2>&1"
639	get_prop receive_resume_token $recvfs > /$streamfs/resume_token
640}
641
642#
643# Setup filesystems for the resumable send/receive tests
644#
645# $1 The "send" filesystem
646# $2 The "recv" filesystem
647#
648function test_fs_setup
649{
650	typeset sendfs=$1
651	typeset recvfs=$2
652	typeset streamfs=$3
653	typeset sendpool=${sendfs%%/*}
654	typeset recvpool=${recvfs%%/*}
655
656	datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
657	datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
658	datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
659
660	if datasetexists $sendfs || zfs create -o compress=lz4 $sendfs; then
661		mk_files 1000 256 0 $sendfs &
662		mk_files 1000 131072 0 $sendfs &
663		mk_files 100 1048576 0 $sendfs &
664		mk_files 10 10485760 0 $sendfs &
665		mk_files 1 104857600 0 $sendfs &
666		log_must wait
667		log_must zfs snapshot $sendfs@a
668
669		rm_files 200 256 0 $sendfs &
670		rm_files 200 131072 0 $sendfs &
671		rm_files 20 1048576 0 $sendfs &
672		rm_files 2 10485760 0 $sendfs &
673		log_must wait
674
675		mk_files 400 256 0 $sendfs &
676		mk_files 400 131072 0 $sendfs &
677		mk_files 40 1048576 0 $sendfs &
678		mk_files 4 10485760 0 $sendfs &
679		log_must wait
680
681		log_must zfs snapshot $sendfs@b
682		log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
683		log_must eval "zfs send -v -i @a $sendfs@b " \
684		    ">/$sendpool/incremental.zsend"
685	fi
686
687	log_must zfs create -o compress=lz4 $streamfs
688}
689
690#
691# Check to see if the specified features are set in a send stream.
692# The values for these features are found in include/sys/zfs_ioctl.h
693#
694# $1 The stream file
695# $2-$n The flags expected in the stream
696#
697function stream_has_features
698{
699	typeset file=$1
700	shift
701
702	[[ -f $file ]] || log_fail "Couldn't find file: $file"
703	typeset flags=$(zstream dump $file | \
704	    awk '/features =/ {features = $3} END {print features}')
705	typeset -A feature
706	feature[dedup]="1"
707	feature[dedupprops]="2"
708	feature[sa_spill]="4"
709	feature[embed_data]="10000"
710	feature[lz4]="20000"
711	feature[mooch_byteswap]="40000"
712	feature[large_blocks]="80000"
713	feature[resuming]="100000"
714	feature[redacted]="200000"
715	feature[compressed]="400000"
716	feature[longname]="10000000"
717
718	typeset flag known derived=0
719	for flag in "$@"; do
720		known=${feature[$flag]}
721		[[ -z $known ]] && log_fail "Unknown feature: $flag"
722
723		derived=$(printf "%x" $((0x${flags} & 0x${feature[$flag]})))
724		[[ $derived = $known ]] || return 1
725	done
726
727	return 0
728}
729
730#
731# Given a send stream, verify that the size of the stream matches what's
732# expected based on the source or target dataset. If the stream is an
733# incremental stream, subtract the size of the source snapshot before
734# comparing. This function does not currently handle incremental streams
735# that remove data.
736#
737# $1 The zstream dump output file
738# $2 The dataset to compare against
739#    This can be a source of a send or recv target (fs, not snapshot)
740# $3 The percentage below which verification is deemed a failure
741# $4 The source snapshot of an incremental send
742#
743
744function verify_stream_size
745{
746	typeset stream=$1
747	typeset ds=$2
748	typeset percent=${3:-90}
749	typeset inc_src=$4
750
751	[[ -f $stream ]] || log_fail "No such file: $stream"
752	datasetexists $ds || log_fail "No such dataset: $ds"
753
754	typeset stream_size=$(zstream dump $stream | sed -n \
755	    's/	Total payload size = \(.*\) (0x.*)/\1/p')
756
757	typeset inc_size=0
758	if [[ -n $inc_src ]]; then
759		inc_size=$(get_prop lrefer $inc_src)
760		if stream_has_features $stream compressed; then
761			inc_size=$(get_prop refer $inc_src)
762		fi
763	fi
764
765	if stream_has_features $stream compressed; then
766		ds_size=$(get_prop refer $ds)
767	else
768		ds_size=$(get_prop lrefer $ds)
769	fi
770	ds_size=$((ds_size - inc_size))
771
772	log_must within_percent $stream_size $ds_size $percent
773}
774
775# Cleanup function for tests involving resumable send
776function resume_cleanup
777{
778	typeset sendfs=$1
779	typeset streamfs=$2
780	typeset sendpool=${sendfs%%/*}
781
782	datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
783	datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
784	cleanup_pool $POOL2
785	rm -f /$sendpool/initial.zsend /$sendpool/incremental.zsend
786}
787
788# Randomly set the property to one of the enumerated values.
789function rand_set_prop
790{
791	typeset dtst=$1
792	typeset prop=$2
793	shift 2
794	typeset value=$(random_get $@)
795
796	log_must eval "zfs set $prop='$value' $dtst"
797}
798
799# Generate a recursive checksum of a filesystem which includes the file
800# contents and any associated extended attributes.
801function recursive_cksum
802{
803	case "$(uname)" in
804	FreeBSD)
805		find $1 -type f -exec sh -c 'xxh128sum {}; \
806		    lsextattr -q system {} | xxh128sum; \
807		    lsextattr -q user {} | xxh128sum' \; \
808		    | sort -k 2 | awk '{ print $1 }' | xxh128digest
809		;;
810	*)
811		find $1 -type f -exec sh -c 'xxh128sum {}; getfattr \
812		    --absolute-names --only-values -d {} | xxh128sum' \
813		    \; | sort -k 2 | awk '{ print $1 }' | xxh128digest
814		;;
815	esac
816}
817