xref: /illumos-gate/usr/src/test/zfs-tests/tests/functional/rsend/rsend.kshlib (revision e86372a01d2d16a5dd4a64e144ed978ba17fe7dd)
1#
2# CDDL HEADER START
3#
4# The contents of this file are subject to the terms of the
5# Common Development and Distribution License (the "License").
6# You may not use this file except in compliance with the License.
7#
8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9# or http://www.opensolaris.org/os/licensing.
10# See the License for the specific language governing permissions
11# and limitations under the License.
12#
13# When distributing Covered Code, include this CDDL HEADER in each
14# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15# If applicable, add the following below this CDDL HEADER, with the
16# fields enclosed by brackets "[]" replaced with your own identifying
17# information: Portions Copyright [yyyy] [name of copyright owner]
18#
19# CDDL HEADER END
20#
21
22#
23# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24# Use is subject to license terms.
25#
26
27#
28# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
29#
30
31. $STF_SUITE/include/libtest.shlib
32. $STF_SUITE/include/math.shlib
33. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib
34. $STF_SUITE/tests/functional/rsend/rsend.cfg
35
36#
37# Set up test model which includes various datasets
38#
39#               @final
40#               @snapB
41#               @init
42#               |
43#   ______ pclone
44#  |      /
45#  |@psnap
46#  ||                         @final
47#  ||@final       @final      @snapC
48#  ||@snapC       @snapC      @snapB
49#  ||@snapA       @snapB      @snapA
50#  ||@init        @init       @init
51#  |||            |           |
52# $pool -------- $FS ------- fs1 ------- fs2
53#    \             \\_____     \          |
54#     vol           vol   \____ \         @fsnap
55#      |              |        \ \              \
56#      @init          @vsnap   |  ------------ fclone
57#      @snapA         @init \  |                    |
58#      @final         @snapB \ |                    @init
59#                     @snapC  vclone                @snapA
60#                     @final       |                @final
61#                                 @init
62#                                 @snapC
63#                                 @final
64#
65# $1 pool name
66#
67function setup_test_model
68{
69	typeset pool=$1
70
71	log_must zfs create -p $pool/$FS/fs1/fs2
72
73	log_must zfs snapshot $pool@psnap
74	log_must zfs clone $pool@psnap $pool/pclone
75
76	if is_global_zone ; then
77		log_must zfs create -V 16M $pool/vol
78		log_must zfs create -V 16M $pool/$FS/vol
79
80		log_must zfs snapshot $pool/$FS/vol@vsnap
81		log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
82	fi
83
84	log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
85	log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
86	log_must zfs snapshot -r $pool@init
87
88	log_must snapshot_tree $pool@snapA
89	log_must snapshot_tree $pool@snapC
90	log_must snapshot_tree $pool/pclone@snapB
91	log_must snapshot_tree $pool/$FS@snapB
92	log_must snapshot_tree $pool/$FS@snapC
93	log_must snapshot_tree $pool/$FS/fs1@snapA
94	log_must snapshot_tree $pool/$FS/fs1@snapB
95	log_must snapshot_tree $pool/$FS/fs1@snapC
96	log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
97
98	if is_global_zone ; then
99		log_must zfs snapshot $pool/vol@snapA
100		log_must zfs snapshot $pool/$FS/vol@snapB
101		log_must zfs snapshot $pool/$FS/vol@snapC
102		log_must zfs snapshot $pool/$FS/vclone@snapC
103	fi
104
105	log_must zfs snapshot -r $pool@final
106
107	return 0
108}
109
110#
111# Cleanup the BACKDIR and given pool content and all the sub datasets
112#
113# $1 pool name
114#
115function cleanup_pool
116{
117	typeset pool=$1
118	log_must rm -rf $BACKDIR/*
119
120	if is_global_zone ; then
121		log_must zfs destroy -Rf $pool
122	else
123		typeset list=$(zfs list -H -r -t all -o name $pool)
124		for ds in $list ; do
125			if [[ $ds != $pool ]] ; then
126				if datasetexists $ds ; then
127					log_must zfs destroy -Rf $ds
128				fi
129			fi
130		done
131	fi
132
133	typeset mntpnt=$(get_prop mountpoint $pool)
134	if ! ismounted $pool ; then
135		# Make sure mountpoint directory is empty
136		if [[ -d $mntpnt ]]; then
137			log_must rm -rf $mntpnt/*
138		fi
139
140		log_must zfs mount $pool
141	fi
142	if [[ -d $mntpnt ]]; then
143		rm -rf $mntpnt/*
144	fi
145
146	return 0
147}
148
149function cleanup_pools
150{
151	cleanup_pool $POOL2
152	destroy_pool $POOL3
153}
154
155#
156# Detect if the given two filesystems have same sub-datasets
157#
158# $1 source filesystem
159# $2 destination filesystem
160#
161function cmp_ds_subs
162{
163	typeset src_fs=$1
164	typeset dst_fs=$2
165
166	zfs list -r -H -t all -o name $src_fs > $BACKDIR/src1
167	zfs list -r -H -t all -o name $dst_fs > $BACKDIR/dst1
168
169	eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src
170	eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst
171
172	diff $BACKDIR/src $BACKDIR/dst
173	typeset -i ret=$?
174
175	rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1
176
177	return $ret
178}
179
180#
181# Compare all the directores and files in two filesystems
182#
183# $1 source filesystem
184# $2 destination filesystem
185#
186function cmp_ds_cont
187{
188	typeset src_fs=$1
189	typeset dst_fs=$2
190
191	typeset srcdir dstdir
192	srcdir=$(get_prop mountpoint $src_fs)
193	dstdir=$(get_prop mountpoint $dst_fs)
194
195	diff -r $srcdir $dstdir > /dev/null 2>&1
196	return $?
197}
198
199#
200# Compare the given two dataset properties
201#
202# $1 dataset 1
203# $2 dataset 2
204#
205function cmp_ds_prop
206{
207	typeset dtst1=$1
208	typeset dtst2=$2
209
210	for item in "type" "origin" "volblocksize" "aclinherit" "aclmode" \
211	    "atime" "canmount" "checksum" "compression" "copies" "devices" \
212	    "dnodesize" "exec" "quota" "readonly" "recordsize" "reservation" \
213	    "setuid" "sharenfs" "snapdir" "version" "volsize" "xattr" "zoned" \
214	    "mountpoint";
215	do
216		zfs get -H -o property,value,source $item $dtst1 >> \
217		    $BACKDIR/dtst1
218		zfs get -H -o property,value,source $item $dtst2 >> \
219		    $BACKDIR/dtst2
220	done
221
222	eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1
223	eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2
224
225	diff $BACKDIR/dtst1 $BACKDIR/dtst2
226	typeset -i ret=$?
227
228	rm -f $BACKDIR/dtst1 $BACKDIR/dtst2
229
230	return $ret
231
232}
233
234#
235# Random create directories and files
236#
237# $1 directory
238#
239function random_tree
240{
241	typeset dir=$1
242
243	if [[ -d $dir ]]; then
244		rm -rf $dir
245	fi
246	mkdir -p $dir
247	typeset -i ret=$?
248
249	typeset -i nl nd nf
250	((nl = RANDOM % 6 + 1))
251	((nd = RANDOM % 3 ))
252	((nf = RANDOM % 5 ))
253	mktree -b $dir -l $nl -d $nd -f $nf
254	((ret |= $?))
255
256	return $ret
257}
258
259#
260# Put data in filesystem and take snapshot
261#
262# $1 snapshot name
263#
264function snapshot_tree
265{
266	typeset snap=$1
267	typeset ds=${snap%%@*}
268	typeset type=$(get_prop "type" $ds)
269
270	typeset -i ret=0
271	if [[ $type == "filesystem" ]]; then
272		typeset mntpnt=$(get_prop mountpoint $ds)
273		((ret |= $?))
274
275		if ((ret == 0)) ; then
276			eval random_tree $mntpnt/${snap##$ds}
277			((ret |= $?))
278		fi
279	fi
280
281	if ((ret == 0)) ; then
282		zfs snapshot $snap
283		((ret |= $?))
284	fi
285
286	return $ret
287}
288
289#
290# Destroy the given snapshot and stuff
291#
292# $1 snapshot
293#
294function destroy_tree
295{
296	typeset -i ret=0
297	typeset snap
298	for snap in "$@" ; do
299		zfs destroy $snap
300		ret=$?
301
302		typeset ds=${snap%%@*}
303		typeset type=$(get_prop "type" $ds)
304		if [[ $type == "filesystem" ]]; then
305			typeset mntpnt=$(get_prop mountpoint $ds)
306			((ret |= $?))
307
308			if ((ret != 0)); then
309				rm -r $mntpnt/$snap
310				((ret |= $?))
311			fi
312		fi
313
314		if ((ret != 0)); then
315			return $ret
316		fi
317	done
318
319	return 0
320}
321
322#
323# Get all the sub-datasets of give dataset with specific suffix
324#
325# $1 Given dataset
326# $2 Suffix
327#
328function getds_with_suffix
329{
330	typeset ds=$1
331	typeset suffix=$2
332
333	typeset list=$(zfs list -r -H -t all -o name $ds | grep "$suffix$")
334
335	echo $list
336}
337
338#
339# Output inherited properties whitch is edited for file system
340#
341function fs_inherit_prop
342{
343	typeset fs_prop
344	if is_global_zone ; then
345		fs_prop=$(zfs inherit 2>&1 | \
346		    awk '$2=="YES" && $3=="YES" {print $1}')
347		if ! is_te_enabled ; then
348		        fs_prop=$(echo $fs_prop | grep -v "mlslabel")
349		fi
350	else
351		fs_prop=$(zfs inherit 2>&1 | \
352		    awk '$2=="YES" && $3=="YES" {print $1}'|
353		    egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned")
354	fi
355
356	echo $fs_prop
357}
358
359#
360# Output inherited properties for volume
361#
362function vol_inherit_prop
363{
364	echo "checksum readonly"
365}
366
367#
368# Get the destination dataset to compare
369#
370function get_dst_ds
371{
372	typeset srcfs=$1
373	typeset dstfs=$2
374
375	#
376	# If the srcfs is not pool
377	#
378	if ! zpool list $srcfs > /dev/null 2>&1 ; then
379		eval dstfs="$dstfs/${srcfs#*/}"
380	fi
381
382	echo $dstfs
383}
384
385#
386# Make test files
387#
388# $1 Number of files to create
389# $2 Maximum file size
390# $3 File ID offset
391# $4 File system to create the files on
392#
393function mk_files
394{
395	nfiles=$1
396	maxsize=$2
397	file_id_offset=$3
398	fs=$4
399
400	for ((i=0; i<$nfiles; i=i+1)); do
401		dd if=/dev/urandom \
402		    of=/$fs/file-$maxsize-$((i+$file_id_offset)) \
403		    bs=$(($RANDOM * $RANDOM % $maxsize)) \
404		    count=1 >/dev/null 2>&1 || log_fail \
405		    "Failed to create /$fs/file-$maxsize-$((i+$file_id_offset))"
406	done
407	echo Created $nfiles files of random sizes up to $maxsize bytes
408}
409
410#
411# Remove test files
412#
413# $1 Number of files to remove
414# $2 Maximum file size
415# $3 File ID offset
416# $4 File system to remove the files from
417#
418function rm_files
419{
420	nfiles=$1
421	maxsize=$2
422	file_id_offset=$3
423	fs=$4
424
425	for ((i=0; i<$nfiles; i=i+1)); do
426		rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
427	done
428	echo Removed $nfiles files of random sizes up to $maxsize bytes
429}
430
431#
432# Simulate a random set of operations which could be reasonably expected
433# to occur on an average filesystem.
434#
435# $1 Number of files to modify
436# $2 Maximum file size
437# $3 File system to modify the file on
438# $4 Enabled xattrs (optional)
439#
440function churn_files
441{
442        nfiles=$1
443        maxsize=$2
444        fs=$3
445        xattrs=${4:-1}
446
447        #
448        # Remove roughly half of the files in order to make it more
449        # likely that a dnode will be reallocated.
450        #
451        for ((i=0; i<$nfiles; i=i+1)); do
452                file_name="/$fs/file-$i"
453
454                if [[ -e $file_name ]]; then
455                        if [ $((RANDOM % 2)) -eq 0 ]; then
456                                rm $file_name || \
457                                    log_fail "Failed to remove $file_name"
458                        fi
459                fi
460        done
461
462        #
463        # Remount the filesystem to simulate normal usage.  This resets
464        # the last allocated object id allowing for new objects to be
465        # reallocated in the locations of previously freed objects.
466        #
467        log_must zfs unmount $fs
468        log_must zfs mount $fs
469
470        for i in {0..$nfiles}; do
471                file_name="/$fs/file-$i"
472                file_size=$((($RANDOM * $RANDOM % ($maxsize - 1)) + 1))
473
474                #
475                # When the file exists modify it in one of five ways to
476                # simulate normal usage:
477                # - (20%) Remove and set and extended attribute on the file
478                # - (20%) Overwrite the existing file
479                # - (20%) Truncate the existing file to a random length
480                # - (20%) Truncate the existing file to zero length
481                # - (20%) Remove the file
482                #
483                # Otherwise create the missing file.  20% of the created
484                # files will be small and use embedded block pointers, the
485                # remainder with have random sizes up to the maximum size.
486                # Three extended attributes are attached to all of the files.
487                #
488                if [[ -e $file_name ]]; then
489                        value=$((RANDOM % 5))
490                        if [ $value -eq 0 -a $xattrs -ne 0 ]; then
491                                attrname="testattr$((RANDOM % 3))"
492				attrlen="$(((RANDOM % 1000) + 1))"
493				attrvalue="$(random_string VALID_NAME_CHAR \
494				    $attrlen)"
495                                attr -qr $attrname $file_name || \
496                                    log_fail "Failed to remove $attrname"
497				attr -qs $attrname \
498				    -V "$attrvalue" $file_name || \
499                                    log_fail "Failed to set $attrname"
500                        elif [ $value -eq 1 ]; then
501                                dd if=/dev/urandom of=$file_name \
502                                    bs=$file_size count=1 >/dev/null 2>&1 || \
503                                    log_fail "Failed to overwrite $file_name"
504                        elif [ $value -eq 2 ]; then
505                                truncate -s $file_size $file_name || \
506                                    log_fail "Failed to truncate $file_name"
507                        elif [ $value -eq 3 ]; then
508                                truncate -s 0 $file_name || \
509                                    log_fail "Failed to truncate $file_name"
510                        else
511                                rm $file_name || \
512                                    log_fail "Failed to remove $file_name"
513                        fi
514                else
515                        if [ $((RANDOM % 5)) -eq 0 ]; then
516                                file_size=$((($RANDOM % 64) + 1))
517                        fi
518
519                        dd if=/dev/urandom of=$file_name \
520                            bs=$file_size count=1 >/dev/null 2>&1 || \
521                            log_fail "Failed to create $file_name"
522
523                        if [ $xattrs -ne 0 ]; then
524                                for j in {0..2}; do
525                                        attrname="testattr$j"
526					attrlen="$(((RANDOM % 1000) + 1))"
527					attrvalue="$(random_string \
528					    VALID_NAME_CHAR $attrlen)"
529					attr -qs $attrname \
530					    -V "$attrvalue" $file_name || \
531					    log_fail "Failed to set $attrname"
532                                done
533                        fi
534                fi
535        done
536
537        return 0
538}
539
540#
541# Mess up file contents
542#
543# $1 The file path
544#
545function mess_file
546{
547	file=$1
548
549	filesize=$(stat -c '%s' $file)
550	offset=$(($RANDOM * $RANDOM % $filesize))
551	if (($RANDOM % 7 <= 1)); then
552		#
553		# We corrupt 2 bytes to minimize the chance that we
554		# write the same value that's already there.
555		#
556		log_must eval "dd if=/dev/random of=$file conv=notrunc " \
557		    "bs=1 count=2 oseek=$offset >/dev/null 2>&1"
558	else
559		log_must truncate -s $offset $file
560	fi
561}
562
563#
564# Diff the send/receive filesystems
565#
566# $1 The sent filesystem
567# $2 The received filesystem
568#
569function file_check
570{
571	sendfs=$1
572	recvfs=$2
573
574	if [[ -d /$recvfs/.zfs/snapshot/a && -d \
575	    /$sendfs/.zfs/snapshot/a ]]; then
576		diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
577		[[ $? -eq 0 ]] || log_fail "Differences found in snap a"
578	fi
579	if [[ -d /$recvfs/.zfs/snapshot/b && -d \
580	    /$sendfs/.zfs/snapshot/b ]]; then
581		diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
582		[[ $? -eq 0 ]] || log_fail "Differences found in snap b"
583	fi
584}
585
586#
587# Resume test helper
588#
589# $1 The ZFS send command
590# $2 The filesystem where the streams are sent
591# $3 The receive filesystem
592#
593function resume_test
594{
595	sendcmd=$1
596	streamfs=$2
597	recvfs=$3
598
599	stream_num=1
600	log_must eval "$sendcmd >/$streamfs/$stream_num"
601
602	for ((i=0; i<2; i=i+1)); do
603		mess_file /$streamfs/$stream_num
604		log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num
605		stream_num=$((stream_num+1))
606
607		token=$(zfs get -Hp -o value receive_resume_token $recvfs)
608		log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
609		[[ -f /$streamfs/$stream_num ]] || \
610		    log_fail "NO FILE /$streamfs/$stream_num"
611	done
612	log_must zfs recv -suv $recvfs </$streamfs/$stream_num
613}
614
615#
616# Setup filesystems for the resumable send/receive tests
617#
618# $1 The "send" filesystem
619# $2 The "recv" filesystem
620#
621function test_fs_setup
622{
623	typeset sendfs=$1
624	typeset recvfs=$2
625	typeset streamfs=$3
626	typeset sendpool=${sendfs%%/*}
627	typeset recvpool=${recvfs%%/*}
628
629	datasetexists $sendfs && log_must zfs destroy -r $sendpool
630	datasetexists $recvfs && log_must zfs destroy -r $recvpool
631	datasetexists $streamfs && log_must zfs destroy -r $streamfs
632
633	if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
634		mk_files 1000 256 0 $sendfs &
635		mk_files 1000 131072 0 $sendfs &
636		mk_files 100 1048576 0 $sendfs &
637		mk_files 10 10485760 0 $sendfs &
638		mk_files 1 104857600 0 $sendfs &
639		log_must wait
640		log_must zfs snapshot $sendfs@a
641
642		rm_files 200 256 0 $sendfs &
643		rm_files 200 131072 0 $sendfs &
644		rm_files 20 1048576 0 $sendfs &
645		rm_files 2 10485760 0 $sendfs &
646		log_must wait
647
648		mk_files 400 256 0 $sendfs &
649		mk_files 400 131072 0 $sendfs &
650		mk_files 40 1048576 0 $sendfs &
651		mk_files 4 10485760 0 $sendfs &
652		log_must wait
653
654		log_must zfs snapshot $sendfs@b
655		log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
656		log_must eval "zfs send -v -i @a $sendfs@b " \
657		    ">/$sendpool/incremental.zsend"
658	fi
659
660	log_must zfs create -o compress=lz4 $streamfs
661}
662
663#
664# Check to see if the specified features are set in a send stream.
665# The values for these features are found in uts/common/fs/zfs/sys/zfs_ioctl.h
666#
667# $1 The stream file
668# $2-$n The flags expected in the stream
669#
670function stream_has_features
671{
672	typeset file=$1
673	shift
674
675	[[ -f $file ]] || log_fail "Couldn't find file: $file"
676	typeset flags=$(cat $file | zstreamdump | awk '/features =/ {print $3}')
677	typeset -A feature
678	feature[dedup]="1"
679	feature[dedupprops]="2"
680	feature[sa_spill]="4"
681	feature[embed_data]="10000"
682	feature[lz4]="20000"
683	feature[mooch_byteswap]="40000"
684	feature[large_blocks]="80000"
685	feature[resuming]="100000"
686	feature[redacted]="200000"
687	feature[compressed]="400000"
688
689	typeset flag known derived=0
690	for flag in "$@"; do
691		known=${feature[$flag]}
692		[[ -z $known ]] && log_fail "Unknown feature: $flag"
693
694		derived=$(echo "$flags & ${feature[$flag]} = X" | mdb | sed 's/ //g')
695		[[ $derived = $known ]] || return 1
696	done
697
698	return 0
699}
700
701#
702# Parse zstreamdump -v output.  The output varies for each kind of record:
703# BEGIN records are simply output as "BEGIN"
704# END records are output as "END"
705# OBJECT records become "OBJECT <object num>"
706# FREEOBJECTS records become "FREEOBJECTS <startobj> <numobjs>"
707# FREE records become "<record type> <start> <length>"
708# WRITE records become:
709# "<record type> <compression type> <start> <logical size> <compressed size>
710#  <data size>"
711#
712function parse_dump
713{
714	sed '/^WRITE/{N;s/\n/ /;}' | grep "^[A-Z]" | awk '{
715	    if ($1 == "BEGIN" || $1 == "END") print $1
716	    if ($1 == "OBJECT") print $1" "$4
717	    if ($1 == "FREEOBJECTS") print $1" "$4" "$7
718	    if ($1 == "FREE") print $1" "$7" "$10
719	    if ($1 == "WRITE") print $1" "$15" "$21" "$24" "$27" "$30}'
720}
721
722#
723# Given a send stream, verify that the size of the stream matches what's
724# expected based on the source or target dataset. If the stream is an
725# incremental stream, subtract the size of the source snapshot before
726# comparing. This function does not currently handle incremental streams
727# that remove data.
728#
729# $1 The zstreamdump output file
730# $2 The dataset to compare against
731#    This can be a source of a send or recv target (fs, not snapshot)
732# $3 The percentage below which verification is deemed a failure
733# $4 The source snapshot of an incremental send
734#
735
736function verify_stream_size
737{
738	typeset stream=$1
739	typeset ds=$2
740	typeset percent=${3:-90}
741	typeset inc_src=$4
742
743	[[ -f $stream ]] || log_fail "No such file: $stream"
744	datasetexists $ds || log_fail "No such dataset: $ds"
745
746	typeset stream_size=$(cat $stream | zstreamdump | sed -n \
747	    's/	Total write size = \(.*\) (0x.*)/\1/p')
748
749	typeset inc_size=0
750	if [[ -n $inc_src ]]; then
751		inc_size=$(get_prop lrefer $inc_src)
752		if stream_has_features $stream compressed; then
753			inc_size=$(get_prop refer $inc_src)
754		fi
755	fi
756
757	if stream_has_features $stream compressed; then
758		ds_size=$(get_prop refer $ds)
759	else
760		ds_size=$(get_prop lrefer $ds)
761	fi
762	ds_size=$((ds_size - inc_size))
763
764	within_percent $stream_size $ds_size $percent || log_fail \
765	    "$stream_size $ds_size differed by too much"
766}
767
768# Cleanup function for tests involving resumable send
769function resume_cleanup
770{
771	typeset sendfs=$1
772	typeset streamfs=$2
773	typeset sendpool=${sendfs%%/*}
774
775	datasetexists $sendfs && log_must zfs destroy -r $sendfs
776	datasetexists $streamfs && log_must zfs destroy -r $streamfs
777	cleanup_pool $POOL2
778	rm -f /$sendpool/initial.zsend /$sendpool/incremental.zsend
779}
780
781# Randomly set the property to one of the enumerated values.
782function rand_set_prop
783{
784	typeset dtst=$1
785	typeset prop=$2
786	shift 2
787	typeset value=$(random_get $@)
788
789	log_must eval "zfs set $prop='$value' $dtst"
790}
791
792# Generate a recursive checksum of a filesystems contents.  Only file
793# data is included in the checksum (no meta data, or xattrs).
794function recursive_cksum
795{
796	find $1 -type f -exec sha256sum {} \; | \
797	    sort -k 2 | awk '{ print $1 }' | sha256sum
798}
799