xref: /freebsd/sys/contrib/openzfs/scripts/zfs-tests.sh (revision a90b9d0159070121c221b966469c3e36d912bf82)
1#!/bin/sh
2# shellcheck disable=SC2154
3#
4# CDDL HEADER START
5#
6# The contents of this file are subject to the terms of the
7# Common Development and Distribution License, Version 1.0 only
8# (the "License").  You may not use this file except in compliance
9# with the License.
10#
11# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
12# or https://opensource.org/licenses/CDDL-1.0.
13# See the License for the specific language governing permissions
14# and limitations under the License.
15#
16# When distributing Covered Code, include this CDDL HEADER in each
17# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
18# If applicable, add the following below this CDDL HEADER, with the
19# fields enclosed by brackets "[]" replaced with your own identifying
20# information: Portions Copyright [yyyy] [name of copyright owner]
21#
22# CDDL HEADER END
23#
24
25#
26# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
27#
28
29SCRIPT_COMMON=${SCRIPT_COMMON:-${0%/*}/common.sh}
30. "${SCRIPT_COMMON}" || exit
31
32PROG=zfs-tests.sh
33VERBOSE="no"
34QUIET=""
35DEBUG=""
36CLEANUP="yes"
37CLEANUPALL="no"
38KMSG=""
39LOOPBACK="yes"
40STACK_TRACER="no"
41FILESIZE="4G"
42DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
43RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
44FILEDIR=${FILEDIR:-/var/tmp}
45DISKS=${DISKS:-""}
46SINGLETEST=""
47SINGLETESTUSER="root"
48TAGS=""
49ITERATIONS=1
50ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
51ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
52UNAME=$(uname)
53RERUN=""
54KMEMLEAK=""
55
56# Override some defaults if on FreeBSD
57if [ "$UNAME" = "FreeBSD" ] ; then
58	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
59	LOSETUP=/sbin/mdconfig
60	DMSETUP=/sbin/gpart
61else
62	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
63	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
64	LOSETUP=${LOSETUP:-/sbin/losetup}
65	DMSETUP=${DMSETUP:-/sbin/dmsetup}
66fi
67
68#
69# Log an informational message when additional verbosity is enabled.
70#
71msg() {
72	if [ "$VERBOSE" = "yes" ]; then
73		echo "$@"
74	fi
75}
76
77#
78# Log a failure message, cleanup, and return an error.
79#
80fail() {
81	echo "$PROG: $1" >&2
82	cleanup
83	exit 1
84}
85
86cleanup_freebsd_loopback() {
87	for TEST_LOOPBACK in ${LOOPBACKS}; do
88		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
89			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
90			    echo "Failed to destroy: ${TEST_LOOPBACK}"
91		fi
92	done
93}
94
95cleanup_linux_loopback() {
96	for TEST_LOOPBACK in ${LOOPBACKS}; do
97		LOOP_DEV="${TEST_LOOPBACK##*/}"
98		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
99		    awk -v l="${LOOP_DEV}" '$0 ~ l {print $1}')
100
101		if [ -n "$DM_DEV" ]; then
102			sudo "${DMSETUP}" remove "${DM_DEV}" ||
103			    echo "Failed to remove: ${DM_DEV}"
104		fi
105
106		if [ -n "${TEST_LOOPBACK}" ]; then
107			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
108			    echo "Failed to remove: ${TEST_LOOPBACK}"
109		fi
110	done
111}
112
113#
114# Attempt to remove loopback devices and files which where created earlier
115# by this script to run the test framework.  The '-k' option may be passed
116# to the script to suppress cleanup for debugging purposes.
117#
118cleanup() {
119	if [ "$CLEANUP" = "no" ]; then
120		return 0
121	fi
122
123
124	if [ "$LOOPBACK" = "yes" ]; then
125		if [ "$UNAME" = "FreeBSD" ] ; then
126			cleanup_freebsd_loopback
127		else
128			cleanup_linux_loopback
129		fi
130	fi
131
132	# shellcheck disable=SC2086
133	rm -f ${FILES} >/dev/null 2>&1
134
135	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
136		rm -Rf "$STF_PATH"
137	fi
138}
139trap cleanup EXIT
140
141#
142# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
143# loopback devices, and files.  This is a useful way to cleanup a previous
144# test run failure which has left the system in an unknown state.  This can
145# be dangerous and should only be used in a dedicated test environment.
146#
147cleanup_all() {
148	TEST_POOLS=$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | grep testpool)
149	if [ "$UNAME" = "FreeBSD" ] ; then
150		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
151	else
152		TEST_LOOPBACKS=$("${LOSETUP}" -a | awk -F: '/file-vdev/ {print $1}')
153	fi
154	TEST_FILES=$(ls "${FILEDIR}"/file-vdev* /var/tmp/file-vdev* 2>/dev/null)
155
156	msg
157	msg "--- Cleanup ---"
158	# shellcheck disable=2116,2086
159	msg "Removing pool(s):     $(echo ${TEST_POOLS})"
160	for TEST_POOL in $TEST_POOLS; do
161		sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
162	done
163
164	if [ "$UNAME" != "FreeBSD" ] ; then
165		msg "Removing all dm(s):   $(sudo "${DMSETUP}" ls |
166		    grep loop | tr '\n' ' ')"
167		sudo "${DMSETUP}" remove_all
168	fi
169
170	# shellcheck disable=2116,2086
171	msg "Removing loopback(s): $(echo ${TEST_LOOPBACKS})"
172	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
173		if [ "$UNAME" = "FreeBSD" ] ; then
174			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
175		else
176			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
177		fi
178	done
179
180	# shellcheck disable=2116,2086
181	msg "Removing files(s):    $(echo ${TEST_FILES})"
182	# shellcheck disable=2086
183	sudo rm -f ${TEST_FILES}
184}
185
186#
187# Takes a name as the only arguments and looks for the following variations
188# on that name.  If one is found it is returned.
189#
190# $RUNFILE_DIR/<name>
191# $RUNFILE_DIR/<name>.run
192# <name>
193# <name>.run
194#
195find_runfile() {
196	NAME=$1
197
198	if [ -f "$RUNFILE_DIR/$NAME" ]; then
199		echo "$RUNFILE_DIR/$NAME"
200	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
201		echo "$RUNFILE_DIR/$NAME.run"
202	elif [ -f "$NAME" ]; then
203		echo "$NAME"
204	elif [ -f "$NAME.run" ]; then
205		echo "$NAME.run"
206	else
207		return 1
208	fi
209}
210
211#
212# Symlink file if it appears under any of the given paths.
213#
214create_links() {
215	dir_list="$1"
216	file_list="$2"
217
218	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
219
220	for i in $file_list; do
221		for j in $dir_list; do
222			[ ! -e "$STF_PATH/$i" ] || continue
223
224			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
225				ln -sf "$j/$i" "$STF_PATH/$i" || \
226				    fail "Couldn't link $i"
227				break
228			fi
229		done
230
231		[ ! -e "$STF_PATH/$i" ] && \
232		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
233	done
234	STF_MISSING_BIN=${STF_MISSING_BIN# }
235}
236
237#
238# Constrain the path to limit the available binaries to a known set.
239# When running in-tree a top level ./bin/ directory is created for
240# convenience, otherwise a temporary directory is used.
241#
242constrain_path() {
243	. "$STF_SUITE/include/commands.cfg"
244
245	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
246	# install to /usr/local/sbin. To avoid testing the wrong utils we
247	# need /usr/local to come before / in the path search order.
248	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
249	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
250
251	if [ "$INTREE" = "yes" ]; then
252		# Constrained path set to $(top_builddir)/tests/zfs-tests/bin
253		STF_PATH="$BIN_DIR"
254		STF_PATH_REMOVE="no"
255		STF_MISSING_BIN=""
256		if [ ! -d "$STF_PATH" ]; then
257			mkdir "$STF_PATH"
258			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
259		fi
260
261		# Special case links for standard zfs utilities
262		create_links "$CMD_DIR" "$ZFS_FILES"
263
264		# Special case links for zfs test suite utilities
265		create_links "$CMD_DIR/tests/zfs-tests/cmd" "$ZFSTEST_FILES"
266	else
267		# Constrained path set to /var/tmp/constrained_path.*
268		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
269		STF_PATH=$(mktemp -d "$SYSTEMDIR")
270		STF_PATH_REMOVE="yes"
271		STF_MISSING_BIN=""
272
273		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
274
275		# Special case links for standard zfs utilities
276		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
277
278		# Special case links for zfs test suite utilities
279		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
280	fi
281
282	# Standard system utilities
283	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
284	if [ "$UNAME" = "FreeBSD" ] ; then
285		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
286	else
287		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
288	fi
289	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
290
291	# Exceptions
292	if [ "$UNAME" = "Linux" ] ; then
293		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
294		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
295		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
296		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
297	elif [ "$UNAME" = "FreeBSD" ] ; then
298		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
299	fi
300}
301
302#
303# Output a useful usage message.
304#
305usage() {
306cat << EOF
307USAGE:
308$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
309
310DESCRIPTION:
311	ZFS Test Suite launch script
312
313OPTIONS:
314	-h          Show this message
315	-v          Verbose zfs-tests.sh output
316	-q          Quiet test-runner output
317	-D          Debug; show all test output immediately (noisy)
318	-x          Remove all testpools, dm, lo, and files (unsafe)
319	-k          Disable cleanup after test failure
320	-K          Log test names to /dev/kmsg
321	-f          Use files only, disables block device tests
322	-S          Enable stack tracer (negative performance impact)
323	-c          Only create and populate constrained path
324	-R          Automatically rerun failing tests
325	-m          Enable kmemleak reporting (Linux only)
326	-n NFSFILE  Use the nfsfile to determine the NFS configuration
327	-I NUM      Number of iterations
328	-d DIR      Use world-writable DIR for files and loopback devices
329	-s SIZE     Use vdevs of SIZE (default: 4G)
330	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
331	-t PATH|NAME  Run single test at PATH relative to test suite,
332	                or search for test by NAME
333	-T TAGS     Comma separated list of tags (default: 'functional')
334	-u USER     Run single test as USER (default: root)
335
336EXAMPLES:
337# Run the default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and output the configuration used.
338$0 -v
339
340# Run a smaller suite of tests designed to run more quickly.
341$0 -r linux-fast
342
343# Run a single test
344$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
345
346# Run a single test by name
347$0 -t zfs_bookmark_cliargs
348
349# Cleanup a previous run of the test suite prior to testing, run the
350# default ($(echo "${DEFAULT_RUNFILES}" | sed 's/\.run//')) suite of tests and perform no cleanup on exit.
351$0 -x
352
353EOF
354}
355
356while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do
357	case $OPTION in
358	h)
359		usage
360		exit 1
361		;;
362	v)
363		VERBOSE="yes"
364		;;
365	q)
366		QUIET="yes"
367		;;
368	x)
369		CLEANUPALL="yes"
370		;;
371	k)
372		CLEANUP="no"
373		;;
374	K)
375		KMSG="yes"
376		;;
377	f)
378		LOOPBACK="no"
379		;;
380	S)
381		STACK_TRACER="yes"
382		;;
383	c)
384		constrain_path
385		exit
386		;;
387	R)
388		RERUN="yes"
389		;;
390	m)
391		KMEMLEAK="yes"
392		;;
393	n)
394		nfsfile=$OPTARG
395		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
396		export NFS=1
397		. "$nfsfile"
398		;;
399	d)
400		FILEDIR="$OPTARG"
401		;;
402	D)
403		DEBUG="yes"
404		;;
405	I)
406		ITERATIONS="$OPTARG"
407		if [ "$ITERATIONS" -le 0 ]; then
408			fail "Iterations must be greater than 0."
409		fi
410		;;
411	s)
412		FILESIZE="$OPTARG"
413		;;
414	r)
415		RUNFILES="$OPTARG"
416		;;
417	t)
418		if [ -n "$SINGLETEST" ]; then
419			fail "-t can only be provided once."
420		fi
421		SINGLETEST="$OPTARG"
422		;;
423	T)
424		TAGS="$OPTARG"
425		;;
426	u)
427		SINGLETESTUSER="$OPTARG"
428		;;
429	?)
430		usage
431		exit
432		;;
433	*)
434		;;
435	esac
436done
437
438shift $((OPTIND-1))
439
440FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
441LOOPBACKS=${LOOPBACKS:-""}
442
443if [ -n "$SINGLETEST" ]; then
444	if [ -n "$TAGS" ]; then
445		fail "-t and -T are mutually exclusive."
446	fi
447	RUNFILE_DIR="/var/tmp"
448	RUNFILES="zfs-tests.$$.run"
449	[ -n "$QUIET" ] && SINGLEQUIET="True" || SINGLEQUIET="False"
450
451	cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
452[DEFAULT]
453pre =
454quiet = $SINGLEQUIET
455pre_user = root
456user = $SINGLETESTUSER
457timeout = 600
458post_user = root
459post =
460outputdir = /var/tmp/test_results
461EOF
462	if [ "$SINGLETEST" = "${SINGLETEST%/*}" ] ; then
463		NEWSINGLETEST=$(find "$STF_SUITE" -name "$SINGLETEST*" -print -quit)
464		if [ -z "$NEWSINGLETEST" ] ; then
465			fail "couldn't find test matching '$SINGLETEST'"
466		fi
467		SINGLETEST=$NEWSINGLETEST
468	fi
469
470	SINGLETESTDIR="${SINGLETEST%/*}"
471	SETUPDIR="$SINGLETESTDIR"
472	[ "${SETUPDIR#/}" = "$SETUPDIR" ] && SETUPDIR="$STF_SUITE/$SINGLETESTDIR"
473	[ -x "$SETUPDIR/setup.ksh"   ] && SETUPSCRIPT="setup"     || SETUPSCRIPT=
474	[ -x "$SETUPDIR/cleanup.ksh" ] && CLEANUPSCRIPT="cleanup" || CLEANUPSCRIPT=
475
476	SINGLETESTFILE="${SINGLETEST##*/}"
477	cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
478
479[$SINGLETESTDIR]
480tests = ['$SINGLETESTFILE']
481pre = $SETUPSCRIPT
482post = $CLEANUPSCRIPT
483tags = ['functional']
484EOF
485fi
486
487#
488# Use default tag if none was specified
489#
490TAGS=${TAGS:='functional'}
491
492#
493# Attempt to locate the runfiles describing the test workload.
494#
495R=""
496IFS=,
497for RUNFILE in $RUNFILES; do
498	if [ -n "$RUNFILE" ]; then
499		SAVED_RUNFILE="$RUNFILE"
500		RUNFILE=$(find_runfile "$RUNFILE") ||
501			fail "Cannot find runfile: $SAVED_RUNFILE"
502		R="$R,$RUNFILE"
503	fi
504
505	if [ ! -r "$RUNFILE" ]; then
506		fail "Cannot read runfile: $RUNFILE"
507	fi
508done
509unset IFS
510RUNFILES=${R#,}
511
512#
513# This script should not be run as root.  Instead the test user, which may
514# be a normal user account, needs to be configured such that it can
515# run commands via sudo passwordlessly.
516#
517if [ "$(id -u)" = "0" ]; then
518	fail "This script must not be run as root."
519fi
520
521if [ "$(sudo id -un)" != "root" ]; then
522	fail "Passwordless sudo access required."
523fi
524
525#
526# Constrain the available binaries to a known set.
527#
528constrain_path
529
530#
531# Check if ksh exists
532#
533if [ "$UNAME" = "FreeBSD" ]; then
534	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
535fi
536[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
537[ -e "$STF_SUITE/include/default.cfg" ] || fail \
538    "Missing $STF_SUITE/include/default.cfg file."
539
540#
541# Verify the ZFS module stack is loaded.
542#
543if [ "$STACK_TRACER" = "yes" ]; then
544	sudo "${ZFS_SH}" -S >/dev/null 2>&1
545else
546	sudo "${ZFS_SH}" >/dev/null 2>&1
547fi
548
549#
550# Attempt to cleanup all previous state for a new test run.
551#
552if [ "$CLEANUPALL" = "yes" ]; then
553	cleanup_all
554fi
555
556#
557# By default preserve any existing pools
558#
559if [ -z "${KEEP}" ]; then
560	KEEP="$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | tr -s '[:space:]' ' ')"
561	if [ -z "${KEEP}" ]; then
562		KEEP="rpool"
563	fi
564else
565	KEEP="$(echo "$KEEP" | tr -s '[:space:]' ' ')"
566fi
567
568#
569# NOTE: The following environment variables are undocumented
570# and should be used for testing purposes only:
571#
572# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
573# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
574#
575# See libzfs/libzfs_config.c for more information.
576#
577__ZFS_POOL_EXCLUDE="$KEEP"
578
579. "$STF_SUITE/include/default.cfg"
580
581#
582# No DISKS have been provided so a basic file or loopback based devices
583# must be created for the test suite to use.
584#
585if [ -z "${DISKS}" ]; then
586	#
587	# If this is a performance run, prevent accidental use of
588	# loopback devices.
589	#
590	[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
591
592	#
593	# Create sparse files for the test suite.  These may be used
594	# directory or have loopback devices layered on them.
595	#
596	for TEST_FILE in ${FILES}; do
597		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
598		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
599		    fail "Failed creating: ${TEST_FILE} ($?)"
600	done
601
602	#
603	# If requested setup loopback devices backed by the sparse files.
604	#
605	if [ "$LOOPBACK" = "yes" ]; then
606		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
607
608		for TEST_FILE in ${FILES}; do
609			if [ "$UNAME" = "FreeBSD" ] ; then
610				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
611				if [ -z "$MDDEVICE" ] ; then
612					fail "Failed: ${TEST_FILE} -> loopback"
613				fi
614				DISKS="$DISKS $MDDEVICE"
615				LOOPBACKS="$LOOPBACKS $MDDEVICE"
616			else
617				TEST_LOOPBACK=$(sudo "${LOSETUP}" --show -f "${TEST_FILE}") ||
618				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
619				BASELOOPBACK="${TEST_LOOPBACK##*/}"
620				DISKS="$DISKS $BASELOOPBACK"
621				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
622			fi
623		done
624		DISKS=${DISKS# }
625		LOOPBACKS=${LOOPBACKS# }
626	else
627		DISKS="$FILES"
628	fi
629fi
630
631#
632# It may be desirable to test with fewer disks than the default when running
633# the performance tests, but the functional tests require at least three.
634#
635NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
636if [ "$TAGS" != "perf" ]; then
637	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
638fi
639
640#
641# Disable SELinux until the ZFS Test Suite has been updated accordingly.
642#
643if command -v setenforce >/dev/null; then
644	sudo setenforce permissive >/dev/null 2>&1
645fi
646
647#
648# Enable internal ZFS debug log and clear it.
649#
650if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
651	sudo sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
652	sudo sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
653fi
654
655msg
656msg "--- Configuration ---"
657msg "Runfiles:        $RUNFILES"
658msg "STF_TOOLS:       $STF_TOOLS"
659msg "STF_SUITE:       $STF_SUITE"
660msg "STF_PATH:        $STF_PATH"
661msg "FILEDIR:         $FILEDIR"
662msg "FILES:           $FILES"
663msg "LOOPBACKS:       $LOOPBACKS"
664msg "DISKS:           $DISKS"
665msg "NUM_DISKS:       $NUM_DISKS"
666msg "FILESIZE:        $FILESIZE"
667msg "ITERATIONS:      $ITERATIONS"
668msg "TAGS:            $TAGS"
669msg "STACK_TRACER:    $STACK_TRACER"
670msg "Keep pool(s):    $KEEP"
671msg "Missing util(s): $STF_MISSING_BIN"
672msg ""
673
674export STF_TOOLS
675export STF_SUITE
676export STF_PATH
677export DISKS
678export FILEDIR
679export KEEP
680export __ZFS_POOL_EXCLUDE
681export TESTFAIL_CALLBACKS
682
683mktemp_file() {
684	if [ "$UNAME" = "FreeBSD" ]; then
685		mktemp -u "${FILEDIR}/$1.XXXXXX"
686	else
687		mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
688	fi
689}
690mkdir -p "$FILEDIR" || :
691RESULTS_FILE=$(mktemp_file zts-results)
692REPORT_FILE=$(mktemp_file zts-report)
693
694#
695# Run all the tests as specified.
696#
697msg "${TEST_RUNNER}" \
698    "${QUIET:+-q}" \
699    "${DEBUG:+-D}" \
700    "${KMEMLEAK:+-m}" \
701    "${KMSG:+-K}" \
702    "-c \"${RUNFILES}\"" \
703    "-T \"${TAGS}\"" \
704    "-i \"${STF_SUITE}\"" \
705    "-I \"${ITERATIONS}\""
706{ PATH=$STF_PATH \
707    ${TEST_RUNNER} \
708    ${QUIET:+-q} \
709    ${DEBUG:+-D} \
710    ${KMEMLEAK:+-m} \
711    ${KMSG:+-K} \
712    -c "${RUNFILES}" \
713    -T "${TAGS}" \
714    -i "${STF_SUITE}" \
715    -I "${ITERATIONS}" \
716    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
717read -r RUNRESULT <"$REPORT_FILE"
718
719#
720# Analyze the results.
721#
722${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
723RESULT=$?
724
725if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
726	MAYBES="$($ZTS_REPORT --list-maybes)"
727	TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
728	TEST_LIST=$(mktemp_file test-list)
729	grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
730	for test_name in $MAYBES; do
731		grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
732	done
733	{ PATH=$STF_PATH \
734	    ${TEST_RUNNER} \
735	        ${QUIET:+-q} \
736	        ${DEBUG:+-D} \
737	        ${KMEMLEAK:+-m} \
738	    -c "${RUNFILES}" \
739	    -T "${TAGS}" \
740	    -i "${STF_SUITE}" \
741	    -I "${ITERATIONS}" \
742	    -l "${TEST_LIST}" \
743	    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
744	read -r RUNRESULT <"$REPORT_FILE"
745	#
746	# Analyze the results.
747	#
748	${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
749	RESULT=$?
750fi
751
752
753cat "$REPORT_FILE"
754
755RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
756if [ -d "$RESULTS_DIR" ]; then
757	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
758fi
759
760rm -f "$RESULTS_FILE" "$REPORT_FILE" "$TEST_LIST" "$TEMP_RESULTS_FILE"
761
762if [ -n "$SINGLETEST" ]; then
763	rm -f "$RUNFILES" >/dev/null 2>&1
764fi
765
766[ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"
767