xref: /freebsd/sys/contrib/openzfs/scripts/zfs-tests.sh (revision 5c4aa6257210502c93ad65882a8a4842d984bae2)
1#!/bin/sh
2# shellcheck disable=SC2154
3#
4# CDDL HEADER START
5#
6# The contents of this file are subject to the terms of the
7# Common Development and Distribution License, Version 1.0 only
8# (the "License").  You may not use this file except in compliance
9# with the License.
10#
11# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
12# or http://www.opensolaris.org/os/licensing.
13# See the License for the specific language governing permissions
14# and limitations under the License.
15#
16# When distributing Covered Code, include this CDDL HEADER in each
17# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
18# If applicable, add the following below this CDDL HEADER, with the
19# fields enclosed by brackets "[]" replaced with your own identifying
20# information: Portions Copyright [yyyy] [name of copyright owner]
21#
22# CDDL HEADER END
23#
24
25#
26# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
27#
28
29BASE_DIR=$(dirname "$0")
30SCRIPT_COMMON=common.sh
31if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
32. "${BASE_DIR}/${SCRIPT_COMMON}"
33else
34echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
35fi
36
37PROG=zfs-tests.sh
38VERBOSE="no"
39QUIET=""
40CLEANUP="yes"
41CLEANUPALL="no"
42LOOPBACK="yes"
43STACK_TRACER="no"
44FILESIZE="4G"
45DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
46RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
47FILEDIR=${FILEDIR:-/var/tmp}
48DISKS=${DISKS:-""}
49SINGLETEST=""
50SINGLETESTUSER="root"
51TAGS=""
52ITERATIONS=1
53ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
54ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
55UNAME=$(uname -s)
56RERUN=""
57KMEMLEAK=""
58
59# Override some defaults if on FreeBSD
60if [ "$UNAME" = "FreeBSD" ] ; then
61	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
62	LOSETUP=/sbin/mdconfig
63	DMSETUP=/sbin/gpart
64else
65	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
66	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
67	LOSETUP=${LOSETUP:-/sbin/losetup}
68	DMSETUP=${DMSETUP:-/sbin/dmsetup}
69fi
70
71#
72# Log an informational message when additional verbosity is enabled.
73#
74msg() {
75	if [ "$VERBOSE" = "yes" ]; then
76		echo "$@"
77	fi
78}
79
80#
81# Log a failure message, cleanup, and return an error.
82#
83fail() {
84	echo "$PROG: $1" >&2
85	cleanup
86	exit 1
87}
88
89cleanup_freebsd_loopback() {
90	for TEST_LOOPBACK in ${LOOPBACKS}; do
91		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
92			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
93			    echo "Failed to destroy: ${TEST_LOOPBACK}"
94		fi
95	done
96}
97
98cleanup_linux_loopback() {
99	for TEST_LOOPBACK in ${LOOPBACKS}; do
100		LOOP_DEV="${TEST_LOOPBACK##*/}"
101		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
102		    grep "${LOOP_DEV}" | cut -f1)
103
104		if [ -n "$DM_DEV" ]; then
105			sudo "${DMSETUP}" remove "${DM_DEV}" ||
106			    echo "Failed to remove: ${DM_DEV}"
107		fi
108
109		if [ -n "${TEST_LOOPBACK}" ]; then
110			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
111			    echo "Failed to remove: ${TEST_LOOPBACK}"
112		fi
113	done
114}
115
116#
117# Attempt to remove loopback devices and files which where created earlier
118# by this script to run the test framework.  The '-k' option may be passed
119# to the script to suppress cleanup for debugging purposes.
120#
121cleanup() {
122	if [ "$CLEANUP" = "no" ]; then
123		return 0
124	fi
125
126
127	if [ "$LOOPBACK" = "yes" ]; then
128		if [ "$UNAME" = "FreeBSD" ] ; then
129			cleanup_freebsd_loopback
130		else
131			cleanup_linux_loopback
132		fi
133	fi
134
135	for TEST_FILE in ${FILES}; do
136		rm -f "${TEST_FILE}" >/dev/null 2>&1
137	done
138
139	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
140		rm -Rf "$STF_PATH"
141	fi
142}
143trap cleanup EXIT
144
145#
146# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
147# loopback devices, and files.  This is a useful way to cleanup a previous
148# test run failure which has left the system in an unknown state.  This can
149# be dangerous and should only be used in a dedicated test environment.
150#
151cleanup_all() {
152	TEST_POOLS=$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name | grep testpool)
153	if [ "$UNAME" = "FreeBSD" ] ; then
154		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
155	else
156		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
157	fi
158	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
159
160	msg
161	msg "--- Cleanup ---"
162	msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
163	for TEST_POOL in $TEST_POOLS; do
164		sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
165	done
166
167	if [ "$UNAME" != "FreeBSD" ] ; then
168		msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
169		    grep loop | tr '\n' ' ')"
170		sudo "${DMSETUP}" remove_all
171	fi
172
173	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
174	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
175		if [ "$UNAME" = "FreeBSD" ] ; then
176			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
177		else
178			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
179		fi
180	done
181
182	msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
183	for TEST_FILE in $TEST_FILES; do
184		sudo rm -f "${TEST_FILE}"
185	done
186}
187
188#
189# Takes a name as the only arguments and looks for the following variations
190# on that name.  If one is found it is returned.
191#
192# $RUNFILE_DIR/<name>
193# $RUNFILE_DIR/<name>.run
194# <name>
195# <name>.run
196#
197find_runfile() {
198	NAME=$1
199	RESULT=""
200
201	if [ -f "$RUNFILE_DIR/$NAME" ]; then
202		RESULT="$RUNFILE_DIR/$NAME"
203	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
204		RESULT="$RUNFILE_DIR/$NAME.run"
205	elif [ -f "$NAME" ]; then
206		RESULT="$NAME"
207	elif [ -f "$NAME.run" ]; then
208		RESULT="$NAME.run"
209	fi
210
211	echo "$RESULT"
212}
213
214#
215# Symlink file if it appears under any of the given paths.
216#
217create_links() {
218	dir_list="$1"
219	file_list="$2"
220
221	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
222
223	for i in $file_list; do
224		for j in $dir_list; do
225			[ ! -e "$STF_PATH/$i" ] || continue
226
227			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
228				ln -sf "$j/$i" "$STF_PATH/$i" || \
229				    fail "Couldn't link $i"
230				break
231			fi
232		done
233
234		[ ! -e "$STF_PATH/$i" ] && \
235		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
236	done
237	STF_MISSING_BIN=${STF_MISSING_BIN# }
238}
239
240#
241# Constrain the path to limit the available binaries to a known set.
242# When running in-tree a top level ./bin/ directory is created for
243# convenience, otherwise a temporary directory is used.
244#
245constrain_path() {
246	. "$STF_SUITE/include/commands.cfg"
247
248	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
249	# install to /usr/local/sbin. To avoid testing the wrong utils we
250	# need /usr/local to come before / in the path search order.
251	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
252	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
253
254	if [ "$INTREE" = "yes" ]; then
255		# Constrained path set to ./zfs/bin/
256		STF_PATH="$BIN_DIR"
257		STF_PATH_REMOVE="no"
258		STF_MISSING_BIN=""
259		if [ ! -d "$STF_PATH" ]; then
260			mkdir "$STF_PATH"
261			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
262		fi
263
264		# Special case links for standard zfs utilities
265		DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
266		    ! -name .libs \) -print | tr '\n' ' ')"
267		create_links "$DIRS" "$ZFS_FILES"
268
269		# Special case links for zfs test suite utilities
270		DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
271		    ! -name .libs \) -print | tr '\n' ' ')"
272		create_links "$DIRS" "$ZFSTEST_FILES"
273	else
274		# Constrained path set to /var/tmp/constrained_path.*
275		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
276		STF_PATH=$(mktemp -d "$SYSTEMDIR")
277		STF_PATH_REMOVE="yes"
278		STF_MISSING_BIN=""
279
280		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
281
282		# Special case links for standard zfs utilities
283		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
284
285		# Special case links for zfs test suite utilities
286		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
287	fi
288
289	# Standard system utilities
290	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
291	if [ "$UNAME" = "FreeBSD" ] ; then
292		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
293	else
294		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
295	fi
296	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
297
298	# Exceptions
299	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
300	if [ "$UNAME" = "Linux" ] ; then
301		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
302		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
303		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
304		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
305		ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
306		ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
307	elif [ "$UNAME" = "FreeBSD" ] ; then
308		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
309	fi
310}
311
312#
313# Output a useful usage message.
314#
315usage() {
316cat << EOF
317USAGE:
318$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
319
320DESCRIPTION:
321	ZFS Test Suite launch script
322
323OPTIONS:
324	-h          Show this message
325	-v          Verbose zfs-tests.sh output
326	-q          Quiet test-runner output
327	-x          Remove all testpools, dm, lo, and files (unsafe)
328	-k          Disable cleanup after test failure
329	-f          Use files only, disables block device tests
330	-S          Enable stack tracer (negative performance impact)
331	-c          Only create and populate constrained path
332	-R          Automatically rerun failing tests
333	-m          Enable kmemleak reporting (Linux only)
334	-n NFSFILE  Use the nfsfile to determine the NFS configuration
335	-I NUM      Number of iterations
336	-d DIR      Use DIR for files and loopback devices
337	-s SIZE     Use vdevs of SIZE (default: 4G)
338	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
339	-t PATH     Run single test at PATH relative to test suite
340	-T TAGS     Comma separated list of tags (default: 'functional')
341	-u USER     Run single test as USER (default: root)
342
343EXAMPLES:
344# Run the default (linux) suite of tests and output the configuration used.
345$0 -v
346
347# Run a smaller suite of tests designed to run more quickly.
348$0 -r linux-fast
349
350# Run a single test
351$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
352
353# Cleanup a previous run of the test suite prior to testing, run the
354# default (linux) suite of tests and perform no cleanup on exit.
355$0 -x
356
357EOF
358}
359
360while getopts 'hvqxkfScRmn:d:s:r:?t:T:u:I:' OPTION; do
361	case $OPTION in
362	h)
363		usage
364		exit 1
365		;;
366	v)
367		VERBOSE="yes"
368		;;
369	q)
370		QUIET="yes"
371		;;
372	x)
373		CLEANUPALL="yes"
374		;;
375	k)
376		CLEANUP="no"
377		;;
378	f)
379		LOOPBACK="no"
380		;;
381	S)
382		STACK_TRACER="yes"
383		;;
384	c)
385		constrain_path
386		exit
387		;;
388	R)
389		RERUN="yes"
390		;;
391	m)
392		KMEMLEAK="yes"
393		;;
394	n)
395		nfsfile=$OPTARG
396		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
397		export NFS=1
398		. "$nfsfile"
399		;;
400	d)
401		FILEDIR="$OPTARG"
402		;;
403	I)
404		ITERATIONS="$OPTARG"
405		if [ "$ITERATIONS" -le 0 ]; then
406			fail "Iterations must be greater than 0."
407		fi
408		;;
409	s)
410		FILESIZE="$OPTARG"
411		;;
412	r)
413		RUNFILES="$OPTARG"
414		;;
415	t)
416		if [ -n "$SINGLETEST" ]; then
417			fail "-t can only be provided once."
418		fi
419		SINGLETEST="$OPTARG"
420		;;
421	T)
422		TAGS="$OPTARG"
423		;;
424	u)
425		SINGLETESTUSER="$OPTARG"
426		;;
427	?)
428		usage
429		exit
430		;;
431	*)
432		;;
433	esac
434done
435
436shift $((OPTIND-1))
437
438FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
439LOOPBACKS=${LOOPBACKS:-""}
440
441if [ -n "$SINGLETEST" ]; then
442	if [ -n "$TAGS" ]; then
443		fail "-t and -T are mutually exclusive."
444	fi
445	RUNFILE_DIR="/var/tmp"
446	RUNFILES="zfs-tests.$$.run"
447	SINGLEQUIET="False"
448
449	if [ -n "$QUIET" ]; then
450		SINGLEQUIET="True"
451	fi
452
453	cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
454[DEFAULT]
455pre =
456quiet = $SINGLEQUIET
457pre_user = root
458user = $SINGLETESTUSER
459timeout = 600
460post_user = root
461post =
462outputdir = /var/tmp/test_results
463EOF
464	SINGLETESTDIR=$(dirname "$SINGLETEST")
465	SINGLETESTFILE=$(basename "$SINGLETEST")
466	SETUPSCRIPT=
467	CLEANUPSCRIPT=
468
469	if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
470		SETUPSCRIPT="setup"
471	fi
472
473	if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
474		CLEANUPSCRIPT="cleanup"
475	fi
476
477	cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
478
479[$SINGLETESTDIR]
480tests = ['$SINGLETESTFILE']
481pre = $SETUPSCRIPT
482post = $CLEANUPSCRIPT
483tags = ['functional']
484EOF
485fi
486
487#
488# Use default tag if none was specified
489#
490TAGS=${TAGS:='functional'}
491
492#
493# Attempt to locate the runfiles describing the test workload.
494#
495R=""
496IFS=,
497for RUNFILE in $RUNFILES; do
498	if [ -n "$RUNFILE" ]; then
499		SAVED_RUNFILE="$RUNFILE"
500		RUNFILE=$(find_runfile "$RUNFILE")
501		[ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
502		R="$R,$RUNFILE"
503	fi
504
505	if [ ! -r "$RUNFILE" ]; then
506		fail "Cannot read runfile: $RUNFILE"
507	fi
508done
509unset IFS
510RUNFILES=${R#,}
511
512#
513# This script should not be run as root.  Instead the test user, which may
514# be a normal user account, needs to be configured such that it can
515# run commands via sudo passwordlessly.
516#
517if [ "$(id -u)" = "0" ]; then
518	fail "This script must not be run as root."
519fi
520
521if [ "$(sudo whoami)" != "root" ]; then
522	fail "Passwordless sudo access required."
523fi
524
525#
526# Constrain the available binaries to a known set.
527#
528constrain_path
529
530#
531# Check if ksh exists
532#
533if [ "$UNAME" = "FreeBSD" ]; then
534	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
535fi
536[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
537[ -e "$STF_SUITE/include/default.cfg" ] || fail \
538    "Missing $STF_SUITE/include/default.cfg file."
539
540#
541# Verify the ZFS module stack is loaded.
542#
543if [ "$STACK_TRACER" = "yes" ]; then
544	sudo "${ZFS_SH}" -S >/dev/null 2>&1
545else
546	sudo "${ZFS_SH}" >/dev/null 2>&1
547fi
548
549#
550# Attempt to cleanup all previous state for a new test run.
551#
552if [ "$CLEANUPALL" = "yes" ]; then
553	cleanup_all
554fi
555
556#
557# By default preserve any existing pools
558# NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
559# space-delimited to newline-delimited.
560#
561if [ -z "${KEEP}" ]; then
562	KEEP="$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name)"
563	if [ -z "${KEEP}" ]; then
564		KEEP="rpool"
565	fi
566else
567	KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')"
568fi
569
570#
571# NOTE: The following environment variables are undocumented
572# and should be used for testing purposes only:
573#
574# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
575# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
576#
577# See libzfs/libzfs_config.c for more information.
578#
579if [ "$UNAME" = "FreeBSD" ] ; then
580	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
581else
582	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
583fi
584
585. "$STF_SUITE/include/default.cfg"
586
587#
588# No DISKS have been provided so a basic file or loopback based devices
589# must be created for the test suite to use.
590#
591if [ -z "${DISKS}" ]; then
592	#
593	# If this is a performance run, prevent accidental use of
594	# loopback devices.
595	#
596	[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
597
598	#
599	# Create sparse files for the test suite.  These may be used
600	# directory or have loopback devices layered on them.
601	#
602	for TEST_FILE in ${FILES}; do
603		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
604		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
605		    fail "Failed creating: ${TEST_FILE} ($?)"
606	done
607
608	#
609	# If requested setup loopback devices backed by the sparse files.
610	#
611	if [ "$LOOPBACK" = "yes" ]; then
612		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
613
614		for TEST_FILE in ${FILES}; do
615			if [ "$UNAME" = "FreeBSD" ] ; then
616				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
617				if [ -z "$MDDEVICE" ] ; then
618					fail "Failed: ${TEST_FILE} -> loopback"
619				fi
620				DISKS="$DISKS $MDDEVICE"
621				LOOPBACKS="$LOOPBACKS $MDDEVICE"
622			else
623				TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
624				sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
625				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
626				BASELOOPBACK="${TEST_LOOPBACK##*/}"
627				DISKS="$DISKS $BASELOOPBACK"
628				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
629			fi
630		done
631		DISKS=${DISKS# }
632		LOOPBACKS=${LOOPBACKS# }
633	else
634		DISKS="$FILES"
635	fi
636fi
637
638#
639# It may be desirable to test with fewer disks than the default when running
640# the performance tests, but the functional tests require at least three.
641#
642NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
643if [ "$TAGS" != "perf" ]; then
644	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
645fi
646
647#
648# Disable SELinux until the ZFS Test Suite has been updated accordingly.
649#
650if [ -x "$STF_PATH/setenforce" ]; then
651	sudo setenforce permissive >/dev/null 2>&1
652fi
653
654#
655# Enable internal ZFS debug log and clear it.
656#
657if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
658	sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
659	sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
660fi
661
662msg
663msg "--- Configuration ---"
664msg "Runfiles:        $RUNFILES"
665msg "STF_TOOLS:       $STF_TOOLS"
666msg "STF_SUITE:       $STF_SUITE"
667msg "STF_PATH:        $STF_PATH"
668msg "FILEDIR:         $FILEDIR"
669msg "FILES:           $FILES"
670msg "LOOPBACKS:       $LOOPBACKS"
671msg "DISKS:           $DISKS"
672msg "NUM_DISKS:       $NUM_DISKS"
673msg "FILESIZE:        $FILESIZE"
674msg "ITERATIONS:      $ITERATIONS"
675msg "TAGS:            $TAGS"
676msg "STACK_TRACER:    $STACK_TRACER"
677msg "Keep pool(s):    $KEEP"
678msg "Missing util(s): $STF_MISSING_BIN"
679msg ""
680
681export STF_TOOLS
682export STF_SUITE
683export STF_PATH
684export DISKS
685export FILEDIR
686export KEEP
687export __ZFS_POOL_EXCLUDE
688export TESTFAIL_CALLBACKS
689export PATH=$STF_PATH
690
691mktemp_file() {
692	if [ "$UNAME" = "FreeBSD" ]; then
693		mktemp -u "${FILEDIR}/$1.XXXXXX"
694	else
695		mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
696	fi
697}
698mkdir -p "$FILEDIR" || :
699RESULTS_FILE=$(mktemp_file zts-results)
700REPORT_FILE=$(mktemp_file zts-report)
701
702#
703# Run all the tests as specified.
704#
705msg "${TEST_RUNNER}" \
706    "${QUIET:+-q}" \
707    "${KMEMLEAK:+-m}" \
708    "-c \"${RUNFILES}\"" \
709    "-T \"${TAGS}\"" \
710    "-i \"${STF_SUITE}\"" \
711    "-I \"${ITERATIONS}\""
712{ ${TEST_RUNNER} \
713    ${QUIET:+-q} \
714    ${KMEMLEAK:+-m} \
715    -c "${RUNFILES}" \
716    -T "${TAGS}" \
717    -i "${STF_SUITE}" \
718    -I "${ITERATIONS}" \
719    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
720read -r RUNRESULT <"$REPORT_FILE"
721
722#
723# Analyze the results.
724#
725${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
726RESULT=$?
727
728if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
729	MAYBES="$($ZTS_REPORT --list-maybes)"
730	TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
731	TEST_LIST=$(mktemp_file test-list)
732	grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
733	for test_name in $MAYBES; do
734		grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
735	done
736	{ ${TEST_RUNNER} \
737            ${QUIET:+-q} \
738            ${KMEMLEAK:+-m} \
739	    -c "${RUNFILES}" \
740	    -T "${TAGS}" \
741	    -i "${STF_SUITE}" \
742	    -I "${ITERATIONS}" \
743	    -l "${TEST_LIST}" \
744	    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
745	read -r RUNRESULT <"$REPORT_FILE"
746	#
747	# Analyze the results.
748	#
749	${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
750	RESULT=$?
751fi
752
753
754cat "$REPORT_FILE"
755
756RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
757if [ -d "$RESULTS_DIR" ]; then
758	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
759fi
760
761rm -f "$RESULTS_FILE" "$REPORT_FILE"
762
763if [ -n "$SINGLETEST" ]; then
764	rm -f "$RUNFILES" >/dev/null 2>&1
765fi
766
767[ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"
768