xref: /freebsd/sys/contrib/openzfs/scripts/zfs-tests.sh (revision 13ec1e3155c7e9bf037b12af186351b7fa9b9450)
1#!/bin/sh
2# shellcheck disable=SC2154
3#
4# CDDL HEADER START
5#
6# The contents of this file are subject to the terms of the
7# Common Development and Distribution License, Version 1.0 only
8# (the "License").  You may not use this file except in compliance
9# with the License.
10#
11# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
12# or http://www.opensolaris.org/os/licensing.
13# See the License for the specific language governing permissions
14# and limitations under the License.
15#
16# When distributing Covered Code, include this CDDL HEADER in each
17# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
18# If applicable, add the following below this CDDL HEADER, with the
19# fields enclosed by brackets "[]" replaced with your own identifying
20# information: Portions Copyright [yyyy] [name of copyright owner]
21#
22# CDDL HEADER END
23#
24
25#
26# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
27#
28
29BASE_DIR=$(dirname "$0")
30SCRIPT_COMMON=common.sh
31if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
32. "${BASE_DIR}/${SCRIPT_COMMON}"
33else
34echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
35fi
36
37PROG=zfs-tests.sh
38VERBOSE="no"
39QUIET=""
40CLEANUP="yes"
41CLEANUPALL="no"
42LOOPBACK="yes"
43STACK_TRACER="no"
44FILESIZE="4G"
45DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
46RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
47FILEDIR=${FILEDIR:-/var/tmp}
48DISKS=${DISKS:-""}
49SINGLETEST=""
50SINGLETESTUSER="root"
51TAGS=""
52ITERATIONS=1
53ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
54ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
55UNAME=$(uname -s)
56RERUN=""
57
58# Override some defaults if on FreeBSD
59if [ "$UNAME" = "FreeBSD" ] ; then
60	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
61	LOSETUP=/sbin/mdconfig
62	DMSETUP=/sbin/gpart
63else
64	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
65	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
66	LOSETUP=${LOSETUP:-/sbin/losetup}
67	DMSETUP=${DMSETUP:-/sbin/dmsetup}
68fi
69
70#
71# Log an informational message when additional verbosity is enabled.
72#
73msg() {
74	if [ "$VERBOSE" = "yes" ]; then
75		echo "$@"
76	fi
77}
78
79#
80# Log a failure message, cleanup, and return an error.
81#
82fail() {
83	echo "$PROG: $1" >&2
84	cleanup
85	exit 1
86}
87
88cleanup_freebsd_loopback() {
89	for TEST_LOOPBACK in ${LOOPBACKS}; do
90		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
91			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
92			    echo "Failed to destroy: ${TEST_LOOPBACK}"
93		fi
94	done
95}
96
97cleanup_linux_loopback() {
98	for TEST_LOOPBACK in ${LOOPBACKS}; do
99		LOOP_DEV="${TEST_LOOPBACK##*/}"
100		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
101		    grep "${LOOP_DEV}" | cut -f1)
102
103		if [ -n "$DM_DEV" ]; then
104			sudo "${DMSETUP}" remove "${DM_DEV}" ||
105			    echo "Failed to remove: ${DM_DEV}"
106		fi
107
108		if [ -n "${TEST_LOOPBACK}" ]; then
109			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
110			    echo "Failed to remove: ${TEST_LOOPBACK}"
111		fi
112	done
113}
114
115#
116# Attempt to remove loopback devices and files which where created earlier
117# by this script to run the test framework.  The '-k' option may be passed
118# to the script to suppress cleanup for debugging purposes.
119#
120cleanup() {
121	if [ "$CLEANUP" = "no" ]; then
122		return 0
123	fi
124
125
126	if [ "$LOOPBACK" = "yes" ]; then
127		if [ "$UNAME" = "FreeBSD" ] ; then
128			cleanup_freebsd_loopback
129		else
130			cleanup_linux_loopback
131		fi
132	fi
133
134	for TEST_FILE in ${FILES}; do
135		rm -f "${TEST_FILE}" >/dev/null 2>&1
136	done
137
138	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
139		rm -Rf "$STF_PATH"
140	fi
141}
142trap cleanup EXIT
143
144#
145# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
146# loopback devices, and files.  This is a useful way to cleanup a previous
147# test run failure which has left the system in an unknown state.  This can
148# be dangerous and should only be used in a dedicated test environment.
149#
150cleanup_all() {
151	TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool)
152	if [ "$UNAME" = "FreeBSD" ] ; then
153		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
154	else
155		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
156	fi
157	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
158
159	msg
160	msg "--- Cleanup ---"
161	msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
162	for TEST_POOL in $TEST_POOLS; do
163		sudo "$ZPOOL" destroy "${TEST_POOL}"
164	done
165
166	if [ "$UNAME" != "FreeBSD" ] ; then
167		msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
168		    grep loop | tr '\n' ' ')"
169		sudo "${DMSETUP}" remove_all
170	fi
171
172	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
173	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
174		if [ "$UNAME" = "FreeBSD" ] ; then
175			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
176		else
177			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
178		fi
179	done
180
181	msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
182	for TEST_FILE in $TEST_FILES; do
183		sudo rm -f "${TEST_FILE}"
184	done
185}
186
187#
188# Takes a name as the only arguments and looks for the following variations
189# on that name.  If one is found it is returned.
190#
191# $RUNFILE_DIR/<name>
192# $RUNFILE_DIR/<name>.run
193# <name>
194# <name>.run
195#
196find_runfile() {
197	NAME=$1
198	RESULT=""
199
200	if [ -f "$RUNFILE_DIR/$NAME" ]; then
201		RESULT="$RUNFILE_DIR/$NAME"
202	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
203		RESULT="$RUNFILE_DIR/$NAME.run"
204	elif [ -f "$NAME" ]; then
205		RESULT="$NAME"
206	elif [ -f "$NAME.run" ]; then
207		RESULT="$NAME.run"
208	fi
209
210	echo "$RESULT"
211}
212
213#
214# Symlink file if it appears under any of the given paths.
215#
216create_links() {
217	dir_list="$1"
218	file_list="$2"
219
220	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
221
222	for i in $file_list; do
223		for j in $dir_list; do
224			[ ! -e "$STF_PATH/$i" ] || continue
225
226			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
227				ln -sf "$j/$i" "$STF_PATH/$i" || \
228				    fail "Couldn't link $i"
229				break
230			fi
231		done
232
233		[ ! -e "$STF_PATH/$i" ] && \
234		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
235	done
236	STF_MISSING_BIN=${STF_MISSING_BIN# }
237}
238
239#
240# Constrain the path to limit the available binaries to a known set.
241# When running in-tree a top level ./bin/ directory is created for
242# convenience, otherwise a temporary directory is used.
243#
244constrain_path() {
245	. "$STF_SUITE/include/commands.cfg"
246
247	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
248	# install to /usr/local/sbin. To avoid testing the wrong utils we
249	# need /usr/local to come before / in the path search order.
250	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
251	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
252
253	if [ "$INTREE" = "yes" ]; then
254		# Constrained path set to ./zfs/bin/
255		STF_PATH="$BIN_DIR"
256		STF_PATH_REMOVE="no"
257		STF_MISSING_BIN=""
258		if [ ! -d "$STF_PATH" ]; then
259			mkdir "$STF_PATH"
260			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
261		fi
262
263		# Special case links for standard zfs utilities
264		DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
265		    ! -name .libs \) -print | tr '\n' ' ')"
266		create_links "$DIRS" "$ZFS_FILES"
267
268		# Special case links for zfs test suite utilities
269		DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
270		    ! -name .libs \) -print | tr '\n' ' ')"
271		create_links "$DIRS" "$ZFSTEST_FILES"
272	else
273		# Constrained path set to /var/tmp/constrained_path.*
274		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
275		STF_PATH=$(mktemp -d "$SYSTEMDIR")
276		STF_PATH_REMOVE="yes"
277		STF_MISSING_BIN=""
278
279		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
280
281		# Special case links for standard zfs utilities
282		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
283
284		# Special case links for zfs test suite utilities
285		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
286	fi
287
288	# Standard system utilities
289	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
290	if [ "$UNAME" = "FreeBSD" ] ; then
291		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
292	else
293		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
294	fi
295	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
296
297	# Exceptions
298	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
299	if [ "$UNAME" = "Linux" ] ; then
300		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
301		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
302		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
303		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
304		ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
305		ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
306	elif [ "$UNAME" = "FreeBSD" ] ; then
307		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
308	fi
309}
310
311#
312# Output a useful usage message.
313#
314usage() {
315cat << EOF
316USAGE:
317$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
318
319DESCRIPTION:
320	ZFS Test Suite launch script
321
322OPTIONS:
323	-h          Show this message
324	-v          Verbose zfs-tests.sh output
325	-q          Quiet test-runner output
326	-x          Remove all testpools, dm, lo, and files (unsafe)
327	-k          Disable cleanup after test failure
328	-f          Use files only, disables block device tests
329	-S          Enable stack tracer (negative performance impact)
330	-c          Only create and populate constrained path
331	-R          Automatically rerun failing tests
332	-n NFSFILE  Use the nfsfile to determine the NFS configuration
333	-I NUM      Number of iterations
334	-d DIR      Use DIR for files and loopback devices
335	-s SIZE     Use vdevs of SIZE (default: 4G)
336	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
337	-t PATH     Run single test at PATH relative to test suite
338	-T TAGS     Comma separated list of tags (default: 'functional')
339	-u USER     Run single test as USER (default: root)
340
341EXAMPLES:
342# Run the default (linux) suite of tests and output the configuration used.
343$0 -v
344
345# Run a smaller suite of tests designed to run more quickly.
346$0 -r linux-fast
347
348# Run a single test
349$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
350
351# Cleanup a previous run of the test suite prior to testing, run the
352# default (linux) suite of tests and perform no cleanup on exit.
353$0 -x
354
355EOF
356}
357
358while getopts 'hvqxkfScRn:d:s:r:?t:T:u:I:' OPTION; do
359	case $OPTION in
360	h)
361		usage
362		exit 1
363		;;
364	v)
365		VERBOSE="yes"
366		;;
367	q)
368		QUIET="yes"
369		;;
370	x)
371		CLEANUPALL="yes"
372		;;
373	k)
374		CLEANUP="no"
375		;;
376	f)
377		LOOPBACK="no"
378		;;
379	S)
380		STACK_TRACER="yes"
381		;;
382	c)
383		constrain_path
384		exit
385		;;
386	R)
387		RERUN="yes"
388		;;
389	n)
390		nfsfile=$OPTARG
391		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
392		export NFS=1
393		. "$nfsfile"
394		;;
395	d)
396		FILEDIR="$OPTARG"
397		;;
398	I)
399		ITERATIONS="$OPTARG"
400		if [ "$ITERATIONS" -le 0 ]; then
401			fail "Iterations must be greater than 0."
402		fi
403		;;
404	s)
405		FILESIZE="$OPTARG"
406		;;
407	r)
408		RUNFILES="$OPTARG"
409		;;
410	t)
411		if [ -n "$SINGLETEST" ]; then
412			fail "-t can only be provided once."
413		fi
414		SINGLETEST="$OPTARG"
415		;;
416	T)
417		TAGS="$OPTARG"
418		;;
419	u)
420		SINGLETESTUSER="$OPTARG"
421		;;
422	?)
423		usage
424		exit
425		;;
426	*)
427		;;
428	esac
429done
430
431shift $((OPTIND-1))
432
433FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
434LOOPBACKS=${LOOPBACKS:-""}
435
436if [ -n "$SINGLETEST" ]; then
437	if [ -n "$TAGS" ]; then
438		fail "-t and -T are mutually exclusive."
439	fi
440	RUNFILE_DIR="/var/tmp"
441	RUNFILES="zfs-tests.$$.run"
442	SINGLEQUIET="False"
443
444	if [ -n "$QUIET" ]; then
445		SINGLEQUIET="True"
446	fi
447
448	cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
449[DEFAULT]
450pre =
451quiet = $SINGLEQUIET
452pre_user = root
453user = $SINGLETESTUSER
454timeout = 600
455post_user = root
456post =
457outputdir = /var/tmp/test_results
458EOF
459	SINGLETESTDIR=$(dirname "$SINGLETEST")
460	SINGLETESTFILE=$(basename "$SINGLETEST")
461	SETUPSCRIPT=
462	CLEANUPSCRIPT=
463
464	if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
465		SETUPSCRIPT="setup"
466	fi
467
468	if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
469		CLEANUPSCRIPT="cleanup"
470	fi
471
472	cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
473
474[$SINGLETESTDIR]
475tests = ['$SINGLETESTFILE']
476pre = $SETUPSCRIPT
477post = $CLEANUPSCRIPT
478tags = ['functional']
479EOF
480fi
481
482#
483# Use default tag if none was specified
484#
485TAGS=${TAGS:='functional'}
486
487#
488# Attempt to locate the runfiles describing the test workload.
489#
490R=""
491IFS=,
492for RUNFILE in $RUNFILES; do
493	if [ -n "$RUNFILE" ]; then
494		SAVED_RUNFILE="$RUNFILE"
495		RUNFILE=$(find_runfile "$RUNFILE")
496		[ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
497		R="$R,$RUNFILE"
498	fi
499
500	if [ ! -r "$RUNFILE" ]; then
501		fail "Cannot read runfile: $RUNFILE"
502	fi
503done
504unset IFS
505RUNFILES=${R#,}
506
507#
508# This script should not be run as root.  Instead the test user, which may
509# be a normal user account, needs to be configured such that it can
510# run commands via sudo passwordlessly.
511#
512if [ "$(id -u)" = "0" ]; then
513	fail "This script must not be run as root."
514fi
515
516if [ "$(sudo whoami)" != "root" ]; then
517	fail "Passwordless sudo access required."
518fi
519
520#
521# Constrain the available binaries to a known set.
522#
523constrain_path
524
525#
526# Check if ksh exists
527#
528if [ "$UNAME" = "FreeBSD" ]; then
529	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
530fi
531[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
532[ -e "$STF_SUITE/include/default.cfg" ] || fail \
533    "Missing $STF_SUITE/include/default.cfg file."
534
535#
536# Verify the ZFS module stack is loaded.
537#
538if [ "$STACK_TRACER" = "yes" ]; then
539	sudo "${ZFS_SH}" -S >/dev/null 2>&1
540else
541	sudo "${ZFS_SH}" >/dev/null 2>&1
542fi
543
544#
545# Attempt to cleanup all previous state for a new test run.
546#
547if [ "$CLEANUPALL" = "yes" ]; then
548	cleanup_all
549fi
550
551#
552# By default preserve any existing pools
553# NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
554# space-delimited to newline-delimited.
555#
556if [ -z "${KEEP}" ]; then
557	KEEP="$(sudo "$ZPOOL" list -H -o name)"
558	if [ -z "${KEEP}" ]; then
559		KEEP="rpool"
560	fi
561else
562	KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')"
563fi
564
565#
566# NOTE: The following environment variables are undocumented
567# and should be used for testing purposes only:
568#
569# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
570# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
571#
572# See libzfs/libzfs_config.c for more information.
573#
574if [ "$UNAME" = "FreeBSD" ] ; then
575	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
576else
577	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
578fi
579
580. "$STF_SUITE/include/default.cfg"
581
582#
583# No DISKS have been provided so a basic file or loopback based devices
584# must be created for the test suite to use.
585#
586if [ -z "${DISKS}" ]; then
587	#
588	# If this is a performance run, prevent accidental use of
589	# loopback devices.
590	#
591	[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
592
593	#
594	# Create sparse files for the test suite.  These may be used
595	# directory or have loopback devices layered on them.
596	#
597	for TEST_FILE in ${FILES}; do
598		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
599		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
600		    fail "Failed creating: ${TEST_FILE} ($?)"
601	done
602
603	#
604	# If requested setup loopback devices backed by the sparse files.
605	#
606	if [ "$LOOPBACK" = "yes" ]; then
607		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
608
609		for TEST_FILE in ${FILES}; do
610			if [ "$UNAME" = "FreeBSD" ] ; then
611				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
612				if [ -z "$MDDEVICE" ] ; then
613					fail "Failed: ${TEST_FILE} -> loopback"
614				fi
615				DISKS="$DISKS $MDDEVICE"
616				LOOPBACKS="$LOOPBACKS $MDDEVICE"
617			else
618				TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
619				sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
620				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
621				BASELOOPBACK="${TEST_LOOPBACK##*/}"
622				DISKS="$DISKS $BASELOOPBACK"
623				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
624			fi
625		done
626		DISKS=${DISKS# }
627		LOOPBACKS=${LOOPBACKS# }
628	else
629		DISKS="$FILES"
630	fi
631fi
632
633#
634# It may be desirable to test with fewer disks than the default when running
635# the performance tests, but the functional tests require at least three.
636#
637NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
638if [ "$TAGS" != "perf" ]; then
639	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
640fi
641
642#
643# Disable SELinux until the ZFS Test Suite has been updated accordingly.
644#
645if [ -x "$STF_PATH/setenforce" ]; then
646	sudo setenforce permissive >/dev/null 2>&1
647fi
648
649#
650# Enable internal ZFS debug log and clear it.
651#
652if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
653	sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
654	sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
655fi
656
657msg
658msg "--- Configuration ---"
659msg "Runfiles:        $RUNFILES"
660msg "STF_TOOLS:       $STF_TOOLS"
661msg "STF_SUITE:       $STF_SUITE"
662msg "STF_PATH:        $STF_PATH"
663msg "FILEDIR:         $FILEDIR"
664msg "FILES:           $FILES"
665msg "LOOPBACKS:       $LOOPBACKS"
666msg "DISKS:           $DISKS"
667msg "NUM_DISKS:       $NUM_DISKS"
668msg "FILESIZE:        $FILESIZE"
669msg "ITERATIONS:      $ITERATIONS"
670msg "TAGS:            $TAGS"
671msg "STACK_TRACER:    $STACK_TRACER"
672msg "Keep pool(s):    $KEEP"
673msg "Missing util(s): $STF_MISSING_BIN"
674msg ""
675
676export STF_TOOLS
677export STF_SUITE
678export STF_PATH
679export DISKS
680export FILEDIR
681export KEEP
682export __ZFS_POOL_EXCLUDE
683export TESTFAIL_CALLBACKS
684export PATH=$STF_PATH
685
686if [ "$UNAME" = "FreeBSD" ] ; then
687	mkdir -p "$FILEDIR" || true
688	RESULTS_FILE=$(mktemp -u "${FILEDIR}/zts-results.XXXXXX")
689	REPORT_FILE=$(mktemp -u "${FILEDIR}/zts-report.XXXXXX")
690else
691	RESULTS_FILE=$(mktemp -u -t zts-results.XXXXXX -p "$FILEDIR")
692	REPORT_FILE=$(mktemp -u -t zts-report.XXXXXX -p "$FILEDIR")
693fi
694
695#
696# Run all the tests as specified.
697#
698msg "${TEST_RUNNER} ${QUIET:+-q}" \
699    "-c \"${RUNFILES}\"" \
700    "-T \"${TAGS}\"" \
701    "-i \"${STF_SUITE}\"" \
702    "-I \"${ITERATIONS}\""
703${TEST_RUNNER} ${QUIET:+-q} \
704    -c "${RUNFILES}" \
705    -T "${TAGS}" \
706    -i "${STF_SUITE}" \
707    -I "${ITERATIONS}" \
708    2>&1 | tee "$RESULTS_FILE"
709#
710# Analyze the results.
711#
712${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
713RESULT=$?
714
715if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
716	MAYBES="$($ZTS_REPORT --list-maybes)"
717	TEMP_RESULTS_FILE=$(mktemp -u -t zts-results-tmp.XXXXX -p "$FILEDIR")
718	TEST_LIST=$(mktemp -u -t test-list.XXXXX -p "$FILEDIR")
719	grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
720	for test_name in $MAYBES; do
721		grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
722	done
723	${TEST_RUNNER} ${QUIET:+-q} \
724	    -c "${RUNFILES}" \
725	    -T "${TAGS}" \
726	    -i "${STF_SUITE}" \
727	    -I "${ITERATIONS}" \
728	    -l "${TEST_LIST}" \
729	    2>&1 | tee "$RESULTS_FILE"
730	#
731	# Analyze the results.
732	#
733	${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
734	RESULT=$?
735fi
736
737
738cat "$REPORT_FILE"
739
740RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
741if [ -d "$RESULTS_DIR" ]; then
742	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
743fi
744
745rm -f "$RESULTS_FILE" "$REPORT_FILE"
746
747if [ -n "$SINGLETEST" ]; then
748	rm -f "$RUNFILES" >/dev/null 2>&1
749fi
750
751exit "${RESULT}"
752