xref: /freebsd/sys/contrib/openzfs/scripts/zfs-tests.sh (revision 5e801ac66d24704442eba426ed13c3effb8a34e7)
1#!/bin/sh
2# shellcheck disable=SC2154
3#
4# CDDL HEADER START
5#
6# The contents of this file are subject to the terms of the
7# Common Development and Distribution License, Version 1.0 only
8# (the "License").  You may not use this file except in compliance
9# with the License.
10#
11# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
12# or http://www.opensolaris.org/os/licensing.
13# See the License for the specific language governing permissions
14# and limitations under the License.
15#
16# When distributing Covered Code, include this CDDL HEADER in each
17# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
18# If applicable, add the following below this CDDL HEADER, with the
19# fields enclosed by brackets "[]" replaced with your own identifying
20# information: Portions Copyright [yyyy] [name of copyright owner]
21#
22# CDDL HEADER END
23#
24
25#
26# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
27#
28
29BASE_DIR=$(dirname "$0")
30SCRIPT_COMMON=common.sh
31if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
32. "${BASE_DIR}/${SCRIPT_COMMON}"
33else
34echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
35fi
36
37PROG=zfs-tests.sh
38VERBOSE="no"
39QUIET=""
40CLEANUP="yes"
41CLEANUPALL="no"
42KMSG=""
43LOOPBACK="yes"
44STACK_TRACER="no"
45FILESIZE="4G"
46DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
47RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
48FILEDIR=${FILEDIR:-/var/tmp}
49DISKS=${DISKS:-""}
50SINGLETEST=""
51SINGLETESTUSER="root"
52TAGS=""
53ITERATIONS=1
54ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
55ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
56UNAME=$(uname -s)
57RERUN=""
58KMEMLEAK=""
59
60# Override some defaults if on FreeBSD
61if [ "$UNAME" = "FreeBSD" ] ; then
62	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
63	LOSETUP=/sbin/mdconfig
64	DMSETUP=/sbin/gpart
65else
66	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
67	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
68	LOSETUP=${LOSETUP:-/sbin/losetup}
69	DMSETUP=${DMSETUP:-/sbin/dmsetup}
70fi
71
72#
73# Log an informational message when additional verbosity is enabled.
74#
75msg() {
76	if [ "$VERBOSE" = "yes" ]; then
77		echo "$@"
78	fi
79}
80
81#
82# Log a failure message, cleanup, and return an error.
83#
84fail() {
85	echo "$PROG: $1" >&2
86	cleanup
87	exit 1
88}
89
90cleanup_freebsd_loopback() {
91	for TEST_LOOPBACK in ${LOOPBACKS}; do
92		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
93			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
94			    echo "Failed to destroy: ${TEST_LOOPBACK}"
95		fi
96	done
97}
98
99cleanup_linux_loopback() {
100	for TEST_LOOPBACK in ${LOOPBACKS}; do
101		LOOP_DEV="${TEST_LOOPBACK##*/}"
102		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
103		    grep "${LOOP_DEV}" | cut -f1)
104
105		if [ -n "$DM_DEV" ]; then
106			sudo "${DMSETUP}" remove "${DM_DEV}" ||
107			    echo "Failed to remove: ${DM_DEV}"
108		fi
109
110		if [ -n "${TEST_LOOPBACK}" ]; then
111			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
112			    echo "Failed to remove: ${TEST_LOOPBACK}"
113		fi
114	done
115}
116
117#
118# Attempt to remove loopback devices and files which where created earlier
119# by this script to run the test framework.  The '-k' option may be passed
120# to the script to suppress cleanup for debugging purposes.
121#
122cleanup() {
123	if [ "$CLEANUP" = "no" ]; then
124		return 0
125	fi
126
127
128	if [ "$LOOPBACK" = "yes" ]; then
129		if [ "$UNAME" = "FreeBSD" ] ; then
130			cleanup_freebsd_loopback
131		else
132			cleanup_linux_loopback
133		fi
134	fi
135
136	for TEST_FILE in ${FILES}; do
137		rm -f "${TEST_FILE}" >/dev/null 2>&1
138	done
139
140	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
141		rm -Rf "$STF_PATH"
142	fi
143}
144trap cleanup EXIT
145
146#
147# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
148# loopback devices, and files.  This is a useful way to cleanup a previous
149# test run failure which has left the system in an unknown state.  This can
150# be dangerous and should only be used in a dedicated test environment.
151#
152cleanup_all() {
153	TEST_POOLS=$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name | grep testpool)
154	if [ "$UNAME" = "FreeBSD" ] ; then
155		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
156	else
157		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
158	fi
159	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
160
161	msg
162	msg "--- Cleanup ---"
163	msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
164	for TEST_POOL in $TEST_POOLS; do
165		sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
166	done
167
168	if [ "$UNAME" != "FreeBSD" ] ; then
169		msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
170		    grep loop | tr '\n' ' ')"
171		sudo "${DMSETUP}" remove_all
172	fi
173
174	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
175	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
176		if [ "$UNAME" = "FreeBSD" ] ; then
177			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
178		else
179			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
180		fi
181	done
182
183	msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
184	for TEST_FILE in $TEST_FILES; do
185		sudo rm -f "${TEST_FILE}"
186	done
187}
188
189#
190# Takes a name as the only arguments and looks for the following variations
191# on that name.  If one is found it is returned.
192#
193# $RUNFILE_DIR/<name>
194# $RUNFILE_DIR/<name>.run
195# <name>
196# <name>.run
197#
198find_runfile() {
199	NAME=$1
200	RESULT=""
201
202	if [ -f "$RUNFILE_DIR/$NAME" ]; then
203		RESULT="$RUNFILE_DIR/$NAME"
204	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
205		RESULT="$RUNFILE_DIR/$NAME.run"
206	elif [ -f "$NAME" ]; then
207		RESULT="$NAME"
208	elif [ -f "$NAME.run" ]; then
209		RESULT="$NAME.run"
210	fi
211
212	echo "$RESULT"
213}
214
215#
216# Symlink file if it appears under any of the given paths.
217#
218create_links() {
219	dir_list="$1"
220	file_list="$2"
221
222	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
223
224	for i in $file_list; do
225		for j in $dir_list; do
226			[ ! -e "$STF_PATH/$i" ] || continue
227
228			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
229				ln -sf "$j/$i" "$STF_PATH/$i" || \
230				    fail "Couldn't link $i"
231				break
232			fi
233		done
234
235		[ ! -e "$STF_PATH/$i" ] && \
236		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
237	done
238	STF_MISSING_BIN=${STF_MISSING_BIN# }
239}
240
241#
242# Constrain the path to limit the available binaries to a known set.
243# When running in-tree a top level ./bin/ directory is created for
244# convenience, otherwise a temporary directory is used.
245#
246constrain_path() {
247	. "$STF_SUITE/include/commands.cfg"
248
249	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
250	# install to /usr/local/sbin. To avoid testing the wrong utils we
251	# need /usr/local to come before / in the path search order.
252	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
253	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
254
255	if [ "$INTREE" = "yes" ]; then
256		# Constrained path set to ./zfs/bin/
257		STF_PATH="$BIN_DIR"
258		STF_PATH_REMOVE="no"
259		STF_MISSING_BIN=""
260		if [ ! -d "$STF_PATH" ]; then
261			mkdir "$STF_PATH"
262			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
263		fi
264
265		# Special case links for standard zfs utilities
266		DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
267		    ! -name .libs \) -print | tr '\n' ' ')"
268		create_links "$DIRS" "$ZFS_FILES"
269
270		# Special case links for zfs test suite utilities
271		DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
272		    ! -name .libs \) -print | tr '\n' ' ')"
273		create_links "$DIRS" "$ZFSTEST_FILES"
274	else
275		# Constrained path set to /var/tmp/constrained_path.*
276		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
277		STF_PATH=$(mktemp -d "$SYSTEMDIR")
278		STF_PATH_REMOVE="yes"
279		STF_MISSING_BIN=""
280
281		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
282
283		# Special case links for standard zfs utilities
284		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
285
286		# Special case links for zfs test suite utilities
287		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
288	fi
289
290	# Standard system utilities
291	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
292	if [ "$UNAME" = "FreeBSD" ] ; then
293		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
294	else
295		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
296	fi
297	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
298
299	# Exceptions
300	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
301	if [ "$UNAME" = "Linux" ] ; then
302		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
303		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
304		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
305		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
306		ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
307		ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
308	elif [ "$UNAME" = "FreeBSD" ] ; then
309		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
310	fi
311}
312
313#
314# Output a useful usage message.
315#
316usage() {
317cat << EOF
318USAGE:
319$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
320
321DESCRIPTION:
322	ZFS Test Suite launch script
323
324OPTIONS:
325	-h          Show this message
326	-v          Verbose zfs-tests.sh output
327	-q          Quiet test-runner output
328	-x          Remove all testpools, dm, lo, and files (unsafe)
329	-k          Disable cleanup after test failure
330	-K          Log test names to /dev/kmsg
331	-f          Use files only, disables block device tests
332	-S          Enable stack tracer (negative performance impact)
333	-c          Only create and populate constrained path
334	-R          Automatically rerun failing tests
335	-m          Enable kmemleak reporting (Linux only)
336	-n NFSFILE  Use the nfsfile to determine the NFS configuration
337	-I NUM      Number of iterations
338	-d DIR      Use DIR for files and loopback devices
339	-s SIZE     Use vdevs of SIZE (default: 4G)
340	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
341	-t PATH     Run single test at PATH relative to test suite
342	-T TAGS     Comma separated list of tags (default: 'functional')
343	-u USER     Run single test as USER (default: root)
344
345EXAMPLES:
346# Run the default (linux) suite of tests and output the configuration used.
347$0 -v
348
349# Run a smaller suite of tests designed to run more quickly.
350$0 -r linux-fast
351
352# Run a single test
353$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
354
355# Cleanup a previous run of the test suite prior to testing, run the
356# default (linux) suite of tests and perform no cleanup on exit.
357$0 -x
358
359EOF
360}
361
362while getopts 'hvqxkKfScRmn:d:s:r:?t:T:u:I:' OPTION; do
363	case $OPTION in
364	h)
365		usage
366		exit 1
367		;;
368	v)
369		VERBOSE="yes"
370		;;
371	q)
372		QUIET="yes"
373		;;
374	x)
375		CLEANUPALL="yes"
376		;;
377	k)
378		CLEANUP="no"
379		;;
380	K)
381		KMSG="yes"
382		;;
383	f)
384		LOOPBACK="no"
385		;;
386	S)
387		STACK_TRACER="yes"
388		;;
389	c)
390		constrain_path
391		exit
392		;;
393	R)
394		RERUN="yes"
395		;;
396	m)
397		KMEMLEAK="yes"
398		;;
399	n)
400		nfsfile=$OPTARG
401		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
402		export NFS=1
403		. "$nfsfile"
404		;;
405	d)
406		FILEDIR="$OPTARG"
407		;;
408	I)
409		ITERATIONS="$OPTARG"
410		if [ "$ITERATIONS" -le 0 ]; then
411			fail "Iterations must be greater than 0."
412		fi
413		;;
414	s)
415		FILESIZE="$OPTARG"
416		;;
417	r)
418		RUNFILES="$OPTARG"
419		;;
420	t)
421		if [ -n "$SINGLETEST" ]; then
422			fail "-t can only be provided once."
423		fi
424		SINGLETEST="$OPTARG"
425		;;
426	T)
427		TAGS="$OPTARG"
428		;;
429	u)
430		SINGLETESTUSER="$OPTARG"
431		;;
432	?)
433		usage
434		exit
435		;;
436	*)
437		;;
438	esac
439done
440
441shift $((OPTIND-1))
442
443FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
444LOOPBACKS=${LOOPBACKS:-""}
445
446if [ -n "$SINGLETEST" ]; then
447	if [ -n "$TAGS" ]; then
448		fail "-t and -T are mutually exclusive."
449	fi
450	RUNFILE_DIR="/var/tmp"
451	RUNFILES="zfs-tests.$$.run"
452	SINGLEQUIET="False"
453
454	if [ -n "$QUIET" ]; then
455		SINGLEQUIET="True"
456	fi
457
458	cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
459[DEFAULT]
460pre =
461quiet = $SINGLEQUIET
462pre_user = root
463user = $SINGLETESTUSER
464timeout = 600
465post_user = root
466post =
467outputdir = /var/tmp/test_results
468EOF
469	SINGLETESTDIR=$(dirname "$SINGLETEST")
470	SINGLETESTFILE=$(basename "$SINGLETEST")
471	SETUPSCRIPT=
472	CLEANUPSCRIPT=
473
474	if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
475		SETUPSCRIPT="setup"
476	fi
477
478	if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
479		CLEANUPSCRIPT="cleanup"
480	fi
481
482	cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
483
484[$SINGLETESTDIR]
485tests = ['$SINGLETESTFILE']
486pre = $SETUPSCRIPT
487post = $CLEANUPSCRIPT
488tags = ['functional']
489EOF
490fi
491
492#
493# Use default tag if none was specified
494#
495TAGS=${TAGS:='functional'}
496
497#
498# Attempt to locate the runfiles describing the test workload.
499#
500R=""
501IFS=,
502for RUNFILE in $RUNFILES; do
503	if [ -n "$RUNFILE" ]; then
504		SAVED_RUNFILE="$RUNFILE"
505		RUNFILE=$(find_runfile "$RUNFILE")
506		[ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
507		R="$R,$RUNFILE"
508	fi
509
510	if [ ! -r "$RUNFILE" ]; then
511		fail "Cannot read runfile: $RUNFILE"
512	fi
513done
514unset IFS
515RUNFILES=${R#,}
516
517#
518# This script should not be run as root.  Instead the test user, which may
519# be a normal user account, needs to be configured such that it can
520# run commands via sudo passwordlessly.
521#
522if [ "$(id -u)" = "0" ]; then
523	fail "This script must not be run as root."
524fi
525
526if [ "$(sudo whoami)" != "root" ]; then
527	fail "Passwordless sudo access required."
528fi
529
530#
531# Constrain the available binaries to a known set.
532#
533constrain_path
534
535#
536# Check if ksh exists
537#
538if [ "$UNAME" = "FreeBSD" ]; then
539	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
540fi
541[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
542[ -e "$STF_SUITE/include/default.cfg" ] || fail \
543    "Missing $STF_SUITE/include/default.cfg file."
544
545#
546# Verify the ZFS module stack is loaded.
547#
548if [ "$STACK_TRACER" = "yes" ]; then
549	sudo "${ZFS_SH}" -S >/dev/null 2>&1
550else
551	sudo "${ZFS_SH}" >/dev/null 2>&1
552fi
553
554#
555# Attempt to cleanup all previous state for a new test run.
556#
557if [ "$CLEANUPALL" = "yes" ]; then
558	cleanup_all
559fi
560
561#
562# By default preserve any existing pools
563# NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
564# space-delimited to newline-delimited.
565#
566if [ -z "${KEEP}" ]; then
567	KEEP="$(sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -H -o name)"
568	if [ -z "${KEEP}" ]; then
569		KEEP="rpool"
570	fi
571else
572	KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')"
573fi
574
575#
576# NOTE: The following environment variables are undocumented
577# and should be used for testing purposes only:
578#
579# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
580# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
581#
582# See libzfs/libzfs_config.c for more information.
583#
584if [ "$UNAME" = "FreeBSD" ] ; then
585	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
586else
587	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
588fi
589
590. "$STF_SUITE/include/default.cfg"
591
592#
593# No DISKS have been provided so a basic file or loopback based devices
594# must be created for the test suite to use.
595#
596if [ -z "${DISKS}" ]; then
597	#
598	# If this is a performance run, prevent accidental use of
599	# loopback devices.
600	#
601	[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
602
603	#
604	# Create sparse files for the test suite.  These may be used
605	# directory or have loopback devices layered on them.
606	#
607	for TEST_FILE in ${FILES}; do
608		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
609		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
610		    fail "Failed creating: ${TEST_FILE} ($?)"
611	done
612
613	#
614	# If requested setup loopback devices backed by the sparse files.
615	#
616	if [ "$LOOPBACK" = "yes" ]; then
617		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
618
619		for TEST_FILE in ${FILES}; do
620			if [ "$UNAME" = "FreeBSD" ] ; then
621				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
622				if [ -z "$MDDEVICE" ] ; then
623					fail "Failed: ${TEST_FILE} -> loopback"
624				fi
625				DISKS="$DISKS $MDDEVICE"
626				LOOPBACKS="$LOOPBACKS $MDDEVICE"
627			else
628				TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
629				sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
630				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
631				BASELOOPBACK="${TEST_LOOPBACK##*/}"
632				DISKS="$DISKS $BASELOOPBACK"
633				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
634			fi
635		done
636		DISKS=${DISKS# }
637		LOOPBACKS=${LOOPBACKS# }
638	else
639		DISKS="$FILES"
640	fi
641fi
642
643#
644# It may be desirable to test with fewer disks than the default when running
645# the performance tests, but the functional tests require at least three.
646#
647NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
648if [ "$TAGS" != "perf" ]; then
649	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
650fi
651
652#
653# Disable SELinux until the ZFS Test Suite has been updated accordingly.
654#
655if [ -x "$STF_PATH/setenforce" ]; then
656	sudo setenforce permissive >/dev/null 2>&1
657fi
658
659#
660# Enable internal ZFS debug log and clear it.
661#
662if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
663	sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
664	sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
665fi
666
667msg
668msg "--- Configuration ---"
669msg "Runfiles:        $RUNFILES"
670msg "STF_TOOLS:       $STF_TOOLS"
671msg "STF_SUITE:       $STF_SUITE"
672msg "STF_PATH:        $STF_PATH"
673msg "FILEDIR:         $FILEDIR"
674msg "FILES:           $FILES"
675msg "LOOPBACKS:       $LOOPBACKS"
676msg "DISKS:           $DISKS"
677msg "NUM_DISKS:       $NUM_DISKS"
678msg "FILESIZE:        $FILESIZE"
679msg "ITERATIONS:      $ITERATIONS"
680msg "TAGS:            $TAGS"
681msg "STACK_TRACER:    $STACK_TRACER"
682msg "Keep pool(s):    $KEEP"
683msg "Missing util(s): $STF_MISSING_BIN"
684msg ""
685
686export STF_TOOLS
687export STF_SUITE
688export STF_PATH
689export DISKS
690export FILEDIR
691export KEEP
692export __ZFS_POOL_EXCLUDE
693export TESTFAIL_CALLBACKS
694export PATH=$STF_PATH
695
696mktemp_file() {
697	if [ "$UNAME" = "FreeBSD" ]; then
698		mktemp -u "${FILEDIR}/$1.XXXXXX"
699	else
700		mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
701	fi
702}
703mkdir -p "$FILEDIR" || :
704RESULTS_FILE=$(mktemp_file zts-results)
705REPORT_FILE=$(mktemp_file zts-report)
706
707#
708# Run all the tests as specified.
709#
710msg "${TEST_RUNNER}" \
711    "${QUIET:+-q}" \
712    "${KMEMLEAK:+-m}" \
713    "${KMSG:+-K}" \
714    "-c \"${RUNFILES}\"" \
715    "-T \"${TAGS}\"" \
716    "-i \"${STF_SUITE}\"" \
717    "-I \"${ITERATIONS}\""
718{ ${TEST_RUNNER} \
719    ${QUIET:+-q} \
720    ${KMEMLEAK:+-m} \
721    ${KMSG:+-K} \
722    -c "${RUNFILES}" \
723    -T "${TAGS}" \
724    -i "${STF_SUITE}" \
725    -I "${ITERATIONS}" \
726    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
727read -r RUNRESULT <"$REPORT_FILE"
728
729#
730# Analyze the results.
731#
732${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
733RESULT=$?
734
735if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
736	MAYBES="$($ZTS_REPORT --list-maybes)"
737	TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
738	TEST_LIST=$(mktemp_file test-list)
739	grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
740	for test_name in $MAYBES; do
741		grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
742	done
743	{ ${TEST_RUNNER} \
744            ${QUIET:+-q} \
745            ${KMEMLEAK:+-m} \
746	    -c "${RUNFILES}" \
747	    -T "${TAGS}" \
748	    -i "${STF_SUITE}" \
749	    -I "${ITERATIONS}" \
750	    -l "${TEST_LIST}" \
751	    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
752	read -r RUNRESULT <"$REPORT_FILE"
753	#
754	# Analyze the results.
755	#
756	${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
757	RESULT=$?
758fi
759
760
761cat "$REPORT_FILE"
762
763RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
764if [ -d "$RESULTS_DIR" ]; then
765	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
766fi
767
768rm -f "$RESULTS_FILE" "$REPORT_FILE"
769
770if [ -n "$SINGLETEST" ]; then
771	rm -f "$RUNFILES" >/dev/null 2>&1
772fi
773
774[ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"
775