xref: /freebsd/sys/contrib/openzfs/scripts/zfs-tests.sh (revision dd41de95a84d979615a2ef11df6850622bf6184e)
1#!/bin/sh
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License, Version 1.0 only
7# (the "License").  You may not use this file except in compliance
8# with the License.
9#
10# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11# or http://www.opensolaris.org/os/licensing.
12# See the License for the specific language governing permissions
13# and limitations under the License.
14#
15# When distributing Covered Code, include this CDDL HEADER in each
16# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17# If applicable, add the following below this CDDL HEADER, with the
18# fields enclosed by brackets "[]" replaced with your own identifying
19# information: Portions Copyright [yyyy] [name of copyright owner]
20#
21# CDDL HEADER END
22#
23
24BASE_DIR=$(dirname "$0")
25SCRIPT_COMMON=common.sh
26if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
27. "${BASE_DIR}/${SCRIPT_COMMON}"
28else
29echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
30fi
31
32PROG=zfs-tests.sh
33VERBOSE="no"
34QUIET=""
35CLEANUP="yes"
36CLEANUPALL="no"
37LOOPBACK="yes"
38STACK_TRACER="no"
39FILESIZE="4G"
40DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
41RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
42FILEDIR=${FILEDIR:-/var/tmp}
43DISKS=${DISKS:-""}
44SINGLETEST=""
45SINGLETESTUSER="root"
46TAGS=""
47ITERATIONS=1
48ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
49ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
50UNAME=$(uname -s)
51
52# Override some defaults if on FreeBSD
53if [ "$UNAME" = "FreeBSD" ] ; then
54	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
55	LOSETUP=/sbin/mdconfig
56	DMSETUP=/sbin/gpart
57else
58	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
59	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
60	LOSETUP=${LOSETUP:-/sbin/losetup}
61	DMSETUP=${DMSETUP:-/sbin/dmsetup}
62fi
63
64#
65# Log an informational message when additional verbosity is enabled.
66#
67msg() {
68	if [ "$VERBOSE" = "yes" ]; then
69		echo "$@"
70	fi
71}
72
73#
74# Log a failure message, cleanup, and return an error.
75#
76fail() {
77	echo "$PROG: $1" >&2
78	cleanup
79	exit 1
80}
81
82cleanup_freebsd_loopback() {
83	for TEST_LOOPBACK in ${LOOPBACKS}; do
84		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
85			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
86			    echo "Failed to destroy: ${TEST_LOOPBACK}"
87		fi
88	done
89}
90
91cleanup_linux_loopback() {
92	for TEST_LOOPBACK in ${LOOPBACKS}; do
93		LOOP_DEV=$(basename "$TEST_LOOPBACK")
94		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
95		    grep "${LOOP_DEV}" | cut -f1)
96
97		if [ -n "$DM_DEV" ]; then
98			sudo "${DMSETUP}" remove "${DM_DEV}" ||
99			    echo "Failed to remove: ${DM_DEV}"
100		fi
101
102		if [ -n "${TEST_LOOPBACK}" ]; then
103			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
104			    echo "Failed to remove: ${TEST_LOOPBACK}"
105		fi
106	done
107}
108
109#
110# Attempt to remove loopback devices and files which where created earlier
111# by this script to run the test framework.  The '-k' option may be passed
112# to the script to suppress cleanup for debugging purposes.
113#
114cleanup() {
115	if [ "$CLEANUP" = "no" ]; then
116		return 0
117	fi
118
119
120	if [ "$LOOPBACK" = "yes" ]; then
121		if [ "$UNAME" = "FreeBSD" ] ; then
122			cleanup_freebsd_loopback
123		else
124			cleanup_linux_loopback
125		fi
126	fi
127
128	for TEST_FILE in ${FILES}; do
129		rm -f "${TEST_FILE}" >/dev/null 2>&1
130	done
131
132	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
133		rm -Rf "$STF_PATH"
134	fi
135}
136trap cleanup EXIT
137
138#
139# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
140# loopback devices, and files.  This is a useful way to cleanup a previous
141# test run failure which has left the system in an unknown state.  This can
142# be dangerous and should only be used in a dedicated test environment.
143#
144cleanup_all() {
145	TEST_POOLS=$(sudo "$ZPOOL" list -H -o name | grep testpool)
146	if [ "$UNAME" = "FreeBSD" ] ; then
147		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
148	else
149		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -a|grep file-vdev|cut -f1 -d:)
150	fi
151	TEST_FILES=$(ls /var/tmp/file-vdev* 2>/dev/null)
152
153	msg
154	msg "--- Cleanup ---"
155	msg "Removing pool(s):     $(echo "${TEST_POOLS}" | tr '\n' ' ')"
156	for TEST_POOL in $TEST_POOLS; do
157		sudo "$ZPOOL" destroy "${TEST_POOL}"
158	done
159
160	if [ "$UNAME" != "FreeBSD" ] ; then
161		msg "Removing dm(s):       $(sudo "${DMSETUP}" ls |
162		    grep loop | tr '\n' ' ')"
163		sudo "${DMSETUP}" remove_all
164	fi
165
166	msg "Removing loopback(s): $(echo "${TEST_LOOPBACKS}" | tr '\n' ' ')"
167	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
168		if [ "$UNAME" = "FreeBSD" ] ; then
169			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
170		else
171			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
172		fi
173	done
174
175	msg "Removing files(s):    $(echo "${TEST_FILES}" | tr '\n' ' ')"
176	for TEST_FILE in $TEST_FILES; do
177		sudo rm -f "${TEST_FILE}"
178	done
179}
180
181#
182# Takes a name as the only arguments and looks for the following variations
183# on that name.  If one is found it is returned.
184#
185# $RUNFILE_DIR/<name>
186# $RUNFILE_DIR/<name>.run
187# <name>
188# <name>.run
189#
190find_runfile() {
191	NAME=$1
192	RESULT=""
193
194	if [ -f "$RUNFILE_DIR/$NAME" ]; then
195		RESULT="$RUNFILE_DIR/$NAME"
196	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
197		RESULT="$RUNFILE_DIR/$NAME.run"
198	elif [ -f "$NAME" ]; then
199		RESULT="$NAME"
200	elif [ -f "$NAME.run" ]; then
201		RESULT="$NAME.run"
202	fi
203
204	echo "$RESULT"
205}
206
207#
208# Symlink file if it appears under any of the given paths.
209#
210create_links() {
211	dir_list="$1"
212	file_list="$2"
213
214	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
215
216	for i in $file_list; do
217		for j in $dir_list; do
218			[ ! -e "$STF_PATH/$i" ] || continue
219
220			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
221				ln -sf "$j/$i" "$STF_PATH/$i" || \
222				    fail "Couldn't link $i"
223				break
224			fi
225		done
226
227		[ ! -e "$STF_PATH/$i" ] && \
228		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
229	done
230	STF_MISSING_BIN=${STF_MISSING_BIN# }
231}
232
233#
234# Constrain the path to limit the available binaries to a known set.
235# When running in-tree a top level ./bin/ directory is created for
236# convenience, otherwise a temporary directory is used.
237#
238constrain_path() {
239	. "$STF_SUITE/include/commands.cfg"
240
241	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
242	# install to /usr/local/sbin. To avoid testing the wrong utils we
243	# need /usr/local to come before / in the path search order.
244	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
245	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
246
247	if [ "$INTREE" = "yes" ]; then
248		# Constrained path set to ./zfs/bin/
249		STF_PATH="$BIN_DIR"
250		STF_PATH_REMOVE="no"
251		STF_MISSING_BIN=""
252		if [ ! -d "$STF_PATH" ]; then
253			mkdir "$STF_PATH"
254			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
255		fi
256
257		# Special case links for standard zfs utilities
258		DIRS="$(find "$CMD_DIR" -type d \( ! -name .deps -a \
259		    ! -name .libs \) -print | tr '\n' ' ')"
260		create_links "$DIRS" "$ZFS_FILES"
261
262		# Special case links for zfs test suite utilities
263		DIRS="$(find "$STF_SUITE" -type d \( ! -name .deps -a \
264		    ! -name .libs \) -print | tr '\n' ' ')"
265		create_links "$DIRS" "$ZFSTEST_FILES"
266	else
267		# Constrained path set to /var/tmp/constrained_path.*
268		SYSTEMDIR=${SYSTEMDIR:-/var/tmp/constrained_path.XXXXXX}
269		STF_PATH=$(mktemp -d "$SYSTEMDIR")
270		STF_PATH_REMOVE="yes"
271		STF_MISSING_BIN=""
272
273		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
274
275		# Special case links for standard zfs utilities
276		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
277
278		# Special case links for zfs test suite utilities
279		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
280	fi
281
282	# Standard system utilities
283	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
284	if [ "$UNAME" = "FreeBSD" ] ; then
285		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
286	else
287		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
288	fi
289	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
290
291	# Exceptions
292	ln -fs "$STF_PATH/awk" "$STF_PATH/nawk"
293	if [ "$UNAME" = "Linux" ] ; then
294		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
295		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
296		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
297		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
298		ln -fs "$STF_PATH/exportfs" "$STF_PATH/share"
299		ln -fs "$STF_PATH/exportfs" "$STF_PATH/unshare"
300	elif [ "$UNAME" = "FreeBSD" ] ; then
301		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
302	fi
303}
304
305#
306# Output a useful usage message.
307#
308usage() {
309cat << EOF
310USAGE:
311$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
312
313DESCRIPTION:
314	ZFS Test Suite launch script
315
316OPTIONS:
317	-h          Show this message
318	-v          Verbose zfs-tests.sh output
319	-q          Quiet test-runner output
320	-x          Remove all testpools, dm, lo, and files (unsafe)
321	-k          Disable cleanup after test failure
322	-f          Use files only, disables block device tests
323	-S          Enable stack tracer (negative performance impact)
324	-c          Only create and populate constrained path
325	-n NFSFILE  Use the nfsfile to determine the NFS configuration
326	-I NUM      Number of iterations
327	-d DIR      Use DIR for files and loopback devices
328	-s SIZE     Use vdevs of SIZE (default: 4G)
329	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
330	-t PATH     Run single test at PATH relative to test suite
331	-T TAGS     Comma separated list of tags (default: 'functional')
332	-u USER     Run single test as USER (default: root)
333
334EXAMPLES:
335# Run the default (linux) suite of tests and output the configuration used.
336$0 -v
337
338# Run a smaller suite of tests designed to run more quickly.
339$0 -r linux-fast
340
341# Run a single test
342$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
343
344# Cleanup a previous run of the test suite prior to testing, run the
345# default (linux) suite of tests and perform no cleanup on exit.
346$0 -x
347
348EOF
349}
350
351while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do
352	case $OPTION in
353	h)
354		usage
355		exit 1
356		;;
357	v)
358		# shellcheck disable=SC2034
359		VERBOSE="yes"
360		;;
361	q)
362		QUIET="yes"
363		;;
364	x)
365		CLEANUPALL="yes"
366		;;
367	k)
368		CLEANUP="no"
369		;;
370	f)
371		LOOPBACK="no"
372		;;
373	S)
374		STACK_TRACER="yes"
375		;;
376	c)
377		constrain_path
378		exit
379		;;
380	n)
381		nfsfile=$OPTARG
382		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
383		export NFS=1
384		. "$nfsfile"
385		;;
386	d)
387		FILEDIR="$OPTARG"
388		;;
389	I)
390		ITERATIONS="$OPTARG"
391		if [ "$ITERATIONS" -le 0 ]; then
392			fail "Iterations must be greater than 0."
393		fi
394		;;
395	s)
396		FILESIZE="$OPTARG"
397		;;
398	r)
399		RUNFILES="$OPTARG"
400		;;
401	t)
402		if [ -n "$SINGLETEST" ]; then
403			fail "-t can only be provided once."
404		fi
405		SINGLETEST="$OPTARG"
406		;;
407	T)
408		TAGS="$OPTARG"
409		;;
410	u)
411		SINGLETESTUSER="$OPTARG"
412		;;
413	?)
414		usage
415		exit
416		;;
417	esac
418done
419
420shift $((OPTIND-1))
421
422FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
423LOOPBACKS=${LOOPBACKS:-""}
424
425if [ -n "$SINGLETEST" ]; then
426	if [ -n "$TAGS" ]; then
427		fail "-t and -T are mutually exclusive."
428	fi
429	RUNFILE_DIR="/var/tmp"
430	RUNFILES="zfs-tests.$$.run"
431	SINGLEQUIET="False"
432
433	if [ -n "$QUIET" ]; then
434		SINGLEQUIET="True"
435	fi
436
437	cat >$RUNFILE_DIR/$RUNFILES << EOF
438[DEFAULT]
439pre =
440quiet = $SINGLEQUIET
441pre_user = root
442user = $SINGLETESTUSER
443timeout = 600
444post_user = root
445post =
446outputdir = /var/tmp/test_results
447EOF
448	SINGLETESTDIR=$(dirname "$SINGLETEST")
449	SINGLETESTFILE=$(basename "$SINGLETEST")
450	SETUPSCRIPT=
451	CLEANUPSCRIPT=
452
453	if [ -f "$STF_SUITE/$SINGLETESTDIR/setup.ksh" ]; then
454		SETUPSCRIPT="setup"
455	fi
456
457	if [ -f "$STF_SUITE/$SINGLETESTDIR/cleanup.ksh" ]; then
458		CLEANUPSCRIPT="cleanup"
459	fi
460
461	cat >>$RUNFILE_DIR/$RUNFILES << EOF
462
463[$SINGLETESTDIR]
464tests = ['$SINGLETESTFILE']
465pre = $SETUPSCRIPT
466post = $CLEANUPSCRIPT
467tags = ['functional']
468EOF
469fi
470
471#
472# Use default tag if none was specified
473#
474TAGS=${TAGS:='functional'}
475
476#
477# Attempt to locate the runfiles describing the test workload.
478#
479R=""
480IFS=,
481for RUNFILE in $RUNFILES; do
482	if [ -n "$RUNFILE" ]; then
483		SAVED_RUNFILE="$RUNFILE"
484		RUNFILE=$(find_runfile "$RUNFILE")
485		[ -z "$RUNFILE" ] && fail "Cannot find runfile: $SAVED_RUNFILE"
486		R="$R,$RUNFILE"
487	fi
488
489	if [ ! -r "$RUNFILE" ]; then
490		fail "Cannot read runfile: $RUNFILE"
491	fi
492done
493unset IFS
494RUNFILES=${R#,}
495
496#
497# This script should not be run as root.  Instead the test user, which may
498# be a normal user account, needs to be configured such that it can
499# run commands via sudo passwordlessly.
500#
501if [ "$(id -u)" = "0" ]; then
502	fail "This script must not be run as root."
503fi
504
505if [ "$(sudo whoami)" != "root" ]; then
506	fail "Passwordless sudo access required."
507fi
508
509#
510# Constrain the available binaries to a known set.
511#
512constrain_path
513
514#
515# Check if ksh exists
516#
517if [ "$UNAME" = "FreeBSD" ]; then
518	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
519fi
520[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
521[ -e "$STF_SUITE/include/default.cfg" ] || fail \
522    "Missing $STF_SUITE/include/default.cfg file."
523
524#
525# Verify the ZFS module stack is loaded.
526#
527if [ "$STACK_TRACER" = "yes" ]; then
528	sudo "${ZFS_SH}" -S >/dev/null 2>&1
529else
530	sudo "${ZFS_SH}" >/dev/null 2>&1
531fi
532
533#
534# Attempt to cleanup all previous state for a new test run.
535#
536if [ "$CLEANUPALL" = "yes" ]; then
537	cleanup_all
538fi
539
540#
541# By default preserve any existing pools
542# NOTE: Since 'zpool list' outputs a newline-delimited list convert $KEEP from
543# space-delimited to newline-delimited.
544#
545if [ -z "${KEEP}" ]; then
546	KEEP="$(sudo "$ZPOOL" list -H -o name)"
547	if [ -z "${KEEP}" ]; then
548		KEEP="rpool"
549	fi
550else
551	KEEP="$(echo "$KEEP" | tr '[:blank:]' '\n')"
552fi
553
554#
555# NOTE: The following environment variables are undocumented
556# and should be used for testing purposes only:
557#
558# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
559# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
560#
561# See libzfs/libzfs_config.c for more information.
562#
563if [ "$UNAME" = "FreeBSD" ] ; then
564	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | tr -s '\n' ' ')"
565else
566	__ZFS_POOL_EXCLUDE="$(echo "$KEEP" | sed ':a;N;s/\n/ /g;ba')"
567fi
568
569. "$STF_SUITE/include/default.cfg"
570
571msg
572msg "--- Configuration ---"
573msg "Runfiles:        $RUNFILES"
574msg "STF_TOOLS:       $STF_TOOLS"
575msg "STF_SUITE:       $STF_SUITE"
576msg "STF_PATH:        $STF_PATH"
577
578#
579# No DISKS have been provided so a basic file or loopback based devices
580# must be created for the test suite to use.
581#
582if [ -z "${DISKS}" ]; then
583	#
584	# Create sparse files for the test suite.  These may be used
585	# directory or have loopback devices layered on them.
586	#
587	for TEST_FILE in ${FILES}; do
588		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
589		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
590		    fail "Failed creating: ${TEST_FILE} ($?)"
591	done
592
593	#
594	# If requested setup loopback devices backed by the sparse files.
595	#
596	if [ "$LOOPBACK" = "yes" ]; then
597		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
598
599		for TEST_FILE in ${FILES}; do
600			if [ "$UNAME" = "FreeBSD" ] ; then
601				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
602				if [ -z "$MDDEVICE" ] ; then
603					fail "Failed: ${TEST_FILE} -> loopback"
604				fi
605				DISKS="$DISKS $MDDEVICE"
606				LOOPBACKS="$LOOPBACKS $MDDEVICE"
607			else
608				TEST_LOOPBACK=$(sudo "${LOSETUP}" -f)
609				sudo "${LOSETUP}" "${TEST_LOOPBACK}" "${TEST_FILE}" ||
610				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
611				BASELOOPBACK=$(basename "$TEST_LOOPBACK")
612				DISKS="$DISKS $BASELOOPBACK"
613				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
614			fi
615		done
616		DISKS=${DISKS# }
617		LOOPBACKS=${LOOPBACKS# }
618	else
619		DISKS="$FILES"
620	fi
621fi
622
623NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
624[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
625
626#
627# Disable SELinux until the ZFS Test Suite has been updated accordingly.
628#
629if [ -x "$STF_PATH/setenforce" ]; then
630	sudo setenforce permissive >/dev/null 2>&1
631fi
632
633#
634# Enable internal ZFS debug log and clear it.
635#
636if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
637	sudo /bin/sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
638	sudo /bin/sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
639fi
640
641msg "FILEDIR:         $FILEDIR"
642msg "FILES:           $FILES"
643msg "LOOPBACKS:       $LOOPBACKS"
644msg "DISKS:           $DISKS"
645msg "NUM_DISKS:       $NUM_DISKS"
646msg "FILESIZE:        $FILESIZE"
647msg "ITERATIONS:      $ITERATIONS"
648msg "TAGS:            $TAGS"
649msg "STACK_TRACER:    $STACK_TRACER"
650msg "Keep pool(s):    $KEEP"
651msg "Missing util(s): $STF_MISSING_BIN"
652msg ""
653
654export STF_TOOLS
655export STF_SUITE
656export STF_PATH
657export DISKS
658export FILEDIR
659export KEEP
660export __ZFS_POOL_EXCLUDE
661export TESTFAIL_CALLBACKS
662export PATH=$STF_PATH
663
664if [ "$UNAME" = "FreeBSD" ] ; then
665	mkdir -p "$FILEDIR" || true
666	RESULTS_FILE=$(mktemp -u "${FILEDIR}/zts-results.XXXXXX")
667	REPORT_FILE=$(mktemp -u "${FILEDIR}/zts-report.XXXXXX")
668else
669	RESULTS_FILE=$(mktemp -u -t zts-results.XXXXXX -p "$FILEDIR")
670	REPORT_FILE=$(mktemp -u -t zts-report.XXXXXX -p "$FILEDIR")
671fi
672
673#
674# Run all the tests as specified.
675#
676msg "${TEST_RUNNER} ${QUIET:+-q}" \
677    "-c \"${RUNFILES}\"" \
678    "-T \"${TAGS}\"" \
679    "-i \"${STF_SUITE}\"" \
680    "-I \"${ITERATIONS}\""
681${TEST_RUNNER} ${QUIET:+-q} \
682    -c "${RUNFILES}" \
683    -T "${TAGS}" \
684    -i "${STF_SUITE}" \
685    -I "${ITERATIONS}" \
686    2>&1 | tee "$RESULTS_FILE"
687
688#
689# Analyze the results.
690#
691${ZTS_REPORT} "$RESULTS_FILE" >"$REPORT_FILE"
692RESULT=$?
693cat "$REPORT_FILE"
694
695RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
696if [ -d "$RESULTS_DIR" ]; then
697	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
698fi
699
700rm -f "$RESULTS_FILE" "$REPORT_FILE"
701
702if [ -n "$SINGLETEST" ]; then
703	rm -f "$RUNFILES" >/dev/null 2>&1
704fi
705
706exit ${RESULT}
707