xref: /freebsd/sys/contrib/openzfs/scripts/zfs-tests.sh (revision 61145dc2b94f12f6a47344fb9aac702321880e43)
1#!/usr/bin/env bash
2# SPDX-License-Identifier: CDDL-1.0
3# shellcheck disable=SC2154
4# shellcheck disable=SC2292
5#
6# CDDL HEADER START
7#
8# The contents of this file are subject to the terms of the
9# Common Development and Distribution License, Version 1.0 only
10# (the "License").  You may not use this file except in compliance
11# with the License.
12#
13# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
14# or https://opensource.org/licenses/CDDL-1.0.
15# See the License for the specific language governing permissions
16# and limitations under the License.
17#
18# When distributing Covered Code, include this CDDL HEADER in each
19# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
20# If applicable, add the following below this CDDL HEADER, with the
21# fields enclosed by brackets "[]" replaced with your own identifying
22# information: Portions Copyright [yyyy] [name of copyright owner]
23#
24# CDDL HEADER END
25#
26
27#
28# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
29#
30
31SCRIPT_COMMON=${SCRIPT_COMMON:-${0%/*}/common.sh}
32. "${SCRIPT_COMMON}" || exit
33
34PROG=zfs-tests.sh
35VERBOSE="no"
36QUIET=""
37DEBUG=""
38CLEANUP="yes"
39CLEANUPALL="no"
40KMSG=""
41LOOPBACK="yes"
42STACK_TRACER="no"
43FILESIZE="4G"
44DEFAULT_RUNFILES="common.run,$(uname | tr '[:upper:]' '[:lower:]').run"
45RUNFILES=${RUNFILES:-$DEFAULT_RUNFILES}
46FILEDIR=${FILEDIR:-/var/tmp}
47DISKS=${DISKS:-""}
48SINGLETEST=""
49SINGLETESTUSER="root"
50TAGS=""
51ITERATIONS=1
52ZFS_DBGMSG="$STF_SUITE/callbacks/zfs_dbgmsg.ksh"
53ZFS_DMESG="$STF_SUITE/callbacks/zfs_dmesg.ksh"
54UNAME=$(uname)
55RERUN=""
56KMEMLEAK=""
57
58# Override some defaults if on FreeBSD
59if [ "$UNAME" = "FreeBSD" ] ; then
60	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DMESG"}
61	LOSETUP=/sbin/mdconfig
62	DMSETUP=/sbin/gpart
63else
64	ZFS_MMP="$STF_SUITE/callbacks/zfs_mmp.ksh"
65	TESTFAIL_CALLBACKS=${TESTFAIL_CALLBACKS:-"$ZFS_DBGMSG:$ZFS_DMESG:$ZFS_MMP"}
66	LOSETUP=${LOSETUP:-/sbin/losetup}
67	DMSETUP=${DMSETUP:-/sbin/dmsetup}
68fi
69
70#
71# Log an informational message when additional verbosity is enabled.
72#
73msg() {
74	if [ "$VERBOSE" = "yes" ]; then
75		echo "$@"
76	fi
77}
78
79#
80# Log a failure message, cleanup, and return an error.
81#
82fail() {
83	echo "$PROG: $1" >&2
84	cleanup
85	exit 1
86}
87
88cleanup_freebsd_loopback() {
89	for TEST_LOOPBACK in ${LOOPBACKS}; do
90		if [ -c "/dev/${TEST_LOOPBACK}" ]; then
91			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}" ||
92			    echo "Failed to destroy: ${TEST_LOOPBACK}"
93		fi
94	done
95}
96
97cleanup_linux_loopback() {
98	for TEST_LOOPBACK in ${LOOPBACKS}; do
99		LOOP_DEV="${TEST_LOOPBACK##*/}"
100		DM_DEV=$(sudo "${DMSETUP}" ls 2>/dev/null | \
101		    awk -v l="${LOOP_DEV}" '$0 ~ l {print $1}')
102
103		if [ -n "$DM_DEV" ]; then
104			sudo "${DMSETUP}" remove "${DM_DEV}" ||
105			    echo "Failed to remove: ${DM_DEV}"
106		fi
107
108		if [ -n "${TEST_LOOPBACK}" ]; then
109			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}" ||
110			    echo "Failed to remove: ${TEST_LOOPBACK}"
111		fi
112	done
113}
114
115#
116# Attempt to remove loopback devices and files which where created earlier
117# by this script to run the test framework.  The '-k' option may be passed
118# to the script to suppress cleanup for debugging purposes.
119#
120cleanup() {
121	if [ "$CLEANUP" = "no" ]; then
122		return 0
123	fi
124
125
126	if [ "$LOOPBACK" = "yes" ]; then
127		if [ "$UNAME" = "FreeBSD" ] ; then
128			cleanup_freebsd_loopback
129		else
130			cleanup_linux_loopback
131		fi
132	fi
133
134	# shellcheck disable=SC2086
135	rm -f ${FILES} >/dev/null 2>&1
136
137	if [ "$STF_PATH_REMOVE" = "yes" ] && [ -d "$STF_PATH" ]; then
138		rm -Rf "$STF_PATH"
139	fi
140}
141trap cleanup EXIT
142
143#
144# Attempt to remove all testpools (testpool.XXX), unopened dm devices,
145# loopback devices, and files.  This is a useful way to cleanup a previous
146# test run failure which has left the system in an unknown state.  This can
147# be dangerous and should only be used in a dedicated test environment.
148#
149cleanup_all() {
150	TEST_POOLS=$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | grep testpool)
151	if [ "$UNAME" = "FreeBSD" ] ; then
152		TEST_LOOPBACKS=$(sudo "${LOSETUP}" -l)
153	else
154		TEST_LOOPBACKS=$("${LOSETUP}" -a | awk -F: '/file-vdev/ {print $1}')
155	fi
156	TEST_FILES=$(ls "${FILEDIR}"/file-vdev* 2>/dev/null)
157
158	msg
159	msg "--- Cleanup ---"
160	# shellcheck disable=2116,2086
161	msg "Removing pool(s):     $(echo ${TEST_POOLS})"
162	for TEST_POOL in $TEST_POOLS; do
163		sudo env ASAN_OPTIONS=detect_leaks=false "$ZPOOL" destroy "${TEST_POOL}"
164	done
165
166	if [ "$UNAME" != "FreeBSD" ] ; then
167		msg "Removing all dm(s):   $(sudo "${DMSETUP}" ls |
168		    grep loop | tr '\n' ' ')"
169		sudo "${DMSETUP}" remove_all
170	fi
171
172	# shellcheck disable=2116,2086
173	msg "Removing loopback(s): $(echo ${TEST_LOOPBACKS})"
174	for TEST_LOOPBACK in $TEST_LOOPBACKS; do
175		if [ "$UNAME" = "FreeBSD" ] ; then
176			sudo "${LOSETUP}" -d -u "${TEST_LOOPBACK}"
177		else
178			sudo "${LOSETUP}" -d "${TEST_LOOPBACK}"
179		fi
180	done
181
182	# shellcheck disable=2116,2086
183	msg "Removing files(s):    $(echo ${TEST_FILES})"
184	# shellcheck disable=2086
185	sudo rm -f ${TEST_FILES}
186}
187
188#
189# Takes a name as the only arguments and looks for the following variations
190# on that name.  If one is found it is returned.
191#
192# $RUNFILE_DIR/<name>
193# $RUNFILE_DIR/<name>.run
194# <name>
195# <name>.run
196#
197find_runfile() {
198	NAME=$1
199
200	if [ -f "$RUNFILE_DIR/$NAME" ]; then
201		echo "$RUNFILE_DIR/$NAME"
202	elif [ -f "$RUNFILE_DIR/$NAME.run" ]; then
203		echo "$RUNFILE_DIR/$NAME.run"
204	elif [ -f "$NAME" ]; then
205		echo "$NAME"
206	elif [ -f "$NAME.run" ]; then
207		echo "$NAME.run"
208	else
209		return 1
210	fi
211}
212
213# Given a TAGS with a format like "1/3" or "2/3" then divide up the test list
214# into portions and print that portion.  So "1/3" for "the first third of the
215# test tags".
216#
217#
218split_tags() {
219	# Get numerator and denominator
220	NUM=$(echo "$TAGS" | cut -d/ -f1)
221	DEN=$(echo "$TAGS" | cut -d/ -f2)
222	# At the point this is called, RUNFILES will contain a comma separated
223	# list of full paths to the runfiles, like:
224	#
225	# "/home/hutter/qemu/tests/runfiles/common.run,/home/hutter/qemu/tests/runfiles/linux.run"
226	#
227	# So to get tags for our selected tests we do:
228	#
229	# 1. Remove unneeded chars: [],\
230	# 2. Print out the last field of each tag line.  This will be the tag
231	#    for the test (like 'zpool_add').
232	# 3. Remove duplicates between the runfiles.  If the same tag is defined
233	#    in multiple runfiles, then when you do '-T <tag>' ZTS is smart
234	#    enough to know to run the tag in each runfile.  So '-T zpool_add'
235	#    will run the zpool_add from common.run and linux.run.
236	# 4. Ignore the 'functional' tag since we only want individual tests
237	# 5. Print out the tests in our faction of all tests.  This uses modulus
238	#    so "1/3" will run tests 1,3,6,9 etc.  That way the tests are
239	#    interleaved so, say, "3/4" isn't running all the zpool_* tests that
240	#    appear alphabetically at the end.
241	# 6. Remove trailing comma from list
242	#
243	# TAGS will then look like:
244	#
245	# "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..."
246
247	# Change the comma to a space for easy processing
248	_RUNFILES=${RUNFILES//","/" "}
249	# shellcheck disable=SC2002,SC2086
250	cat $_RUNFILES | tr -d "[],\'" | awk '/tags = /{print $NF}' | sort | \
251		uniq | grep -v functional | \
252		awk -v num="$NUM" -v den="$DEN" '{ if(NR % den == (num - 1)) {printf "%s,",$0}}' | \
253		sed -E 's/,$//'
254}
255
256#
257# Symlink file if it appears under any of the given paths.
258#
259create_links() {
260	dir_list="$1"
261	file_list="$2"
262
263	[ -n "$STF_PATH" ] || fail "STF_PATH wasn't correctly set"
264
265	for i in $file_list; do
266		for j in $dir_list; do
267			[ ! -e "$STF_PATH/$i" ] || continue
268
269			if [ ! -d "$j/$i" ] && [ -e "$j/$i" ]; then
270				ln -sf "$j/$i" "$STF_PATH/$i" || \
271				    fail "Couldn't link $i"
272				break
273			fi
274		done
275
276		[ ! -e "$STF_PATH/$i" ] && \
277		    STF_MISSING_BIN="$STF_MISSING_BIN $i"
278	done
279	STF_MISSING_BIN=${STF_MISSING_BIN# }
280}
281
282#
283# Constrain the path to limit the available binaries to a known set.
284# When running in-tree a top level ./bin/ directory is created for
285# convenience, otherwise a temporary directory is used.
286#
287constrain_path() {
288	. "$STF_SUITE/include/commands.cfg"
289
290	# On FreeBSD, base system zfs utils are in /sbin and OpenZFS utils
291	# install to /usr/local/sbin. To avoid testing the wrong utils we
292	# need /usr/local to come before / in the path search order.
293	SYSTEM_DIRS="/usr/local/bin /usr/local/sbin"
294	SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR"
295
296	if [ "$INTREE" = "yes" ]; then
297		# Constrained path set to $(top_builddir)/tests/zfs-tests/bin
298		STF_PATH="$BIN_DIR"
299		STF_PATH_REMOVE="no"
300		STF_MISSING_BIN=""
301		if [ ! -d "$STF_PATH" ]; then
302			mkdir "$STF_PATH"
303			chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
304		fi
305
306		# Special case links for standard zfs utilities
307		create_links "$CMD_DIR" "$ZFS_FILES"
308
309		# Special case links for zfs test suite utilities
310		create_links "$CMD_DIR/tests/zfs-tests/cmd" "$ZFSTEST_FILES"
311	else
312		# Constrained path set to $FILEDIR/constrained_path.*
313		SYSTEMDIR=${SYSTEMDIR:-$FILEDIR/constrained_path.XXXXXX}
314		STF_PATH=$(mktemp -d "$SYSTEMDIR")
315		STF_PATH_REMOVE="yes"
316		STF_MISSING_BIN=""
317
318		chmod 755 "$STF_PATH" || fail "Couldn't chmod $STF_PATH"
319
320		# Special case links for standard zfs utilities
321		create_links "$SYSTEM_DIRS" "$ZFS_FILES"
322
323		# Special case links for zfs test suite utilities
324		create_links "$STF_SUITE/bin" "$ZFSTEST_FILES"
325	fi
326
327	# Standard system utilities
328	SYSTEM_FILES="$SYSTEM_FILES_COMMON"
329	if [ "$UNAME" = "FreeBSD" ] ; then
330		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD"
331	else
332		SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX"
333	fi
334	create_links "$SYSTEM_DIRS" "$SYSTEM_FILES"
335
336	# Exceptions
337	if [ "$UNAME" = "Linux" ] ; then
338		ln -fs /sbin/fsck.ext4 "$STF_PATH/fsck"
339		ln -fs /sbin/mkfs.ext4 "$STF_PATH/newfs"
340		ln -fs "$STF_PATH/gzip" "$STF_PATH/compress"
341		ln -fs "$STF_PATH/gunzip" "$STF_PATH/uncompress"
342	elif [ "$UNAME" = "FreeBSD" ] ; then
343		ln -fs /usr/local/bin/ksh93 "$STF_PATH/ksh"
344	fi
345}
346
347#
348# Output a useful usage message.
349#
350usage() {
351cat << EOF
352USAGE:
353$0 [-hvqxkfS] [-s SIZE] [-r RUNFILES] [-t PATH] [-u USER]
354
355DESCRIPTION:
356	ZFS Test Suite launch script
357
358OPTIONS:
359	-h          Show this message
360	-v          Verbose zfs-tests.sh output
361	-q          Quiet test-runner output
362	-D          Debug; show all test output immediately (noisy)
363	-x          Remove all testpools, dm, lo, and files (unsafe)
364	-k          Disable cleanup after test failure
365	-K          Log test names to /dev/kmsg
366	-f          Use files only, disables block device tests
367	-S          Enable stack tracer (negative performance impact)
368	-c          Only create and populate constrained path
369	-R          Automatically rerun failing tests
370	-m          Enable kmemleak reporting (Linux only)
371	-n NFSFILE  Use the nfsfile to determine the NFS configuration
372	-I NUM      Number of iterations
373	-d DIR      Use world-writable DIR for files and loopback devices
374	-s SIZE     Use vdevs of SIZE (default: 4G)
375	-r RUNFILES Run tests in RUNFILES (default: ${DEFAULT_RUNFILES})
376	-t PATH|NAME  Run single test at PATH relative to test suite,
377	                or search for test by NAME
378	-T TAGS     Comma separated list of tags (default: 'functional')
379	            Alternately, specify a fraction like "1/3" or "2/3" to
380		     run the first third of tests or 2nd third of the tests.  This
381		     is useful for splitting up the test amongst different
382		     runners.
383	-u USER     Run single test as USER (default: root)
384
385EXAMPLES:
386# Run the default ${DEFAULT_RUNFILES//\.run/} suite of tests and output the configuration used.
387$0 -v
388
389# Run a smaller suite of tests designed to run more quickly.
390$0 -r linux-fast
391
392# Run a single test
393$0 -t tests/functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh
394
395# Run a single test by name
396$0 -t zfs_bookmark_cliargs
397
398# Cleanup a previous run of the test suite prior to testing, run the
399# default ${DEFAULT_RUNFILES//\.run//} suite of tests and perform no cleanup on exit.
400$0 -x
401
402EOF
403}
404
405while getopts 'hvqxkKfScRmn:d:Ds:r:?t:T:u:I:' OPTION; do
406	case $OPTION in
407	h)
408		usage
409		exit 1
410		;;
411	v)
412		VERBOSE="yes"
413		;;
414	q)
415		QUIET="yes"
416		;;
417	x)
418		CLEANUPALL="yes"
419		;;
420	k)
421		CLEANUP="no"
422		;;
423	K)
424		KMSG="yes"
425		;;
426	f)
427		LOOPBACK="no"
428		;;
429	S)
430		STACK_TRACER="yes"
431		;;
432	c)
433		constrain_path
434		exit
435		;;
436	R)
437		RERUN="yes"
438		;;
439	m)
440		KMEMLEAK="yes"
441		;;
442	n)
443		nfsfile=$OPTARG
444		[ -f "$nfsfile" ] || fail "Cannot read file: $nfsfile"
445		export NFS=1
446		. "$nfsfile"
447		;;
448	d)
449		FILEDIR="$OPTARG"
450		;;
451	D)
452		DEBUG="yes"
453		;;
454	I)
455		ITERATIONS="$OPTARG"
456		if [ "$ITERATIONS" -le 0 ]; then
457			fail "Iterations must be greater than 0."
458		fi
459		;;
460	s)
461		FILESIZE="$OPTARG"
462		;;
463	r)
464		RUNFILES="$OPTARG"
465		;;
466	t)
467		if [ -n "$SINGLETEST" ]; then
468			fail "-t can only be provided once."
469		fi
470		SINGLETEST="$OPTARG"
471		;;
472	T)
473		TAGS="$OPTARG"
474		;;
475	u)
476		SINGLETESTUSER="$OPTARG"
477		;;
478	?)
479		usage
480		exit
481		;;
482	*)
483		;;
484	esac
485done
486
487shift $((OPTIND-1))
488
489FILES=${FILES:-"$FILEDIR/file-vdev0 $FILEDIR/file-vdev1 $FILEDIR/file-vdev2"}
490LOOPBACKS=${LOOPBACKS:-""}
491
492if [ -n "$SINGLETEST" ]; then
493	if [ -n "$TAGS" ]; then
494		fail "-t and -T are mutually exclusive."
495	fi
496	RUNFILE_DIR="$FILEDIR"
497	RUNFILES="zfs-tests.$$.run"
498	[ -n "$QUIET" ] && SINGLEQUIET="True" || SINGLEQUIET="False"
499
500	cat >"${RUNFILE_DIR}/${RUNFILES}" << EOF
501[DEFAULT]
502pre =
503quiet = $SINGLEQUIET
504pre_user = root
505user = $SINGLETESTUSER
506timeout = 600
507post_user = root
508post =
509EOF
510	if [ "$SINGLETEST" = "${SINGLETEST%/*}" ] ; then
511		NEWSINGLETEST=$(find "$STF_SUITE" -name "$SINGLETEST*" -print -quit)
512		if [ -z "$NEWSINGLETEST" ] ; then
513			fail "couldn't find test matching '$SINGLETEST'"
514		fi
515		SINGLETEST=$NEWSINGLETEST
516	fi
517
518	SINGLETESTDIR="${SINGLETEST%/*}"
519	SETUPDIR="$SINGLETESTDIR"
520	[ "${SETUPDIR#/}" = "$SETUPDIR" ] && SETUPDIR="$STF_SUITE/$SINGLETESTDIR"
521	[ -x "$SETUPDIR/setup.ksh"   ] && SETUPSCRIPT="setup"     || SETUPSCRIPT=
522	[ -x "$SETUPDIR/cleanup.ksh" ] && CLEANUPSCRIPT="cleanup" || CLEANUPSCRIPT=
523
524	SINGLETESTFILE="${SINGLETEST##*/}"
525	cat >>"${RUNFILE_DIR}/${RUNFILES}" << EOF
526
527[$SINGLETESTDIR]
528tests = ['$SINGLETESTFILE']
529pre = $SETUPSCRIPT
530post = $CLEANUPSCRIPT
531tags = ['functional']
532EOF
533fi
534
535#
536# Use default tag if none was specified
537#
538TAGS=${TAGS:='functional'}
539
540
541
542#
543# Attempt to locate the runfiles describing the test workload.
544#
545R=""
546IFS=,
547for RUNFILE in $RUNFILES; do
548	if [ -n "$RUNFILE" ]; then
549		SAVED_RUNFILE="$RUNFILE"
550		RUNFILE=$(find_runfile "$RUNFILE") ||
551			fail "Cannot find runfile: $SAVED_RUNFILE"
552		R="$R,$RUNFILE"
553	fi
554
555	if [ ! -r "$RUNFILE" ]; then
556		fail "Cannot read runfile: $RUNFILE"
557	fi
558done
559unset IFS
560RUNFILES=${R#,}
561
562# The tag can be a fraction to indicate which portion of ZTS to run, Like
563#
564# 	"1/3": Run first one third of all tests in runfiles
565#	"2/3": Run second one third of all test in runfiles
566#	"6/10": Run 6th tenth of all tests in runfiles
567#
568# This is useful for splitting up the test across multiple runners.
569#
570# After this code block, TAGS will be transformed from something like
571# "1/3" to a comma separate taglist, like:
572#
573# "append,atime,bootfs,cachefile,checksum,cp_files,deadman,dos_attributes, ..."
574#
575if echo "$TAGS" | grep -Eq '^[0-9]+/[0-9]+$' ; then
576	TAGS=$(split_tags)
577fi
578
579#
580# This script should not be run as root.  Instead the test user, which may
581# be a normal user account, needs to be configured such that it can
582# run commands via sudo passwordlessly.
583#
584if [ "$(id -u)" = "0" ]; then
585	fail "This script must not be run as root."
586fi
587
588if [ "$(sudo id -un)" != "root" ]; then
589	fail "Passwordless sudo access required."
590fi
591
592#
593# Constrain the available binaries to a known set.
594#
595constrain_path
596
597#
598# Check if ksh exists
599#
600if [ "$UNAME" = "FreeBSD" ]; then
601	sudo ln -fs /usr/local/bin/ksh93 /bin/ksh
602fi
603[ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
604[ -e "$STF_SUITE/include/default.cfg" ] || fail \
605    "Missing $STF_SUITE/include/default.cfg file."
606
607#
608# Verify the ZFS module stack is loaded.
609#
610if [ "$STACK_TRACER" = "yes" ]; then
611	sudo "${ZFS_SH}" -S >/dev/null 2>&1
612else
613	sudo "${ZFS_SH}" >/dev/null 2>&1
614fi
615
616#
617# Attempt to cleanup all previous state for a new test run.
618#
619if [ "$CLEANUPALL" = "yes" ]; then
620	cleanup_all
621fi
622
623#
624# By default preserve any existing pools
625#
626if [ -z "${KEEP}" ]; then
627	KEEP="$(ASAN_OPTIONS=detect_leaks=false "$ZPOOL" list -Ho name | tr -s '[:space:]' ' ')"
628	if [ -z "${KEEP}" ]; then
629		KEEP="rpool"
630	fi
631else
632	KEEP="$(echo "$KEEP" | tr -s '[:space:]' ' ')"
633fi
634
635#
636# NOTE: The following environment variables are undocumented
637# and should be used for testing purposes only:
638#
639# __ZFS_POOL_EXCLUDE - don't iterate over the pools it lists
640# __ZFS_POOL_RESTRICT - iterate only over the pools it lists
641#
642# See libzfs/libzfs_config.c for more information.
643#
644__ZFS_POOL_EXCLUDE="$KEEP"
645
646. "$STF_SUITE/include/default.cfg"
647
648#
649# No DISKS have been provided so a basic file or loopback based devices
650# must be created for the test suite to use.
651#
652if [ -z "${DISKS}" ]; then
653	#
654	# If this is a performance run, prevent accidental use of
655	# loopback devices.
656	#
657	[ "$TAGS" = "perf" ] && fail "Running perf tests without disks."
658
659	#
660	# Create sparse files for the test suite.  These may be used
661	# directory or have loopback devices layered on them.
662	#
663	for TEST_FILE in ${FILES}; do
664		[ -f "$TEST_FILE" ] && fail "Failed file exists: ${TEST_FILE}"
665		truncate -s "${FILESIZE}" "${TEST_FILE}" ||
666		    fail "Failed creating: ${TEST_FILE} ($?)"
667	done
668
669	#
670	# If requested setup loopback devices backed by the sparse files.
671	#
672	if [ "$LOOPBACK" = "yes" ]; then
673		test -x "$LOSETUP" || fail "$LOSETUP utility must be installed"
674
675		for TEST_FILE in ${FILES}; do
676			if [ "$UNAME" = "FreeBSD" ] ; then
677				MDDEVICE=$(sudo "${LOSETUP}" -a -t vnode -f "${TEST_FILE}")
678				if [ -z "$MDDEVICE" ] ; then
679					fail "Failed: ${TEST_FILE} -> loopback"
680				fi
681				DISKS="$DISKS $MDDEVICE"
682				LOOPBACKS="$LOOPBACKS $MDDEVICE"
683			else
684				TEST_LOOPBACK=$(sudo "${LOSETUP}" --show -f "${TEST_FILE}") ||
685				    fail "Failed: ${TEST_FILE} -> ${TEST_LOOPBACK}"
686				BASELOOPBACK="${TEST_LOOPBACK##*/}"
687				DISKS="$DISKS $BASELOOPBACK"
688				LOOPBACKS="$LOOPBACKS $TEST_LOOPBACK"
689			fi
690		done
691		DISKS=${DISKS# }
692		LOOPBACKS=${LOOPBACKS# }
693	else
694		DISKS="$FILES"
695	fi
696fi
697
698#
699# It may be desirable to test with fewer disks than the default when running
700# the performance tests, but the functional tests require at least three.
701#
702NUM_DISKS=$(echo "${DISKS}" | awk '{print NF}')
703if [ "$TAGS" != "perf" ]; then
704	[ "$NUM_DISKS" -lt 3 ] && fail "Not enough disks ($NUM_DISKS/3 minimum)"
705fi
706
707#
708# Disable SELinux until the ZFS Test Suite has been updated accordingly.
709#
710if command -v setenforce >/dev/null; then
711	sudo setenforce permissive >/dev/null 2>&1
712fi
713
714#
715# Enable internal ZFS debug log and clear it.
716#
717if [ -e /sys/module/zfs/parameters/zfs_dbgmsg_enable ]; then
718	sudo sh -c "echo 1 >/sys/module/zfs/parameters/zfs_dbgmsg_enable"
719	sudo sh -c "echo 0 >/proc/spl/kstat/zfs/dbgmsg"
720fi
721
722#
723# Set TMPDIR. Some tests run mktemp, and we want those files contained to
724# the work dir the same as any other.
725#
726export TMPDIR="$FILEDIR"
727
728msg
729msg "--- Configuration ---"
730msg "Runfiles:        $RUNFILES"
731msg "STF_TOOLS:       $STF_TOOLS"
732msg "STF_SUITE:       $STF_SUITE"
733msg "STF_PATH:        $STF_PATH"
734msg "FILEDIR:         $FILEDIR"
735msg "TMPDIR:          $TMPDIR"
736msg "FILES:           $FILES"
737msg "LOOPBACKS:       $LOOPBACKS"
738msg "DISKS:           $DISKS"
739msg "NUM_DISKS:       $NUM_DISKS"
740msg "FILESIZE:        $FILESIZE"
741msg "ITERATIONS:      $ITERATIONS"
742msg "TAGS:            $TAGS"
743msg "STACK_TRACER:    $STACK_TRACER"
744msg "Keep pool(s):    $KEEP"
745msg "Missing util(s): $STF_MISSING_BIN"
746msg ""
747
748export STF_TOOLS
749export STF_SUITE
750export STF_PATH
751export DISKS
752export FILEDIR
753export KEEP
754export __ZFS_POOL_EXCLUDE
755export TESTFAIL_CALLBACKS
756
757mktemp_file() {
758	if [ "$UNAME" = "FreeBSD" ]; then
759		mktemp -u "${FILEDIR}/$1.XXXXXX"
760	else
761		mktemp -ut "$1.XXXXXX" -p "$FILEDIR"
762	fi
763}
764mkdir -p "$FILEDIR" || :
765RESULTS_FILE=$(mktemp_file zts-results)
766REPORT_FILE=$(mktemp_file zts-report)
767
768#
769# Run all the tests as specified.
770#
771msg "${TEST_RUNNER}" \
772    "${QUIET:+-q}" \
773    "${DEBUG:+-D}" \
774    "${KMEMLEAK:+-m}" \
775    "${KMSG:+-K}" \
776    "-c \"${RUNFILES}\"" \
777    "-T \"${TAGS}\"" \
778    "-i \"${STF_SUITE}\"" \
779    "-I \"${ITERATIONS}\""
780{ PATH=$STF_PATH \
781    ${TEST_RUNNER} \
782    ${QUIET:+-q} \
783    ${DEBUG:+-D} \
784    ${KMEMLEAK:+-m} \
785    ${KMSG:+-K} \
786    -c "${RUNFILES}" \
787    -T "${TAGS}" \
788    -i "${STF_SUITE}" \
789    -I "${ITERATIONS}" \
790    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
791read -r RUNRESULT <"$REPORT_FILE"
792
793#
794# Analyze the results.
795#
796${ZTS_REPORT} ${RERUN:+--no-maybes} "$RESULTS_FILE" >"$REPORT_FILE"
797RESULT=$?
798
799if [ "$RESULT" -eq "2" ] && [ -n "$RERUN" ]; then
800	MAYBES="$($ZTS_REPORT --list-maybes)"
801	TEMP_RESULTS_FILE=$(mktemp_file zts-results-tmp)
802	TEST_LIST=$(mktemp_file test-list)
803	grep "^Test:.*\[FAIL\]" "$RESULTS_FILE" >"$TEMP_RESULTS_FILE"
804	for test_name in $MAYBES; do
805		grep "$test_name " "$TEMP_RESULTS_FILE" >>"$TEST_LIST"
806	done
807	{ PATH=$STF_PATH \
808	    ${TEST_RUNNER} \
809	        ${QUIET:+-q} \
810	        ${DEBUG:+-D} \
811	        ${KMEMLEAK:+-m} \
812	    -c "${RUNFILES}" \
813	    -T "${TAGS}" \
814	    -i "${STF_SUITE}" \
815	    -I "${ITERATIONS}" \
816	    -l "${TEST_LIST}" \
817	    2>&1; echo $? >"$REPORT_FILE"; } | tee "$RESULTS_FILE"
818	read -r RUNRESULT <"$REPORT_FILE"
819	#
820	# Analyze the results.
821	#
822	${ZTS_REPORT} --no-maybes "$RESULTS_FILE" >"$REPORT_FILE"
823	RESULT=$?
824fi
825
826
827cat "$REPORT_FILE"
828
829RESULTS_DIR=$(awk '/^Log directory/ { print $3 }' "$RESULTS_FILE")
830if [ -d "$RESULTS_DIR" ]; then
831	cat "$RESULTS_FILE" "$REPORT_FILE" >"$RESULTS_DIR/results"
832fi
833
834rm -f "$RESULTS_FILE" "$REPORT_FILE" "$TEST_LIST" "$TEMP_RESULTS_FILE"
835
836if [ -n "$SINGLETEST" ]; then
837	rm -f "$RUNFILES" >/dev/null 2>&1
838fi
839
840[ "$RUNRESULT" -gt 3 ] && exit "$RUNRESULT" || exit "$RESULT"
841