xref: /freebsd/sys/contrib/openzfs/scripts/zloop.sh (revision bc5304a006238115291e7568583632889dffbab9) 
1#!/usr/bin/env bash
2
3#
4# CDDL HEADER START
5#
6# This file and its contents are supplied under the terms of the
7# Common Development and Distribution License ("CDDL"), version 1.0.
8# You may only use this file in accordance with the terms of version
9# 1.0 of the CDDL.
10#
11# A full copy of the text of the CDDL should have accompanied this
12# source.  A copy of the CDDL is also available via the Internet at
13# http://www.illumos.org/license/CDDL.
14#
15# CDDL HEADER END
16#
17
18#
19# Copyright (c) 2015 by Delphix. All rights reserved.
20# Copyright (C) 2016 Lawrence Livermore National Security, LLC.
21# Copyright (c) 2017, Intel Corporation.
22#
23
24BASE_DIR=$(dirname "$0")
25SCRIPT_COMMON=common.sh
26if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
27	. "${BASE_DIR}/${SCRIPT_COMMON}"
28else
29	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
30fi
31
32# shellcheck disable=SC2034
33PROG=zloop.sh
34GDB=${GDB:-gdb}
35
36DEFAULTWORKDIR=/var/tmp
37DEFAULTCOREDIR=/var/tmp/zloop
38
39function usage
40{
41	cat >&2 <<EOF
42
43$0 [-hl] [-c <dump directory>] [-f <vdev directory>]
44  [-m <max core dumps>] [-s <vdev size>] [-t <timeout>]
45  [-I <max iterations>] [-- [extra ztest parameters]]
46
47  This script runs ztest repeatedly with randomized arguments.
48  If a crash is encountered, the ztest logs, any associated
49  vdev files, and core file (if one exists) are moved to the
50  output directory ($DEFAULTCOREDIR by default). Any options
51  after the -- end-of-options marker will be passed to ztest.
52
53  Options:
54    -c  Specify a core dump directory to use.
55    -f  Specify working directory for ztest vdev files.
56    -h  Print this help message.
57    -l  Create 'ztest.core.N' symlink to core directory.
58    -m  Max number of core dumps to allow before exiting.
59    -s  Size of vdev devices.
60    -t  Total time to loop for, in seconds. If not provided,
61        zloop runs forever.
62    -I  Max number of iterations to loop before exiting.
63
64EOF
65}
66
67function or_die
68{
69	# shellcheck disable=SC2068
70	if ! $@; then
71		echo "Command failed: $*"
72		exit 1
73	fi
74}
75
76case $(uname) in
77FreeBSD)
78	coreglob="z*.core"
79	;;
80Linux)
81	# core file helpers
82	origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
83	coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
84
85	if [[ $coreglob = "*" ]]; then
86		echo "Setting core file pattern..."
87		echo "core" > /proc/sys/kernel/core_pattern
88		coreglob="$(grep -E -o '^([^|%[:space:]]*)' \
89		    /proc/sys/kernel/core_pattern)*"
90	fi
91	;;
92*)
93	exit 1
94	;;
95esac
96
97function core_file
98{
99	# shellcheck disable=SC2012,SC2086
100	ls -tr1 $coreglob 2>/dev/null | head -1
101}
102
103function core_prog
104{
105	prog=$ZTEST
106	core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
107	    tr  \' ' ')
108	if [[ "$core_id" == *"zdb "* ]]; then
109		prog=$ZDB
110	fi
111	printf "%s" "$prog"
112}
113
114function store_core
115{
116	core="$(core_file)"
117	if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
118		df -h "$workdir" >>ztest.out
119		coreid=$(date "+zloop-%y%m%d-%H%M%S")
120		foundcrashes=$((foundcrashes + 1))
121
122		# zdb debugging
123		zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
124		zdbdebug=$($zdbcmd 2>&1)
125		echo -e "$zdbcmd\n" >>ztest.zdb
126		echo "$zdbdebug" >>ztest.zdb
127
128		dest=$coredir/$coreid
129		or_die mkdir -p "$dest"
130		or_die mkdir -p "$dest/vdev"
131
132		if [[ $symlink -ne 0 ]]; then
133			or_die ln -sf "$dest" ztest.core.$foundcrashes
134		fi
135
136		echo "*** ztest crash found - moving logs to $dest"
137
138		or_die mv ztest.history "$dest/"
139		or_die mv ztest.zdb "$dest/"
140		or_die mv ztest.out "$dest/"
141		or_die mv "$workdir/ztest*" "$dest/vdev/"
142
143		if [[ -e "$workdir/zpool.cache" ]]; then
144			or_die mv "$workdir/zpool.cache" "$dest/vdev/"
145		fi
146
147		# check for core
148		if [[ -f "$core" ]]; then
149			coreprog=$(core_prog "$core")
150			coredebug=$($GDB --batch --quiet \
151			    -ex "set print thread-events off" \
152			    -ex "printf \"*\n* Backtrace \n*\n\"" \
153			    -ex "bt" \
154			    -ex "printf \"*\n* Libraries \n*\n\"" \
155			    -ex "info sharedlib" \
156			    -ex "printf \"*\n* Threads (full) \n*\n\"" \
157			    -ex "info threads" \
158			    -ex "printf \"*\n* Backtraces \n*\n\"" \
159			    -ex "thread apply all bt" \
160			    -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
161			    -ex "thread apply all bt full" \
162			    -ex "quit" "$coreprog" "$core" 2>&1 | \
163			    grep -v "New LWP")
164
165			# Dump core + logs to stored directory
166			echo "$coredebug" >>"$dest/ztest.gdb"
167			or_die mv "$core" "$dest/"
168
169			# Record info in cores logfile
170			echo "*** core @ $coredir/$coreid/$core:" | \
171			    tee -a ztest.cores
172		fi
173
174		if [[ $coremax -gt 0 ]] &&
175		   [[ $foundcrashes -ge $coremax ]]; then
176			echo "exiting... max $coremax allowed cores"
177			exit 1
178		else
179			echo "continuing..."
180		fi
181	fi
182}
183
184# parse arguments
185# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
186coredir=$DEFAULTCOREDIR
187basedir=$DEFAULTWORKDIR
188rundir="zloop-run"
189timeout=0
190size="512m"
191coremax=0
192symlink=0
193iterations=0
194while getopts ":ht:m:I:s:c:f:l" opt; do
195	case $opt in
196		t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
197		m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
198		I ) [[ $OPTARG ]] && iterations=$OPTARG ;;
199		s ) [[ $OPTARG ]] && size=$OPTARG ;;
200		c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
201		f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
202		l ) symlink=1 ;;
203		h ) usage
204		    exit 2
205		    ;;
206		* ) echo "Invalid argument: -$OPTARG";
207		    usage
208		    exit 1
209	esac
210done
211# pass remaining arguments on to ztest
212shift $((OPTIND - 1))
213
214# enable core dumps
215ulimit -c unlimited
216export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0
217
218if [[ -f "$(core_file)" ]]; then
219	echo -n "There's a core dump here you might want to look at first... "
220	core_file
221	echo
222	exit 1
223fi
224
225if [[ ! -d $coredir ]]; then
226	echo "core dump directory ($coredir) does not exist, creating it."
227	or_die mkdir -p "$coredir"
228fi
229
230if [[ ! -w $coredir ]]; then
231	echo "core dump directory ($coredir) is not writable."
232	exit 1
233fi
234
235or_die rm -f ztest.history
236or_die rm -f ztest.zdb
237or_die rm -f ztest.cores
238
239ztrc=0		# ztest return value
240foundcrashes=0	# number of crashes found so far
241starttime=$(date +%s)
242curtime=$starttime
243iteration=0
244
245# if no timeout was specified, loop forever.
246while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do
247	if (( iterations > 0 )) && (( iteration++ == iterations )); then
248		break
249	fi
250
251	zopt="-G -VVVVV"
252
253	# start each run with an empty directory
254	workdir="$basedir/$rundir"
255	or_die rm -rf "$workdir"
256	or_die mkdir "$workdir"
257
258	# switch between three types of configs
259	# 1/3 basic, 1/3 raidz mix, and 1/3 draid mix
260	choice=$((RANDOM % 3))
261
262	# ashift range 9 - 15
263	align=$(((RANDOM % 2) * 3 + 9))
264
265	# randomly use special classes
266	class="special=random"
267
268	if [[ $choice -eq 0 ]]; then
269		# basic mirror only
270		parity=1
271		mirrors=2
272		draid_data=0
273		draid_spares=0
274		raid_children=0
275		vdevs=2
276		raid_type="raidz"
277	elif [[ $choice -eq 1 ]]; then
278		# fully randomized mirror/raidz (sans dRAID)
279		parity=$(((RANDOM % 3) + 1))
280		mirrors=$(((RANDOM % 3) * 1))
281		draid_data=0
282		draid_spares=0
283		raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
284		vdevs=$(((RANDOM % 3) + 3))
285		raid_type="raidz"
286	else
287		# fully randomized dRAID (sans mirror/raidz)
288		parity=$(((RANDOM % 3) + 1))
289		mirrors=0
290		draid_data=$(((RANDOM % 8) + 3))
291		draid_spares=$(((RANDOM % 2) + parity))
292		stripe=$((draid_data + parity))
293		extra=$((draid_spares + (RANDOM % 4)))
294		raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra))
295		vdevs=$((RANDOM % 3))
296		raid_type="draid"
297	fi
298
299	zopt="$zopt -K $raid_type"
300	zopt="$zopt -m $mirrors"
301	zopt="$zopt -r $raid_children"
302	zopt="$zopt -D $draid_data"
303	zopt="$zopt -S $draid_spares"
304	zopt="$zopt -R $parity"
305	zopt="$zopt -v $vdevs"
306	zopt="$zopt -a $align"
307	zopt="$zopt -C $class"
308	zopt="$zopt -s $size"
309	zopt="$zopt -f $workdir"
310
311	cmd="$ZTEST $zopt $*"
312	desc="$(date '+%m/%d %T') $cmd"
313	echo "$desc" | tee -a ztest.history
314	echo "$desc" >>ztest.out
315	$cmd >>ztest.out 2>&1
316	ztrc=$?
317	grep -E '===|WARNING' ztest.out >>ztest.history
318
319	store_core
320
321	curtime=$(date +%s)
322done
323
324echo "zloop finished, $foundcrashes crashes found"
325
326# restore core pattern.
327case $(uname) in
328Linux)
329	echo "$origcorepattern" > /proc/sys/kernel/core_pattern
330	;;
331*)
332	;;
333esac
334
335uptime >>ztest.out
336
337if [[ $foundcrashes -gt 0 ]]; then
338	exit 1
339fi
340