xref: /freebsd/sys/contrib/openzfs/scripts/zloop.sh (revision e92ffd9b626833ebdbf2742c8ffddc6cd94b963e)
1eda14cbcSMatt Macy#!/usr/bin/env bash
2eda14cbcSMatt Macy
3eda14cbcSMatt Macy#
4eda14cbcSMatt Macy# CDDL HEADER START
5eda14cbcSMatt Macy#
6eda14cbcSMatt Macy# This file and its contents are supplied under the terms of the
7eda14cbcSMatt Macy# Common Development and Distribution License ("CDDL"), version 1.0.
8eda14cbcSMatt Macy# You may only use this file in accordance with the terms of version
9eda14cbcSMatt Macy# 1.0 of the CDDL.
10eda14cbcSMatt Macy#
11eda14cbcSMatt Macy# A full copy of the text of the CDDL should have accompanied this
12eda14cbcSMatt Macy# source.  A copy of the CDDL is also available via the Internet at
13eda14cbcSMatt Macy# http://www.illumos.org/license/CDDL.
14eda14cbcSMatt Macy#
15eda14cbcSMatt Macy# CDDL HEADER END
16eda14cbcSMatt Macy#
17eda14cbcSMatt Macy
18eda14cbcSMatt Macy#
19eda14cbcSMatt Macy# Copyright (c) 2015 by Delphix. All rights reserved.
20eda14cbcSMatt Macy# Copyright (C) 2016 Lawrence Livermore National Security, LLC.
217877fdebSMatt Macy# Copyright (c) 2017, Intel Corporation.
22eda14cbcSMatt Macy#
23eda14cbcSMatt Macy
24eda14cbcSMatt MacyBASE_DIR=$(dirname "$0")
25eda14cbcSMatt MacySCRIPT_COMMON=common.sh
26eda14cbcSMatt Macyif [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
27eda14cbcSMatt Macy	. "${BASE_DIR}/${SCRIPT_COMMON}"
28eda14cbcSMatt Macyelse
29eda14cbcSMatt Macy	echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
30eda14cbcSMatt Macyfi
31eda14cbcSMatt Macy
32eda14cbcSMatt Macy# shellcheck disable=SC2034
33eda14cbcSMatt MacyPROG=zloop.sh
34eda14cbcSMatt MacyGDB=${GDB:-gdb}
35eda14cbcSMatt Macy
36eda14cbcSMatt MacyDEFAULTWORKDIR=/var/tmp
37eda14cbcSMatt MacyDEFAULTCOREDIR=/var/tmp/zloop
38eda14cbcSMatt Macy
39eda14cbcSMatt Macyfunction usage
40eda14cbcSMatt Macy{
413f9d360cSMartin Matuska	cat >&2 <<EOF
423f9d360cSMartin Matuska
433f9d360cSMartin Matuska$0 [-hl] [-c <dump directory>] [-f <vdev directory>]
443f9d360cSMartin Matuska  [-m <max core dumps>] [-s <vdev size>] [-t <timeout>]
453f9d360cSMartin Matuska  [-I <max iterations>] [-- [extra ztest parameters]]
463f9d360cSMartin Matuska
473f9d360cSMartin Matuska  This script runs ztest repeatedly with randomized arguments.
483f9d360cSMartin Matuska  If a crash is encountered, the ztest logs, any associated
493f9d360cSMartin Matuska  vdev files, and core file (if one exists) are moved to the
503f9d360cSMartin Matuska  output directory ($DEFAULTCOREDIR by default). Any options
513f9d360cSMartin Matuska  after the -- end-of-options marker will be passed to ztest.
523f9d360cSMartin Matuska
533f9d360cSMartin Matuska  Options:
543f9d360cSMartin Matuska    -c  Specify a core dump directory to use.
553f9d360cSMartin Matuska    -f  Specify working directory for ztest vdev files.
563f9d360cSMartin Matuska    -h  Print this help message.
573f9d360cSMartin Matuska    -l  Create 'ztest.core.N' symlink to core directory.
583f9d360cSMartin Matuska    -m  Max number of core dumps to allow before exiting.
593f9d360cSMartin Matuska    -s  Size of vdev devices.
603f9d360cSMartin Matuska    -t  Total time to loop for, in seconds. If not provided,
613f9d360cSMartin Matuska        zloop runs forever.
623f9d360cSMartin Matuska    -I  Max number of iterations to loop before exiting.
633f9d360cSMartin Matuska
643f9d360cSMartin MatuskaEOF
65eda14cbcSMatt Macy}
66eda14cbcSMatt Macy
67eda14cbcSMatt Macyfunction or_die
68eda14cbcSMatt Macy{
69eda14cbcSMatt Macy	# shellcheck disable=SC2068
7016038816SMartin Matuska	if ! $@; then
7116038816SMartin Matuska		echo "Command failed: $*"
72eda14cbcSMatt Macy		exit 1
73eda14cbcSMatt Macy	fi
74eda14cbcSMatt Macy}
75eda14cbcSMatt Macy
76eda14cbcSMatt Macycase $(uname) in
77eda14cbcSMatt MacyFreeBSD)
78eda14cbcSMatt Macy	coreglob="z*.core"
79eda14cbcSMatt Macy	;;
80eda14cbcSMatt MacyLinux)
81eda14cbcSMatt Macy	# core file helpers
82eda14cbcSMatt Macy	origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
83eda14cbcSMatt Macy	coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
84eda14cbcSMatt Macy
85eda14cbcSMatt Macy	if [[ $coreglob = "*" ]]; then
86eda14cbcSMatt Macy		echo "Setting core file pattern..."
87eda14cbcSMatt Macy		echo "core" > /proc/sys/kernel/core_pattern
88eda14cbcSMatt Macy		coreglob="$(grep -E -o '^([^|%[:space:]]*)' \
89eda14cbcSMatt Macy		    /proc/sys/kernel/core_pattern)*"
90eda14cbcSMatt Macy	fi
91eda14cbcSMatt Macy	;;
92eda14cbcSMatt Macy*)
93eda14cbcSMatt Macy	exit 1
94eda14cbcSMatt Macy	;;
95eda14cbcSMatt Macyesac
96eda14cbcSMatt Macy
97eda14cbcSMatt Macyfunction core_file
98eda14cbcSMatt Macy{
9916038816SMartin Matuska	# shellcheck disable=SC2012,SC2086
10016038816SMartin Matuska	ls -tr1 $coreglob 2>/dev/null | head -1
101eda14cbcSMatt Macy}
102eda14cbcSMatt Macy
103eda14cbcSMatt Macyfunction core_prog
104eda14cbcSMatt Macy{
105*e92ffd9bSMartin Matuska	# shellcheck disable=SC2154
106eda14cbcSMatt Macy	prog=$ZTEST
107eda14cbcSMatt Macy	core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
108eda14cbcSMatt Macy	    tr  \' ' ')
10916038816SMartin Matuska	if [[ "$core_id" == *"zdb "* ]]; then
110*e92ffd9bSMartin Matuska		# shellcheck disable=SC2154
111eda14cbcSMatt Macy		prog=$ZDB
112eda14cbcSMatt Macy	fi
113eda14cbcSMatt Macy	printf "%s" "$prog"
114eda14cbcSMatt Macy}
115eda14cbcSMatt Macy
116eda14cbcSMatt Macyfunction store_core
117eda14cbcSMatt Macy{
118eda14cbcSMatt Macy	core="$(core_file)"
119eda14cbcSMatt Macy	if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
120eda14cbcSMatt Macy		df -h "$workdir" >>ztest.out
121eda14cbcSMatt Macy		coreid=$(date "+zloop-%y%m%d-%H%M%S")
122eda14cbcSMatt Macy		foundcrashes=$((foundcrashes + 1))
123eda14cbcSMatt Macy
124eda14cbcSMatt Macy		# zdb debugging
125eda14cbcSMatt Macy		zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
126eda14cbcSMatt Macy		zdbdebug=$($zdbcmd 2>&1)
127eda14cbcSMatt Macy		echo -e "$zdbcmd\n" >>ztest.zdb
128eda14cbcSMatt Macy		echo "$zdbdebug" >>ztest.zdb
129eda14cbcSMatt Macy
130eda14cbcSMatt Macy		dest=$coredir/$coreid
131eda14cbcSMatt Macy		or_die mkdir -p "$dest"
132eda14cbcSMatt Macy		or_die mkdir -p "$dest/vdev"
133eda14cbcSMatt Macy
134eda14cbcSMatt Macy		if [[ $symlink -ne 0 ]]; then
135*e92ffd9bSMartin Matuska			or_die ln -sf "$dest" "ztest.core.${foundcrashes}"
136eda14cbcSMatt Macy		fi
137eda14cbcSMatt Macy
138eda14cbcSMatt Macy		echo "*** ztest crash found - moving logs to $dest"
139eda14cbcSMatt Macy
140eda14cbcSMatt Macy		or_die mv ztest.history "$dest/"
141eda14cbcSMatt Macy		or_die mv ztest.zdb "$dest/"
142eda14cbcSMatt Macy		or_die mv ztest.out "$dest/"
143eda14cbcSMatt Macy		or_die mv "$workdir/ztest*" "$dest/vdev/"
144eda14cbcSMatt Macy
145eda14cbcSMatt Macy		if [[ -e "$workdir/zpool.cache" ]]; then
146eda14cbcSMatt Macy			or_die mv "$workdir/zpool.cache" "$dest/vdev/"
147eda14cbcSMatt Macy		fi
148eda14cbcSMatt Macy
149eda14cbcSMatt Macy		# check for core
150eda14cbcSMatt Macy		if [[ -f "$core" ]]; then
151eda14cbcSMatt Macy			coreprog=$(core_prog "$core")
152eda14cbcSMatt Macy			coredebug=$($GDB --batch --quiet \
153eda14cbcSMatt Macy			    -ex "set print thread-events off" \
154eda14cbcSMatt Macy			    -ex "printf \"*\n* Backtrace \n*\n\"" \
155eda14cbcSMatt Macy			    -ex "bt" \
156eda14cbcSMatt Macy			    -ex "printf \"*\n* Libraries \n*\n\"" \
157eda14cbcSMatt Macy			    -ex "info sharedlib" \
158eda14cbcSMatt Macy			    -ex "printf \"*\n* Threads (full) \n*\n\"" \
159eda14cbcSMatt Macy			    -ex "info threads" \
160eda14cbcSMatt Macy			    -ex "printf \"*\n* Backtraces \n*\n\"" \
161eda14cbcSMatt Macy			    -ex "thread apply all bt" \
162eda14cbcSMatt Macy			    -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
163eda14cbcSMatt Macy			    -ex "thread apply all bt full" \
164eda14cbcSMatt Macy			    -ex "quit" "$coreprog" "$core" 2>&1 | \
165eda14cbcSMatt Macy			    grep -v "New LWP")
166eda14cbcSMatt Macy
167eda14cbcSMatt Macy			# Dump core + logs to stored directory
168eda14cbcSMatt Macy			echo "$coredebug" >>"$dest/ztest.gdb"
169eda14cbcSMatt Macy			or_die mv "$core" "$dest/"
170eda14cbcSMatt Macy
171eda14cbcSMatt Macy			# Record info in cores logfile
172eda14cbcSMatt Macy			echo "*** core @ $coredir/$coreid/$core:" | \
173eda14cbcSMatt Macy			    tee -a ztest.cores
174eda14cbcSMatt Macy		fi
175eda14cbcSMatt Macy
176eda14cbcSMatt Macy		if [[ $coremax -gt 0 ]] &&
177eda14cbcSMatt Macy		   [[ $foundcrashes -ge $coremax ]]; then
178eda14cbcSMatt Macy			echo "exiting... max $coremax allowed cores"
179eda14cbcSMatt Macy			exit 1
180eda14cbcSMatt Macy		else
181eda14cbcSMatt Macy			echo "continuing..."
182eda14cbcSMatt Macy		fi
183eda14cbcSMatt Macy	fi
184eda14cbcSMatt Macy}
185eda14cbcSMatt Macy
186eda14cbcSMatt Macy# parse arguments
187eda14cbcSMatt Macy# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
188eda14cbcSMatt Macycoredir=$DEFAULTCOREDIR
189eda14cbcSMatt Macybasedir=$DEFAULTWORKDIR
190eda14cbcSMatt Macyrundir="zloop-run"
191eda14cbcSMatt Macytimeout=0
192eda14cbcSMatt Macysize="512m"
193eda14cbcSMatt Macycoremax=0
194eda14cbcSMatt Macysymlink=0
1953f9d360cSMartin Matuskaiterations=0
1963f9d360cSMartin Matuskawhile getopts ":ht:m:I:s:c:f:l" opt; do
197eda14cbcSMatt Macy	case $opt in
198eda14cbcSMatt Macy		t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
199eda14cbcSMatt Macy		m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
200*e92ffd9bSMartin Matuska		I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;;
201*e92ffd9bSMartin Matuska		s ) [[ -n $OPTARG ]] && size=$OPTARG ;;
202*e92ffd9bSMartin Matuska		c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;;
203*e92ffd9bSMartin Matuska		f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
204eda14cbcSMatt Macy		l ) symlink=1 ;;
205eda14cbcSMatt Macy		h ) usage
206eda14cbcSMatt Macy		    exit 2
207eda14cbcSMatt Macy		    ;;
208eda14cbcSMatt Macy		* ) echo "Invalid argument: -$OPTARG";
209eda14cbcSMatt Macy		    usage
210eda14cbcSMatt Macy		    exit 1
211eda14cbcSMatt Macy	esac
212eda14cbcSMatt Macydone
213eda14cbcSMatt Macy# pass remaining arguments on to ztest
214eda14cbcSMatt Macyshift $((OPTIND - 1))
215eda14cbcSMatt Macy
216eda14cbcSMatt Macy# enable core dumps
217eda14cbcSMatt Macyulimit -c unlimited
218eda14cbcSMatt Macyexport ASAN_OPTIONS=abort_on_error=1:disable_coredump=0
219eda14cbcSMatt Macy
220eda14cbcSMatt Macyif [[ -f "$(core_file)" ]]; then
221eda14cbcSMatt Macy	echo -n "There's a core dump here you might want to look at first... "
222eda14cbcSMatt Macy	core_file
223eda14cbcSMatt Macy	echo
224eda14cbcSMatt Macy	exit 1
225eda14cbcSMatt Macyfi
226eda14cbcSMatt Macy
227eda14cbcSMatt Macyif [[ ! -d $coredir ]]; then
228eda14cbcSMatt Macy	echo "core dump directory ($coredir) does not exist, creating it."
229eda14cbcSMatt Macy	or_die mkdir -p "$coredir"
230eda14cbcSMatt Macyfi
231eda14cbcSMatt Macy
232eda14cbcSMatt Macyif [[ ! -w $coredir ]]; then
233eda14cbcSMatt Macy	echo "core dump directory ($coredir) is not writable."
234eda14cbcSMatt Macy	exit 1
235eda14cbcSMatt Macyfi
236eda14cbcSMatt Macy
237eda14cbcSMatt Macyor_die rm -f ztest.history
238eda14cbcSMatt Macyor_die rm -f ztest.zdb
239eda14cbcSMatt Macyor_die rm -f ztest.cores
240eda14cbcSMatt Macy
241eda14cbcSMatt Macyztrc=0		# ztest return value
242eda14cbcSMatt Macyfoundcrashes=0	# number of crashes found so far
243eda14cbcSMatt Macystarttime=$(date +%s)
244eda14cbcSMatt Macycurtime=$starttime
2453f9d360cSMartin Matuskaiteration=0
246eda14cbcSMatt Macy
247eda14cbcSMatt Macy# if no timeout was specified, loop forever.
2483f9d360cSMartin Matuskawhile (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do
2493f9d360cSMartin Matuska	if (( iterations > 0 )) && (( iteration++ == iterations )); then
2503f9d360cSMartin Matuska		break
2513f9d360cSMartin Matuska	fi
2523f9d360cSMartin Matuska
253eda14cbcSMatt Macy	zopt="-G -VVVVV"
254eda14cbcSMatt Macy
255eda14cbcSMatt Macy	# start each run with an empty directory
256eda14cbcSMatt Macy	workdir="$basedir/$rundir"
257eda14cbcSMatt Macy	or_die rm -rf "$workdir"
258eda14cbcSMatt Macy	or_die mkdir "$workdir"
259eda14cbcSMatt Macy
2607877fdebSMatt Macy	# switch between three types of configs
2617877fdebSMatt Macy	# 1/3 basic, 1/3 raidz mix, and 1/3 draid mix
2627877fdebSMatt Macy	choice=$((RANDOM % 3))
2637877fdebSMatt Macy
2647877fdebSMatt Macy	# ashift range 9 - 15
265eda14cbcSMatt Macy	align=$(((RANDOM % 2) * 3 + 9))
2667877fdebSMatt Macy
2677877fdebSMatt Macy	# randomly use special classes
2687877fdebSMatt Macy	class="special=random"
2697877fdebSMatt Macy
2707877fdebSMatt Macy	if [[ $choice -eq 0 ]]; then
2717877fdebSMatt Macy		# basic mirror only
2727877fdebSMatt Macy		parity=1
2737877fdebSMatt Macy		mirrors=2
2747877fdebSMatt Macy		draid_data=0
2757877fdebSMatt Macy		draid_spares=0
2767877fdebSMatt Macy		raid_children=0
2777877fdebSMatt Macy		vdevs=2
2787877fdebSMatt Macy		raid_type="raidz"
2797877fdebSMatt Macy	elif [[ $choice -eq 1 ]]; then
2807877fdebSMatt Macy		# fully randomized mirror/raidz (sans dRAID)
2817877fdebSMatt Macy		parity=$(((RANDOM % 3) + 1))
2827877fdebSMatt Macy		mirrors=$(((RANDOM % 3) * 1))
2837877fdebSMatt Macy		draid_data=0
2847877fdebSMatt Macy		draid_spares=0
2857877fdebSMatt Macy		raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
2867877fdebSMatt Macy		vdevs=$(((RANDOM % 3) + 3))
2877877fdebSMatt Macy		raid_type="raidz"
2887877fdebSMatt Macy	else
2897877fdebSMatt Macy		# fully randomized dRAID (sans mirror/raidz)
2907877fdebSMatt Macy		parity=$(((RANDOM % 3) + 1))
2917877fdebSMatt Macy		mirrors=0
2927877fdebSMatt Macy		draid_data=$(((RANDOM % 8) + 3))
2937877fdebSMatt Macy		draid_spares=$(((RANDOM % 2) + parity))
2947877fdebSMatt Macy		stripe=$((draid_data + parity))
2957877fdebSMatt Macy		extra=$((draid_spares + (RANDOM % 4)))
2967877fdebSMatt Macy		raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra))
2977877fdebSMatt Macy		vdevs=$((RANDOM % 3))
2987877fdebSMatt Macy		raid_type="draid"
2997877fdebSMatt Macy	fi
3007877fdebSMatt Macy
3017877fdebSMatt Macy	zopt="$zopt -K $raid_type"
302eda14cbcSMatt Macy	zopt="$zopt -m $mirrors"
3037877fdebSMatt Macy	zopt="$zopt -r $raid_children"
3047877fdebSMatt Macy	zopt="$zopt -D $draid_data"
3057877fdebSMatt Macy	zopt="$zopt -S $draid_spares"
306eda14cbcSMatt Macy	zopt="$zopt -R $parity"
307eda14cbcSMatt Macy	zopt="$zopt -v $vdevs"
308eda14cbcSMatt Macy	zopt="$zopt -a $align"
3097877fdebSMatt Macy	zopt="$zopt -C $class"
310eda14cbcSMatt Macy	zopt="$zopt -s $size"
311eda14cbcSMatt Macy	zopt="$zopt -f $workdir"
312eda14cbcSMatt Macy
31316038816SMartin Matuska	cmd="$ZTEST $zopt $*"
314eda14cbcSMatt Macy	desc="$(date '+%m/%d %T') $cmd"
315eda14cbcSMatt Macy	echo "$desc" | tee -a ztest.history
316eda14cbcSMatt Macy	echo "$desc" >>ztest.out
317eda14cbcSMatt Macy	$cmd >>ztest.out 2>&1
318eda14cbcSMatt Macy	ztrc=$?
319eda14cbcSMatt Macy	grep -E '===|WARNING' ztest.out >>ztest.history
320eda14cbcSMatt Macy
321eda14cbcSMatt Macy	store_core
322eda14cbcSMatt Macy
323eda14cbcSMatt Macy	curtime=$(date +%s)
324eda14cbcSMatt Macydone
325eda14cbcSMatt Macy
326eda14cbcSMatt Macyecho "zloop finished, $foundcrashes crashes found"
327eda14cbcSMatt Macy
328eda14cbcSMatt Macy# restore core pattern.
329eda14cbcSMatt Macycase $(uname) in
330eda14cbcSMatt MacyLinux)
331eda14cbcSMatt Macy	echo "$origcorepattern" > /proc/sys/kernel/core_pattern
332eda14cbcSMatt Macy	;;
333eda14cbcSMatt Macy*)
334eda14cbcSMatt Macy	;;
335eda14cbcSMatt Macyesac
336eda14cbcSMatt Macy
337eda14cbcSMatt Macyuptime >>ztest.out
338eda14cbcSMatt Macy
339eda14cbcSMatt Macyif [[ $foundcrashes -gt 0 ]]; then
340eda14cbcSMatt Macy	exit 1
341eda14cbcSMatt Macyfi
342