1eda14cbcSMatt Macy#!/usr/bin/env bash 2eda14cbcSMatt Macy 3eda14cbcSMatt Macy# 4eda14cbcSMatt Macy# CDDL HEADER START 5eda14cbcSMatt Macy# 6eda14cbcSMatt Macy# This file and its contents are supplied under the terms of the 7eda14cbcSMatt Macy# Common Development and Distribution License ("CDDL"), version 1.0. 8eda14cbcSMatt Macy# You may only use this file in accordance with the terms of version 9eda14cbcSMatt Macy# 1.0 of the CDDL. 10eda14cbcSMatt Macy# 11eda14cbcSMatt Macy# A full copy of the text of the CDDL should have accompanied this 12eda14cbcSMatt Macy# source. A copy of the CDDL is also available via the Internet at 13eda14cbcSMatt Macy# http://www.illumos.org/license/CDDL. 14eda14cbcSMatt Macy# 15eda14cbcSMatt Macy# CDDL HEADER END 16eda14cbcSMatt Macy# 17eda14cbcSMatt Macy 18eda14cbcSMatt Macy# 19eda14cbcSMatt Macy# Copyright (c) 2015 by Delphix. All rights reserved. 20eda14cbcSMatt Macy# Copyright (C) 2016 Lawrence Livermore National Security, LLC. 217877fdebSMatt Macy# Copyright (c) 2017, Intel Corporation. 22eda14cbcSMatt Macy# 23eda14cbcSMatt Macy 24eda14cbcSMatt MacyBASE_DIR=$(dirname "$0") 25eda14cbcSMatt MacySCRIPT_COMMON=common.sh 26eda14cbcSMatt Macyif [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then 27eda14cbcSMatt Macy . "${BASE_DIR}/${SCRIPT_COMMON}" 28eda14cbcSMatt Macyelse 29eda14cbcSMatt Macy echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 30eda14cbcSMatt Macyfi 31eda14cbcSMatt Macy 32eda14cbcSMatt Macy# shellcheck disable=SC2034 33eda14cbcSMatt MacyPROG=zloop.sh 34eda14cbcSMatt MacyGDB=${GDB:-gdb} 35eda14cbcSMatt Macy 36eda14cbcSMatt MacyDEFAULTWORKDIR=/var/tmp 37eda14cbcSMatt MacyDEFAULTCOREDIR=/var/tmp/zloop 38eda14cbcSMatt Macy 39eda14cbcSMatt Macyfunction usage 40eda14cbcSMatt Macy{ 413f9d360cSMartin Matuska cat >&2 <<EOF 423f9d360cSMartin Matuska 433f9d360cSMartin Matuska$0 [-hl] [-c <dump directory>] [-f <vdev directory>] 443f9d360cSMartin Matuska [-m <max core dumps>] [-s <vdev size>] [-t <timeout>] 453f9d360cSMartin Matuska [-I <max iterations>] [-- [extra ztest parameters]] 463f9d360cSMartin Matuska 473f9d360cSMartin Matuska This script runs ztest repeatedly with randomized arguments. 483f9d360cSMartin Matuska If a crash is encountered, the ztest logs, any associated 493f9d360cSMartin Matuska vdev files, and core file (if one exists) are moved to the 503f9d360cSMartin Matuska output directory ($DEFAULTCOREDIR by default). Any options 513f9d360cSMartin Matuska after the -- end-of-options marker will be passed to ztest. 523f9d360cSMartin Matuska 533f9d360cSMartin Matuska Options: 543f9d360cSMartin Matuska -c Specify a core dump directory to use. 553f9d360cSMartin Matuska -f Specify working directory for ztest vdev files. 563f9d360cSMartin Matuska -h Print this help message. 573f9d360cSMartin Matuska -l Create 'ztest.core.N' symlink to core directory. 583f9d360cSMartin Matuska -m Max number of core dumps to allow before exiting. 593f9d360cSMartin Matuska -s Size of vdev devices. 603f9d360cSMartin Matuska -t Total time to loop for, in seconds. If not provided, 613f9d360cSMartin Matuska zloop runs forever. 623f9d360cSMartin Matuska -I Max number of iterations to loop before exiting. 633f9d360cSMartin Matuska 643f9d360cSMartin MatuskaEOF 65eda14cbcSMatt Macy} 66eda14cbcSMatt Macy 67eda14cbcSMatt Macyfunction or_die 68eda14cbcSMatt Macy{ 69eda14cbcSMatt Macy # shellcheck disable=SC2068 7016038816SMartin Matuska if ! $@; then 7116038816SMartin Matuska echo "Command failed: $*" 72eda14cbcSMatt Macy exit 1 73eda14cbcSMatt Macy fi 74eda14cbcSMatt Macy} 75eda14cbcSMatt Macy 76eda14cbcSMatt Macycase $(uname) in 77eda14cbcSMatt MacyFreeBSD) 78eda14cbcSMatt Macy coreglob="z*.core" 79eda14cbcSMatt Macy ;; 80eda14cbcSMatt MacyLinux) 81eda14cbcSMatt Macy # core file helpers 82eda14cbcSMatt Macy origcorepattern="$(cat /proc/sys/kernel/core_pattern)" 83eda14cbcSMatt Macy coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" 84eda14cbcSMatt Macy 85eda14cbcSMatt Macy if [[ $coreglob = "*" ]]; then 86eda14cbcSMatt Macy echo "Setting core file pattern..." 87eda14cbcSMatt Macy echo "core" > /proc/sys/kernel/core_pattern 88eda14cbcSMatt Macy coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ 89eda14cbcSMatt Macy /proc/sys/kernel/core_pattern)*" 90eda14cbcSMatt Macy fi 91eda14cbcSMatt Macy ;; 92eda14cbcSMatt Macy*) 93eda14cbcSMatt Macy exit 1 94eda14cbcSMatt Macy ;; 95eda14cbcSMatt Macyesac 96eda14cbcSMatt Macy 97eda14cbcSMatt Macyfunction core_file 98eda14cbcSMatt Macy{ 9916038816SMartin Matuska # shellcheck disable=SC2012,SC2086 10016038816SMartin Matuska ls -tr1 $coreglob 2>/dev/null | head -1 101eda14cbcSMatt Macy} 102eda14cbcSMatt Macy 103eda14cbcSMatt Macyfunction core_prog 104eda14cbcSMatt Macy{ 105*e92ffd9bSMartin Matuska # shellcheck disable=SC2154 106eda14cbcSMatt Macy prog=$ZTEST 107eda14cbcSMatt Macy core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ 108eda14cbcSMatt Macy tr \' ' ') 10916038816SMartin Matuska if [[ "$core_id" == *"zdb "* ]]; then 110*e92ffd9bSMartin Matuska # shellcheck disable=SC2154 111eda14cbcSMatt Macy prog=$ZDB 112eda14cbcSMatt Macy fi 113eda14cbcSMatt Macy printf "%s" "$prog" 114eda14cbcSMatt Macy} 115eda14cbcSMatt Macy 116eda14cbcSMatt Macyfunction store_core 117eda14cbcSMatt Macy{ 118eda14cbcSMatt Macy core="$(core_file)" 119eda14cbcSMatt Macy if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then 120eda14cbcSMatt Macy df -h "$workdir" >>ztest.out 121eda14cbcSMatt Macy coreid=$(date "+zloop-%y%m%d-%H%M%S") 122eda14cbcSMatt Macy foundcrashes=$((foundcrashes + 1)) 123eda14cbcSMatt Macy 124eda14cbcSMatt Macy # zdb debugging 125eda14cbcSMatt Macy zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" 126eda14cbcSMatt Macy zdbdebug=$($zdbcmd 2>&1) 127eda14cbcSMatt Macy echo -e "$zdbcmd\n" >>ztest.zdb 128eda14cbcSMatt Macy echo "$zdbdebug" >>ztest.zdb 129eda14cbcSMatt Macy 130eda14cbcSMatt Macy dest=$coredir/$coreid 131eda14cbcSMatt Macy or_die mkdir -p "$dest" 132eda14cbcSMatt Macy or_die mkdir -p "$dest/vdev" 133eda14cbcSMatt Macy 134eda14cbcSMatt Macy if [[ $symlink -ne 0 ]]; then 135*e92ffd9bSMartin Matuska or_die ln -sf "$dest" "ztest.core.${foundcrashes}" 136eda14cbcSMatt Macy fi 137eda14cbcSMatt Macy 138eda14cbcSMatt Macy echo "*** ztest crash found - moving logs to $dest" 139eda14cbcSMatt Macy 140eda14cbcSMatt Macy or_die mv ztest.history "$dest/" 141eda14cbcSMatt Macy or_die mv ztest.zdb "$dest/" 142eda14cbcSMatt Macy or_die mv ztest.out "$dest/" 143eda14cbcSMatt Macy or_die mv "$workdir/ztest*" "$dest/vdev/" 144eda14cbcSMatt Macy 145eda14cbcSMatt Macy if [[ -e "$workdir/zpool.cache" ]]; then 146eda14cbcSMatt Macy or_die mv "$workdir/zpool.cache" "$dest/vdev/" 147eda14cbcSMatt Macy fi 148eda14cbcSMatt Macy 149eda14cbcSMatt Macy # check for core 150eda14cbcSMatt Macy if [[ -f "$core" ]]; then 151eda14cbcSMatt Macy coreprog=$(core_prog "$core") 152eda14cbcSMatt Macy coredebug=$($GDB --batch --quiet \ 153eda14cbcSMatt Macy -ex "set print thread-events off" \ 154eda14cbcSMatt Macy -ex "printf \"*\n* Backtrace \n*\n\"" \ 155eda14cbcSMatt Macy -ex "bt" \ 156eda14cbcSMatt Macy -ex "printf \"*\n* Libraries \n*\n\"" \ 157eda14cbcSMatt Macy -ex "info sharedlib" \ 158eda14cbcSMatt Macy -ex "printf \"*\n* Threads (full) \n*\n\"" \ 159eda14cbcSMatt Macy -ex "info threads" \ 160eda14cbcSMatt Macy -ex "printf \"*\n* Backtraces \n*\n\"" \ 161eda14cbcSMatt Macy -ex "thread apply all bt" \ 162eda14cbcSMatt Macy -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ 163eda14cbcSMatt Macy -ex "thread apply all bt full" \ 164eda14cbcSMatt Macy -ex "quit" "$coreprog" "$core" 2>&1 | \ 165eda14cbcSMatt Macy grep -v "New LWP") 166eda14cbcSMatt Macy 167eda14cbcSMatt Macy # Dump core + logs to stored directory 168eda14cbcSMatt Macy echo "$coredebug" >>"$dest/ztest.gdb" 169eda14cbcSMatt Macy or_die mv "$core" "$dest/" 170eda14cbcSMatt Macy 171eda14cbcSMatt Macy # Record info in cores logfile 172eda14cbcSMatt Macy echo "*** core @ $coredir/$coreid/$core:" | \ 173eda14cbcSMatt Macy tee -a ztest.cores 174eda14cbcSMatt Macy fi 175eda14cbcSMatt Macy 176eda14cbcSMatt Macy if [[ $coremax -gt 0 ]] && 177eda14cbcSMatt Macy [[ $foundcrashes -ge $coremax ]]; then 178eda14cbcSMatt Macy echo "exiting... max $coremax allowed cores" 179eda14cbcSMatt Macy exit 1 180eda14cbcSMatt Macy else 181eda14cbcSMatt Macy echo "continuing..." 182eda14cbcSMatt Macy fi 183eda14cbcSMatt Macy fi 184eda14cbcSMatt Macy} 185eda14cbcSMatt Macy 186eda14cbcSMatt Macy# parse arguments 187eda14cbcSMatt Macy# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] 188eda14cbcSMatt Macycoredir=$DEFAULTCOREDIR 189eda14cbcSMatt Macybasedir=$DEFAULTWORKDIR 190eda14cbcSMatt Macyrundir="zloop-run" 191eda14cbcSMatt Macytimeout=0 192eda14cbcSMatt Macysize="512m" 193eda14cbcSMatt Macycoremax=0 194eda14cbcSMatt Macysymlink=0 1953f9d360cSMartin Matuskaiterations=0 1963f9d360cSMartin Matuskawhile getopts ":ht:m:I:s:c:f:l" opt; do 197eda14cbcSMatt Macy case $opt in 198eda14cbcSMatt Macy t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; 199eda14cbcSMatt Macy m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; 200*e92ffd9bSMartin Matuska I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;; 201*e92ffd9bSMartin Matuska s ) [[ -n $OPTARG ]] && size=$OPTARG ;; 202*e92ffd9bSMartin Matuska c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;; 203*e92ffd9bSMartin Matuska f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; 204eda14cbcSMatt Macy l ) symlink=1 ;; 205eda14cbcSMatt Macy h ) usage 206eda14cbcSMatt Macy exit 2 207eda14cbcSMatt Macy ;; 208eda14cbcSMatt Macy * ) echo "Invalid argument: -$OPTARG"; 209eda14cbcSMatt Macy usage 210eda14cbcSMatt Macy exit 1 211eda14cbcSMatt Macy esac 212eda14cbcSMatt Macydone 213eda14cbcSMatt Macy# pass remaining arguments on to ztest 214eda14cbcSMatt Macyshift $((OPTIND - 1)) 215eda14cbcSMatt Macy 216eda14cbcSMatt Macy# enable core dumps 217eda14cbcSMatt Macyulimit -c unlimited 218eda14cbcSMatt Macyexport ASAN_OPTIONS=abort_on_error=1:disable_coredump=0 219eda14cbcSMatt Macy 220eda14cbcSMatt Macyif [[ -f "$(core_file)" ]]; then 221eda14cbcSMatt Macy echo -n "There's a core dump here you might want to look at first... " 222eda14cbcSMatt Macy core_file 223eda14cbcSMatt Macy echo 224eda14cbcSMatt Macy exit 1 225eda14cbcSMatt Macyfi 226eda14cbcSMatt Macy 227eda14cbcSMatt Macyif [[ ! -d $coredir ]]; then 228eda14cbcSMatt Macy echo "core dump directory ($coredir) does not exist, creating it." 229eda14cbcSMatt Macy or_die mkdir -p "$coredir" 230eda14cbcSMatt Macyfi 231eda14cbcSMatt Macy 232eda14cbcSMatt Macyif [[ ! -w $coredir ]]; then 233eda14cbcSMatt Macy echo "core dump directory ($coredir) is not writable." 234eda14cbcSMatt Macy exit 1 235eda14cbcSMatt Macyfi 236eda14cbcSMatt Macy 237eda14cbcSMatt Macyor_die rm -f ztest.history 238eda14cbcSMatt Macyor_die rm -f ztest.zdb 239eda14cbcSMatt Macyor_die rm -f ztest.cores 240eda14cbcSMatt Macy 241eda14cbcSMatt Macyztrc=0 # ztest return value 242eda14cbcSMatt Macyfoundcrashes=0 # number of crashes found so far 243eda14cbcSMatt Macystarttime=$(date +%s) 244eda14cbcSMatt Macycurtime=$starttime 2453f9d360cSMartin Matuskaiteration=0 246eda14cbcSMatt Macy 247eda14cbcSMatt Macy# if no timeout was specified, loop forever. 2483f9d360cSMartin Matuskawhile (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do 2493f9d360cSMartin Matuska if (( iterations > 0 )) && (( iteration++ == iterations )); then 2503f9d360cSMartin Matuska break 2513f9d360cSMartin Matuska fi 2523f9d360cSMartin Matuska 253eda14cbcSMatt Macy zopt="-G -VVVVV" 254eda14cbcSMatt Macy 255eda14cbcSMatt Macy # start each run with an empty directory 256eda14cbcSMatt Macy workdir="$basedir/$rundir" 257eda14cbcSMatt Macy or_die rm -rf "$workdir" 258eda14cbcSMatt Macy or_die mkdir "$workdir" 259eda14cbcSMatt Macy 2607877fdebSMatt Macy # switch between three types of configs 2617877fdebSMatt Macy # 1/3 basic, 1/3 raidz mix, and 1/3 draid mix 2627877fdebSMatt Macy choice=$((RANDOM % 3)) 2637877fdebSMatt Macy 2647877fdebSMatt Macy # ashift range 9 - 15 265eda14cbcSMatt Macy align=$(((RANDOM % 2) * 3 + 9)) 2667877fdebSMatt Macy 2677877fdebSMatt Macy # randomly use special classes 2687877fdebSMatt Macy class="special=random" 2697877fdebSMatt Macy 2707877fdebSMatt Macy if [[ $choice -eq 0 ]]; then 2717877fdebSMatt Macy # basic mirror only 2727877fdebSMatt Macy parity=1 2737877fdebSMatt Macy mirrors=2 2747877fdebSMatt Macy draid_data=0 2757877fdebSMatt Macy draid_spares=0 2767877fdebSMatt Macy raid_children=0 2777877fdebSMatt Macy vdevs=2 2787877fdebSMatt Macy raid_type="raidz" 2797877fdebSMatt Macy elif [[ $choice -eq 1 ]]; then 2807877fdebSMatt Macy # fully randomized mirror/raidz (sans dRAID) 2817877fdebSMatt Macy parity=$(((RANDOM % 3) + 1)) 2827877fdebSMatt Macy mirrors=$(((RANDOM % 3) * 1)) 2837877fdebSMatt Macy draid_data=0 2847877fdebSMatt Macy draid_spares=0 2857877fdebSMatt Macy raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) 2867877fdebSMatt Macy vdevs=$(((RANDOM % 3) + 3)) 2877877fdebSMatt Macy raid_type="raidz" 2887877fdebSMatt Macy else 2897877fdebSMatt Macy # fully randomized dRAID (sans mirror/raidz) 2907877fdebSMatt Macy parity=$(((RANDOM % 3) + 1)) 2917877fdebSMatt Macy mirrors=0 2927877fdebSMatt Macy draid_data=$(((RANDOM % 8) + 3)) 2937877fdebSMatt Macy draid_spares=$(((RANDOM % 2) + parity)) 2947877fdebSMatt Macy stripe=$((draid_data + parity)) 2957877fdebSMatt Macy extra=$((draid_spares + (RANDOM % 4))) 2967877fdebSMatt Macy raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra)) 2977877fdebSMatt Macy vdevs=$((RANDOM % 3)) 2987877fdebSMatt Macy raid_type="draid" 2997877fdebSMatt Macy fi 3007877fdebSMatt Macy 3017877fdebSMatt Macy zopt="$zopt -K $raid_type" 302eda14cbcSMatt Macy zopt="$zopt -m $mirrors" 3037877fdebSMatt Macy zopt="$zopt -r $raid_children" 3047877fdebSMatt Macy zopt="$zopt -D $draid_data" 3057877fdebSMatt Macy zopt="$zopt -S $draid_spares" 306eda14cbcSMatt Macy zopt="$zopt -R $parity" 307eda14cbcSMatt Macy zopt="$zopt -v $vdevs" 308eda14cbcSMatt Macy zopt="$zopt -a $align" 3097877fdebSMatt Macy zopt="$zopt -C $class" 310eda14cbcSMatt Macy zopt="$zopt -s $size" 311eda14cbcSMatt Macy zopt="$zopt -f $workdir" 312eda14cbcSMatt Macy 31316038816SMartin Matuska cmd="$ZTEST $zopt $*" 314eda14cbcSMatt Macy desc="$(date '+%m/%d %T') $cmd" 315eda14cbcSMatt Macy echo "$desc" | tee -a ztest.history 316eda14cbcSMatt Macy echo "$desc" >>ztest.out 317eda14cbcSMatt Macy $cmd >>ztest.out 2>&1 318eda14cbcSMatt Macy ztrc=$? 319eda14cbcSMatt Macy grep -E '===|WARNING' ztest.out >>ztest.history 320eda14cbcSMatt Macy 321eda14cbcSMatt Macy store_core 322eda14cbcSMatt Macy 323eda14cbcSMatt Macy curtime=$(date +%s) 324eda14cbcSMatt Macydone 325eda14cbcSMatt Macy 326eda14cbcSMatt Macyecho "zloop finished, $foundcrashes crashes found" 327eda14cbcSMatt Macy 328eda14cbcSMatt Macy# restore core pattern. 329eda14cbcSMatt Macycase $(uname) in 330eda14cbcSMatt MacyLinux) 331eda14cbcSMatt Macy echo "$origcorepattern" > /proc/sys/kernel/core_pattern 332eda14cbcSMatt Macy ;; 333eda14cbcSMatt Macy*) 334eda14cbcSMatt Macy ;; 335eda14cbcSMatt Macyesac 336eda14cbcSMatt Macy 337eda14cbcSMatt Macyuptime >>ztest.out 338eda14cbcSMatt Macy 339eda14cbcSMatt Macyif [[ $foundcrashes -gt 0 ]]; then 340eda14cbcSMatt Macy exit 1 341eda14cbcSMatt Macyfi 342