1#!/usr/bin/env bash 2# SPDX-License-Identifier: CDDL-1.0 3 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# https://opensource.org/license/CDDL-1.0. 13# 14 15# 16# Copyright (c) 2015 by Delphix. All rights reserved. 17# Copyright (C) 2016 Lawrence Livermore National Security, LLC. 18# Copyright (c) 2017, Intel Corporation. 19# 20 21BASE_DIR=${0%/*} 22SCRIPT_COMMON=common.sh 23if [[ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]]; then 24 . "${BASE_DIR}/${SCRIPT_COMMON}" 25else 26 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1 27fi 28 29# shellcheck disable=SC2034 30PROG=zloop.sh 31GDB=${GDB:-gdb} 32 33DEFAULTWORKDIR=/var/tmp 34DEFAULTCOREDIR=/var/tmp/zloop 35 36function usage 37{ 38 cat >&2 <<EOF 39 40$0 [-hlM] [-c <dump directory>] [-f <vdev directory>] 41 [-m <max core dumps>] [-s <vdev size>] [-t <timeout>] 42 [-I <max iterations>] [-- [extra ztest parameters]] 43 44 This script runs ztest repeatedly with randomized arguments. 45 If a crash is encountered, the ztest logs, any associated 46 vdev files, and core file (if one exists) are moved to the 47 output directory ($DEFAULTCOREDIR by default). Any options 48 after the -- end-of-options marker will be passed to ztest. 49 50 Options: 51 -c Specify a core dump directory to use. 52 -f Specify working directory for ztest vdev files. 53 -h Print this help message. 54 -l Create 'ztest.core.N' symlink to core directory. 55 -M Enable multihost testing on a fraction of the iterations. 56 A hostid is exported through ZFS_HOSTID for those runs. 57 -m Max number of core dumps to allow before exiting. 58 -s Size of vdev devices. 59 -t Total time to loop for, in seconds. If not provided, 60 zloop runs forever. 61 -I Max number of iterations to loop before exiting. 62 63EOF 64} 65 66function or_die 67{ 68 if ! "$@"; then 69 echo "Command failed: $*" 70 exit 1 71 fi 72} 73 74case $(uname) in 75FreeBSD) 76 coreglob="z*.core" 77 ;; 78Linux) 79 # core file helpers 80 read -r origcorepattern </proc/sys/kernel/core_pattern 81 coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" 82 83 if [[ $coreglob = "*" ]]; then 84 echo "Setting core file pattern..." 85 echo "core" > /proc/sys/kernel/core_pattern 86 coreglob="$(grep -E -o '^([^|%[:space:]]*)' \ 87 /proc/sys/kernel/core_pattern)*" 88 fi 89 ;; 90*) 91 exit 1 92 ;; 93esac 94 95function core_file 96{ 97 # shellcheck disable=SC2012,SC2086 98 ls -tr1 $coreglob 2>/dev/null | head -1 99} 100 101function core_prog 102{ 103 # shellcheck disable=SC2154 104 prog=$ZTEST 105 core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \ 106 tr \' ' ') 107 if [[ "$core_id" == *"zdb "* ]]; then 108 # shellcheck disable=SC2154 109 prog=$ZDB 110 fi 111 printf "%s" "$prog" 112} 113 114function store_core 115{ 116 core="$(core_file)" 117 if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then 118 df -h "$workdir" >>ztest.out 119 coreid=$(date "+zloop-%y%m%d-%H%M%S") 120 foundcrashes=$((foundcrashes + 1)) 121 122 # zdb debugging 123 zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest" 124 zdbdebug=$($zdbcmd 2>&1) 125 echo -e "$zdbcmd\n" >>ztest.zdb 126 echo "$zdbdebug" >>ztest.zdb 127 128 dest=$coredir/$coreid 129 or_die mkdir -p "$dest/vdev" 130 131 if [[ $symlink -ne 0 ]]; then 132 or_die ln -sf "$dest" "ztest.core.${foundcrashes}" 133 fi 134 135 echo "*** ztest crash found - moving logs to $dest" 136 137 or_die mv ztest.history ztest.zdb ztest.out "$dest/" 138 or_die mv "$workdir/"ztest* "$dest/vdev/" 139 140 if [[ -e "$workdir/zpool.cache" ]]; then 141 or_die mv "$workdir/zpool.cache" "$dest/vdev/" 142 fi 143 144 # check for core 145 if [[ -f "$core" ]]; then 146 coreprog=$(core_prog "$core") 147 coredebug=$($GDB --batch --quiet \ 148 -ex "set print thread-events off" \ 149 -ex "printf \"*\n* Backtrace \n*\n\"" \ 150 -ex "bt" \ 151 -ex "printf \"*\n* Libraries \n*\n\"" \ 152 -ex "info sharedlib" \ 153 -ex "printf \"*\n* Threads (full) \n*\n\"" \ 154 -ex "info threads" \ 155 -ex "printf \"*\n* Backtraces \n*\n\"" \ 156 -ex "thread apply all bt" \ 157 -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ 158 -ex "thread apply all bt full" \ 159 -ex "quit" "$coreprog" "$core" 2>&1 | \ 160 grep -v "New LWP") 161 162 # Dump core + logs to stored directory 163 echo "$coredebug" >>"$dest/ztest.gdb" 164 or_die mv "$core" "$dest/" 165 166 # Record info in cores logfile 167 echo "*** core @ $coredir/$coreid/$core:" | \ 168 tee -a ztest.cores 169 fi 170 171 if [[ $coremax -gt 0 ]] && 172 [[ $foundcrashes -ge $coremax ]]; then 173 echo "exiting... max $coremax allowed cores" 174 exit 1 175 else 176 echo "continuing..." 177 fi 178 fi 179} 180 181# parse arguments 182# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] 183coredir=$DEFAULTCOREDIR 184basedir=$DEFAULTWORKDIR 185rundir="zloop-run" 186timeout=0 187size="512m" 188coremax=0 189symlink=0 190iterations=0 191multihost=0 192while getopts ":ht:m:I:s:c:f:lM" opt; do 193 case $opt in 194 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; 195 m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;; 196 I ) [[ -n $OPTARG ]] && iterations=$OPTARG ;; 197 s ) [[ -n $OPTARG ]] && size=$OPTARG ;; 198 c ) [[ -n $OPTARG ]] && coredir=$OPTARG ;; 199 f ) [[ -n $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;; 200 l ) symlink=1 ;; 201 M ) multihost=1 ;; 202 h ) usage 203 exit 2 204 ;; 205 * ) echo "Invalid argument: -$OPTARG"; 206 usage 207 exit 1 208 esac 209done 210# pass remaining arguments on to ztest 211shift $((OPTIND - 1)) 212 213# enable core dumps 214ulimit -c unlimited 215export ASAN_OPTIONS=abort_on_error=true:halt_on_error=true:allocator_may_return_null=true:disable_coredump=false:detect_stack_use_after_return=true 216export UBSAN_OPTIONS=abort_on_error=true:halt_on_error=true:print_stacktrace=true 217 218if [[ -f "$(core_file)" ]]; then 219 echo -n "There's a core dump here you might want to look at first... " 220 core_file 221 echo 222 exit 1 223fi 224 225if [[ ! -d $coredir ]]; then 226 echo "core dump directory ($coredir) does not exist, creating it." 227 or_die mkdir -p "$coredir" 228fi 229 230if [[ ! -w $coredir ]]; then 231 echo "core dump directory ($coredir) is not writable." 232 exit 1 233fi 234 235or_die rm -f ztest.history ztest.zdb ztest.cores 236 237ztrc=0 # ztest return value 238foundcrashes=0 # number of crashes found so far 239starttime=$(date +%s) 240curtime=$starttime 241iteration=0 242 243# if no timeout was specified, loop forever. 244while (( timeout == 0 )) || (( curtime <= (starttime + timeout) )); do 245 if (( iterations > 0 )) && (( iteration++ == iterations )); then 246 break 247 fi 248 249 zopt="-G -VVVVV" 250 251 # start each run with an empty directory 252 workdir="$basedir/$rundir" 253 or_die rm -rf "$workdir" 254 or_die mkdir "$workdir" 255 256 # ashift range 9 - 15 257 align=$(((RANDOM % 2) * 3 + 9)) 258 259 # choose parity value 260 parity=$(((RANDOM % 3) + 1)) 261 262 draid_data=0 263 draid_spares=0 264 expand=0 265 266 # randomly use special classes 267 class="special=random" 268 269 # choose between four types of configs 270 # (basic, raidz mix, raidz expansion, and draid mix) 271 case $((RANDOM % 4)) in 272 273 # basic mirror configuration 274 0) parity=1 275 mirrors=2 276 raid_children=0 277 vdevs=2 278 raid_type="raidz" 279 ;; 280 281 # fully randomized mirror/raidz (sans dRAID) 282 1) mirrors=$(((RANDOM % 3) * 1)) 283 raid_children=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) 284 vdevs=$(((RANDOM % 3) + 3)) 285 raid_type="raidz" 286 ;; 287 288 # randomized raidz expansion (one top-level raidz vdev) 289 2) mirrors=0 290 vdevs=1 291 # derive initial raidz disk count based on parity choice 292 # P1: 3 - 7 disks 293 # P2: 5 - 9 disks 294 # P3: 7 - 11 disks 295 raid_children=$(((RANDOM % 5) + (parity * 2) + 1)) 296 297 # 1/3 of the time use a dedicated '-X' raidz expansion test 298 if [[ $((RANDOM % 3)) -eq 0 ]]; then 299 zopt="$zopt -X -t 16" 300 expand=1 301 raid_type="raidz" 302 else 303 raid_type="eraidz" 304 fi 305 ;; 306 307 # fully randomized dRAID (sans mirror/raidz) 308 3) mirrors=0 309 draid_data=$(((RANDOM % 8) + 3)) 310 draid_spares=$(((RANDOM % 2) + parity)) 311 stripe=$((draid_data + parity)) 312 extra=$((draid_spares + (RANDOM % 4))) 313 raid_children=$(((((RANDOM % 4) + 1) * stripe) + extra)) 314 vdevs=$((RANDOM % 3)) 315 raid_type="draid" 316 ;; 317 *) 318 # avoid shellcheck SC2249 319 ;; 320 esac 321 322 zopt="$zopt -K $raid_type" 323 zopt="$zopt -m $mirrors" 324 zopt="$zopt -r $raid_children" 325 zopt="$zopt -D $draid_data" 326 zopt="$zopt -S $draid_spares" 327 zopt="$zopt -R $parity" 328 zopt="$zopt -v $vdevs" 329 zopt="$zopt -a $align" 330 zopt="$zopt -C $class" 331 zopt="$zopt -s $size" 332 zopt="$zopt -f $workdir" 333 334 # 335 # Exercise multihost on a fraction of the iterations rather than all 336 # of them: -M suppresses roughly a dozen other ztest operations, so 337 # running it everywhere would remove that coverage from the loop. 338 # A raidz expansion run is skipped because ztest forces -M off for 339 # one, which would spend a multihost iteration on a run that does 340 # no multihost testing. 341 # 342 # ztest needs a non-zero hostid to set the multihost property, and 343 # ZFS_HOSTID provides one without creating /etc/hostid, which the ZTS 344 # mmp test group requires to be absent. 345 # 346 if [[ $multihost -eq 1 ]]; then 347 unset ZFS_HOSTID 348 if [[ $expand -eq 0 ]] && [[ $((RANDOM % 5)) -eq 0 ]]; then 349 zopt="$zopt -M" 350 export ZFS_HOSTID=$((RANDOM + 1)) 351 fi 352 fi 353 354 cmd="$ZTEST $zopt $*" 355 echo "$(date '+%m/%d %T') $cmd" | tee -a ztest.history ztest.out 356 $cmd >>ztest.out 2>&1 357 ztrc=$? 358 grep -E '===|WARNING' ztest.out >>ztest.history 359 360 store_core 361 362 curtime=$(date +%s) 363done 364 365echo "zloop finished, $foundcrashes crashes found" 366 367# restore core pattern. 368case $(uname) in 369Linux) 370 echo "$origcorepattern" > /proc/sys/kernel/core_pattern 371 ;; 372*) 373 ;; 374esac 375 376uptime >>ztest.out 377 378if [[ $foundcrashes -gt 0 ]]; then 379 exit 1 380fi 381