1#!/bin/bash 2 3# 4# CDDL HEADER START 5# 6# This file and its contents are supplied under the terms of the 7# Common Development and Distribution License ("CDDL"), version 1.0. 8# You may only use this file in accordance with the terms of version 9# 1.0 of the CDDL. 10# 11# A full copy of the text of the CDDL should have accompanied this 12# source. A copy of the CDDL is also available via the Internet at 13# http://www.illumos.org/license/CDDL. 14# 15# CDDL HEADER END 16# 17 18# 19# Copyright (c) 2015, 2017 by Delphix. All rights reserved. 20# 21 22set -x 23export BITS=64 24export UMEM_DEBUG=default,verbose 25export UMEM_LOGGING=transaction,contents 26set +x 27 28sparc_32=sparc 29sparc_64=sparcv9 30i386_32=i86 31i386_64=amd64 32ARCH=`uname -p` 33eval 'ARCHBITS=${'"${ARCH}_${BITS}"'}' 34BIN=$ROOT/usr/bin/${ARCHBITS} 35SBIN=$ROOT/usr/sbin/${ARCHBITS} 36DEFAULTWORKDIR=/var/tmp 37DEFAULTCOREDIR=/var/tmp/zloop 38 39function usage 40{ 41 echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \ 42 "[ -- [extra ztest parameters]]\n" \ 43 "\n" \ 44 " This script runs ztest repeatedly with randomized arguments.\n" \ 45 " If a crash is encountered, the ztest logs, any associated\n" \ 46 " vdev files, and core file (if one exists) are moved to the\n" \ 47 " output directory ($DEFAULTCOREDIR by default). Any options\n" \ 48 " after the -- end-of-options marker will be passed to ztest.\n" \ 49 "\n" \ 50 " Options:\n" \ 51 " -t Total time to loop for, in seconds. If not provided,\n" \ 52 " zloop runs forever.\n" \ 53 " -f Specify working directory for ztest vdev files.\n" \ 54 " -c Specify a core dump directory to use.\n" \ 55 " -h Print this help message.\n" \ 56 "" >&2 57} 58 59function or_die 60{ 61 $@ 62 if [[ $? -ne 0 ]]; then 63 echo "Command failed: $@" 64 exit 1 65 fi 66} 67 68function store_core 69{ 70 if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then 71 coreid=$(/bin/date "+zloop-%y%m%d-%H%M%S") 72 foundcrashes=$(($foundcrashes + 1)) 73 74 dest=$coredir/$coreid 75 or_die /bin/mkdir $dest 76 or_die /bin/mkdir $dest/vdev 77 78 echo "*** ztest crash found - moving logs to $coredir/$coreid" 79 80 or_die /bin/mv ztest.history $dest/ 81 or_die /bin/mv ztest.out $dest/ 82 or_die /bin/mv $workdir/ztest* $dest/vdev/ 83 or_die /bin/mv $workdir/zpool.cache $dest/vdev/ 84 85 # check for core 86 if [[ -f core ]]; then 87 corestatus=$(mdb -e "::status" core) 88 corestack=$(mdb -e "::stack" core) 89 90 # Dump core + logs to stored directory 91 echo "$corestatus" >>$dest/status 92 echo "$corestack" >>$dest/status 93 or_die /bin/mv core $dest/ 94 95 # Record info in cores logfile 96 echo "*** core @ $coredir/$coreid/core:" | /bin/tee -a ztest.cores 97 echo "$corestatus" | /bin/tee -a ztest.cores 98 echo "$corestack" | /bin/tee -a ztest.cores 99 echo "" | /bin/tee -a ztest.cores 100 fi 101 echo "continuing..." 102 fi 103} 104 105set -x 106export PATH=${BIN}:${SBIN} 107export LD_LIBRARY_PATH=$ROOT/lib/$BITS:$ROOT/usr/lib/$BITS 108set +x 109 110# parse arguments 111# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] 112coredir=$DEFAULTCOREDIR 113workdir=$DEFAULTWORKDIR 114timeout=0 115while getopts ":ht:c:f:" opt; do 116 case $opt in 117 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; 118 c ) [[ $OPTARG ]] && coredir=$OPTARG ;; 119 f ) [[ $OPTARG ]] && workdir=$(/usr/bin/readlink -f $OPTARG) ;; 120 h ) usage 121 exit 2 122 ;; 123 * ) echo "Invalid argument: -$OPTARG"; 124 usage 125 exit 1 126 esac 127done 128# pass remaining arguments on to ztest 129shift $((OPTIND - 1)) 130 131if [[ -f core ]]; then 132 echo "There's a core dump here you might want to look at first." 133 exit 1 134fi 135 136if [[ ! -d $coredir ]]; then 137 echo "core dump directory ($coredir) does not exist, creating it." 138 or_die /bin/mkdir -p $coredir 139fi 140 141if [[ ! -w $coredir ]]; then 142 echo "core dump directory ($coredir) is not writable." 143 exit 1 144fi 145 146or_die /bin/rm -f ztest.history 147or_die /bin/rm -f ztest.cores 148 149# Allow core files to be written to cwd if that's currently disabled. 150sudo coreadm -e process 151 152ztrc=0 # ztest return value 153foundcrashes=0 # number of crashes found so far 154starttime=$(/bin/date +%s) 155curtime=$starttime 156 157# if no timeout was specified, loop forever. 158while [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do 159 zopt="-VVVVV" 160 161 # switch between common arrangements & fully randomized 162 if [[ $((RANDOM % 2)) -eq 0 ]]; then 163 mirrors=2 164 raidz=0 165 parity=1 166 vdevs=2 167 else 168 mirrors=$(((RANDOM % 3) * 1)) 169 parity=$(((RANDOM % 3) + 1)) 170 raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) 171 vdevs=$(((RANDOM % 3) + 3)) 172 fi 173 align=$(((RANDOM % 2) * 3 + 9)) 174 runtime=$((RANDOM % 100)) 175 passtime=$((RANDOM % (runtime / 3 + 1) + 10)) 176 size=128m 177 178 zopt="$zopt -m $mirrors" 179 zopt="$zopt -r $raidz" 180 zopt="$zopt -R $parity" 181 zopt="$zopt -v $vdevs" 182 zopt="$zopt -a $align" 183 zopt="$zopt -T $runtime" 184 zopt="$zopt -P $passtime" 185 zopt="$zopt -s $size" 186 zopt="$zopt -f $workdir" 187 188 cmd="ztest $zopt $@" 189 desc="$(/bin/date '+%m/%d %T') $cmd" 190 echo "$desc" | /bin/tee -a ztest.history 191 echo "$desc" >>ztest.out 192 $BIN/$cmd >>ztest.out 2>&1 193 ztrc=$? 194 /bin/egrep '===|WARNING' ztest.out >>ztest.history 195 $SBIN/zdb -U $workdir/zpool.cache -DD ztest >>ztest.ddt 2>&1 196 197 store_core 198 199 curtime=$(/bin/date +%s) 200done 201 202echo "zloop finished, $foundcrashes crashes found" 203 204/bin/uptime >>ztest.out 205 206if [[ $foundcrashes -gt 0 ]]; then 207 exit 1 208fi 209