1#!/bin/bash 2 3# 4# CDDL HEADER START 5# 6# This file and its contents are supplied under the terms of the 7# Common Development and Distribution License ("CDDL"), version 1.0. 8# You may only use this file in accordance with the terms of version 9# 1.0 of the CDDL. 10# 11# A full copy of the text of the CDDL should have accompanied this 12# source. A copy of the CDDL is also available via the Internet at 13# http://www.illumos.org/license/CDDL. 14# 15# CDDL HEADER END 16# 17 18# 19# Copyright (c) 2015, 2017 by Delphix. All rights reserved. 20# 21 22set -x 23export UMEM_DEBUG=default,verbose 24export UMEM_LOGGING=transaction,contents 25set +x 26 27DEFAULTWORKDIR=/var/tmp 28DEFAULTCOREDIR=/var/tmp/zloop 29 30function usage 31{ 32 echo -e "\n$0 [-t <timeout>] [-c <dump directory>]" \ 33 "[ -- [extra ztest parameters]]\n" \ 34 "\n" \ 35 " This script runs ztest repeatedly with randomized arguments.\n" \ 36 " If a crash is encountered, the ztest logs, any associated\n" \ 37 " vdev files, and core file (if one exists) are moved to the\n" \ 38 " output directory ($DEFAULTCOREDIR by default). Any options\n" \ 39 " after the -- end-of-options marker will be passed to ztest.\n" \ 40 "\n" \ 41 " Options:\n" \ 42 " -t Total time to loop for, in seconds. If not provided,\n" \ 43 " zloop runs forever.\n" \ 44 " -f Specify working directory for ztest vdev files.\n" \ 45 " -c Specify a core dump directory to use.\n" \ 46 " -h Print this help message.\n" \ 47 "" >&2 48} 49 50function or_die 51{ 52 $@ 53 if [[ $? -ne 0 ]]; then 54 echo "Command failed: $@" 55 exit 1 56 fi 57} 58 59function store_core 60{ 61 if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then 62 coreid=$(/bin/date "+zloop-%y%m%d-%H%M%S") 63 foundcrashes=$(($foundcrashes + 1)) 64 65 dest=$coredir/$coreid 66 or_die /bin/mkdir $dest 67 or_die /bin/mkdir $dest/vdev 68 69 echo "*** ztest crash found - moving logs to $coredir/$coreid" 70 71 or_die /bin/mv ztest.history $dest/ 72 or_die /bin/mv ztest.out $dest/ 73 or_die /bin/mv $workdir/ztest* $dest/vdev/ 74 or_die /bin/mv $workdir/zpool.cache $dest/vdev/ 75 76 # check for core 77 if [[ -f core ]]; then 78 corestatus=$(mdb -e "::status" core) 79 corestack=$(mdb -e "::stack" core) 80 81 # Dump core + logs to stored directory 82 echo "$corestatus" >>$dest/status 83 echo "$corestack" >>$dest/status 84 or_die /bin/mv core $dest/ 85 86 # Record info in cores logfile 87 echo "*** core @ $coredir/$coreid/core:" | /bin/tee -a ztest.cores 88 echo "$corestatus" | /bin/tee -a ztest.cores 89 echo "$corestack" | /bin/tee -a ztest.cores 90 echo "" | /bin/tee -a ztest.cores 91 fi 92 echo "continuing..." 93 fi 94} 95 96# parse arguments 97# expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args] 98coredir=$DEFAULTCOREDIR 99workdir=$DEFAULTWORKDIR 100timeout=0 101while getopts ":ht:c:f:" opt; do 102 case $opt in 103 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;; 104 c ) [[ $OPTARG ]] && coredir=$OPTARG ;; 105 f ) [[ $OPTARG ]] && workdir=$(/usr/bin/readlink -f $OPTARG) ;; 106 h ) usage 107 exit 2 108 ;; 109 * ) echo "Invalid argument: -$OPTARG"; 110 usage 111 exit 1 112 esac 113done 114# pass remaining arguments on to ztest 115shift $((OPTIND - 1)) 116 117if [[ -f core ]]; then 118 echo "There's a core dump here you might want to look at first." 119 exit 1 120fi 121 122if [[ ! -d $coredir ]]; then 123 echo "core dump directory ($coredir) does not exist, creating it." 124 or_die /bin/mkdir -p $coredir 125fi 126 127if [[ ! -w $coredir ]]; then 128 echo "core dump directory ($coredir) is not writable." 129 exit 1 130fi 131 132or_die /bin/rm -f ztest.history 133or_die /bin/rm -f ztest.cores 134 135# Allow core files to be written to cwd if that's currently disabled. 136sudo coreadm -e process 137 138ztrc=0 # ztest return value 139foundcrashes=0 # number of crashes found so far 140starttime=$(/bin/date +%s) 141curtime=$starttime 142 143# if no timeout was specified, loop forever. 144while [[ $timeout -eq 0 ]] || [[ $curtime -le $(($starttime + $timeout)) ]]; do 145 zopt="-VVVVV" 146 147 # switch between common arrangements & fully randomized 148 if [[ $((RANDOM % 2)) -eq 0 ]]; then 149 mirrors=2 150 raidz=0 151 parity=1 152 vdevs=2 153 else 154 mirrors=$(((RANDOM % 3) * 1)) 155 parity=$(((RANDOM % 3) + 1)) 156 raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2))) 157 vdevs=$(((RANDOM % 3) + 3)) 158 fi 159 align=$(((RANDOM % 2) * 3 + 9)) 160 runtime=$((RANDOM % 100)) 161 passtime=$((RANDOM % (runtime / 3 + 1) + 10)) 162 size=128m 163 164 zopt="$zopt -m $mirrors" 165 zopt="$zopt -r $raidz" 166 zopt="$zopt -R $parity" 167 zopt="$zopt -v $vdevs" 168 zopt="$zopt -a $align" 169 zopt="$zopt -T $runtime" 170 zopt="$zopt -P $passtime" 171 zopt="$zopt -s $size" 172 zopt="$zopt -f $workdir" 173 174 cmd="ztest $zopt $@" 175 desc="$(/bin/date '+%m/%d %T') $cmd" 176 echo "$desc" | /bin/tee -a ztest.history 177 echo "$desc" >>ztest.out 178 /usr/bin/$cmd >>ztest.out 2>&1 179 ztrc=$? 180 /bin/egrep '===|WARNING' ztest.out >>ztest.history 181 /usr/sbin/zdb -U $workdir/zpool.cache -DD ztest >>ztest.ddt 2>&1 182 183 store_core 184 185 curtime=$(/bin/date +%s) 186done 187 188echo "zloop finished, $foundcrashes crashes found" 189 190/bin/uptime >>ztest.out 191 192if [[ $foundcrashes -gt 0 ]]; then 193 exit 1 194fi 195