1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Run a series of torture tests, intended for overnight or 5# longer timeframes, and also for large systems. 6# 7# Usage: torture.sh [ options ] 8# 9# Copyright (C) 2020 Facebook, Inc. 10# 11# Authors: Paul E. McKenney <paulmck@kernel.org> 12 13scriptname=$0 14args="$*" 15 16KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM 17PATH=${KVM}/bin:$PATH; export PATH 18. functions.sh 19 20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`" 21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2)) 22HALF_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2)) 23if test "$HALF_ALLOTED_CPUS" -lt 1 24then 25 HALF_ALLOTED_CPUS=1 26fi 27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16)) 28if test "$VERBOSE_BATCH_CPUS" -lt 2 29then 30 VERBOSE_BATCH_CPUS=0 31fi 32 33# Configurations/scenarios. 34configs_rcutorture= 35configs_locktorture= 36configs_scftorture= 37kcsan_kmake_args= 38 39# Default compression, duration, and apportionment. 40compress_kasan_vmlinux="`identify_qemu_vcpus`" 41duration_base=10 42duration_rcutorture_frac=7 43duration_locktorture_frac=1 44duration_scftorture_frac=2 45 46# "yes" or "no" parameters 47do_allmodconfig=yes 48do_rcutorture=yes 49do_locktorture=yes 50do_scftorture=yes 51do_rcuscale=yes 52do_refscale=yes 53do_kvfree=yes 54do_kasan=yes 55do_kcsan=no 56 57# doyesno - Helper function for yes/no arguments 58function doyesno () { 59 if test "$1" = "$2" 60 then 61 echo yes 62 else 63 echo no 64 fi 65} 66 67usage () { 68 echo "Usage: $scriptname optional arguments:" 69 echo " --compress-kasan-vmlinux concurrency" 70 echo " --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\"" 71 echo " --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\"" 72 echo " --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\"" 73 echo " --doall" 74 echo " --doallmodconfig / --do-no-allmodconfig" 75 echo " --do-kasan / --do-no-kasan" 76 echo " --do-kcsan / --do-no-kcsan" 77 echo " --do-kvfree / --do-no-kvfree" 78 echo " --do-locktorture / --do-no-locktorture" 79 echo " --do-none" 80 echo " --do-rcuscale / --do-no-rcuscale" 81 echo " --do-rcutorture / --do-no-rcutorture" 82 echo " --do-refscale / --do-no-refscale" 83 echo " --do-scftorture / --do-no-scftorture" 84 echo " --duration [ <minutes> | <hours>h | <days>d ]" 85 echo " --kcsan-kmake-arg kernel-make-arguments" 86 exit 1 87} 88 89while test $# -gt 0 90do 91 case "$1" in 92 --compress-kasan-vmlinux) 93 checkarg --compress-kasan-vmlinux "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error' 94 compress_kasan_vmlinux=$2 95 shift 96 ;; 97 --config-rcutorture|--configs-rcutorture) 98 checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 99 configs_rcutorture="$configs_rcutorture $2" 100 shift 101 ;; 102 --config-locktorture|--configs-locktorture) 103 checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 104 configs_locktorture="$configs_locktorture $2" 105 shift 106 ;; 107 --config-scftorture|--configs-scftorture) 108 checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 109 configs_scftorture="$configs_scftorture $2" 110 shift 111 ;; 112 --doall) 113 do_allmodconfig=yes 114 do_rcutorture=yes 115 do_locktorture=yes 116 do_scftorture=yes 117 do_rcuscale=yes 118 do_refscale=yes 119 do_kvfree=yes 120 do_kasan=yes 121 do_kcsan=yes 122 ;; 123 --do-allmodconfig|--do-no-allmodconfig) 124 do_allmodconfig=`doyesno "$1" --do-allmodconfig` 125 ;; 126 --do-kasan|--do-no-kasan) 127 do_kasan=`doyesno "$1" --do-kasan` 128 ;; 129 --do-kcsan|--do-no-kcsan) 130 do_kcsan=`doyesno "$1" --do-kcsan` 131 ;; 132 --do-kvfree|--do-no-kvfree) 133 do_kvfree=`doyesno "$1" --do-kvfree` 134 ;; 135 --do-locktorture|--do-no-locktorture) 136 do_locktorture=`doyesno "$1" --do-locktorture` 137 ;; 138 --do-none) 139 do_allmodconfig=no 140 do_rcutorture=no 141 do_locktorture=no 142 do_scftorture=no 143 do_rcuscale=no 144 do_refscale=no 145 do_kvfree=no 146 do_kasan=no 147 do_kcsan=no 148 ;; 149 --do-rcuscale|--do-no-rcuscale) 150 do_rcuscale=`doyesno "$1" --do-rcuscale` 151 ;; 152 --do-rcutorture|--do-no-rcutorture) 153 do_rcutorture=`doyesno "$1" --do-rcutorture` 154 ;; 155 --do-refscale|--do-no-refscale) 156 do_refscale=`doyesno "$1" --do-refscale` 157 ;; 158 --do-scftorture|--do-no-scftorture) 159 do_scftorture=`doyesno "$1" --do-scftorture` 160 ;; 161 --duration) 162 checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error' 163 mult=1 164 if echo "$2" | grep -q 'm$' 165 then 166 mult=1 167 elif echo "$2" | grep -q 'h$' 168 then 169 mult=60 170 elif echo "$2" | grep -q 'd$' 171 then 172 mult=1440 173 fi 174 ts=`echo $2 | sed -e 's/[smhd]$//'` 175 duration_base=$(($ts*mult)) 176 shift 177 ;; 178 --kcsan-kmake-arg|--kcsan-kmake-args) 179 checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' 180 kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`" 181 shift 182 ;; 183 *) 184 echo Unknown argument $1 185 usage 186 ;; 187 esac 188 shift 189done 190 191ds="`date +%Y.%m.%d-%H.%M.%S`-torture" 192startdate="`date`" 193starttime="`get_starttime`" 194 195T=/tmp/torture.sh.$$ 196trap 'rm -rf $T' 0 2 197mkdir $T 198 199echo " --- " $scriptname $args | tee -a $T/log 200echo " --- Results directory: " $ds | tee -a $T/log 201 202# Calculate rcutorture defaults and apportion time 203if test -z "$configs_rcutorture" 204then 205 configs_rcutorture=CFLIST 206fi 207duration_rcutorture=$((duration_base*duration_rcutorture_frac/10)) 208if test "$duration_rcutorture" -eq 0 209then 210 echo " --- Zero time for rcutorture, disabling" | tee -a $T/log 211 do_rcutorture=no 212fi 213 214# Calculate locktorture defaults and apportion time 215if test -z "$configs_locktorture" 216then 217 configs_locktorture=CFLIST 218fi 219duration_locktorture=$((duration_base*duration_locktorture_frac/10)) 220if test "$duration_locktorture" -eq 0 221then 222 echo " --- Zero time for locktorture, disabling" | tee -a $T/log 223 do_locktorture=no 224fi 225 226# Calculate scftorture defaults and apportion time 227if test -z "$configs_scftorture" 228then 229 configs_scftorture=CFLIST 230fi 231duration_scftorture=$((duration_base*duration_scftorture_frac/10)) 232if test "$duration_scftorture" -eq 0 233then 234 echo " --- Zero time for scftorture, disabling" | tee -a $T/log 235 do_scftorture=no 236fi 237 238touch $T/failures 239touch $T/successes 240 241# torture_one - Does a single kvm.sh run. 242# 243# Usage: 244# torture_bootargs="[ kernel boot arguments ]" 245# torture_one flavor [ kvm.sh arguments ] 246# 247# Note that "flavor" is an arbitrary string. Supply --torture if needed. 248# Note that quoting is problematic. So on the command line, pass multiple 249# values with multiple kvm.sh argument instances. 250function torture_one { 251 local cur_bootargs= 252 local boottag= 253 254 echo " --- $curflavor:" Start `date` | tee -a $T/log 255 if test -n "$torture_bootargs" 256 then 257 boottag="--bootargs" 258 cur_bootargs="$torture_bootargs" 259 fi 260 "$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1 261 retcode=$? 262 resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`" 263 if test -z "$resdir" 264 then 265 cat $T/$curflavor.out | tee -a $T/log 266 echo retcode=$retcode | tee -a $T/log 267 fi 268 if test "$retcode" == 0 269 then 270 echo "$curflavor($retcode)" $resdir >> $T/successes 271 else 272 echo "$curflavor($retcode)" $resdir >> $T/failures 273 fi 274} 275 276# torture_set - Does a set of tortures with and without KASAN and KCSAN. 277# 278# Usage: 279# torture_bootargs="[ kernel boot arguments ]" 280# torture_set flavor [ kvm.sh arguments ] 281# 282# Note that "flavor" is an arbitrary string. Supply --torture if needed. 283# Note that quoting is problematic. So on the command line, pass multiple 284# values with multiple kvm.sh argument instances. 285function torture_set { 286 local cur_kcsan_kmake_args= 287 local kcsan_kmake_tag= 288 local flavor=$1 289 shift 290 curflavor=$flavor 291 torture_one "$@" 292 if test "$do_kasan" = "yes" 293 then 294 curflavor=${flavor}-kasan 295 torture_one "$@" --kasan 296 fi 297 if test "$do_kcsan" = "yes" 298 then 299 curflavor=${flavor}-kcsan 300 if test -n "$kcsan_kmake_args" 301 then 302 kcsan_kmake_tag="--kmake-args" 303 cur_kcsan_kmake_args="$kcsan_kmake_args" 304 fi 305 torture_one $* --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan 306 fi 307} 308 309# make allmodconfig 310if test "$do_allmodconfig" = "yes" 311then 312 echo " --- allmodconfig:" Start `date` | tee -a $T/log 313 amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig" 314 mkdir -p "$amcdir" 315 echo " --- make clean" > "$amcdir/Make.out" 2>&1 316 make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1 317 echo " --- make allmodconfig" >> "$amcdir/Make.out" 2>&1 318 make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1 319 echo " --- make " >> "$amcdir/Make.out" 2>&1 320 make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1 321 retcode="$?" 322 echo $retcode > "$amcdir/Make.exitcode" 323 if test "$retcode" == 0 324 then 325 echo "allmodconfig($retcode)" $amcdir >> $T/successes 326 else 327 echo "allmodconfig($retcode)" $amcdir >> $T/failures 328 fi 329fi 330 331# --torture rcu 332if test "$do_rcutorture" = "yes" 333then 334 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000" 335 torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make 336fi 337 338if test "$do_locktorture" = "yes" 339then 340 torture_bootargs="torture.disable_onoff_at_boot" 341 torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make 342fi 343 344if test "$do_scftorture" = "yes" 345then 346 torture_bootargs="scftorture.nthreads=$HALF_ALLOTED_CPUS torture.disable_onoff_at_boot" 347 torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make 348fi 349 350if test "$do_refscale" = yes 351then 352 primlist="`grep '\.name[ ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" 353else 354 primlist= 355fi 356for prim in $primlist 357do 358 torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$HALF_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot" 359 torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --bootargs "verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make 360done 361 362if test "$do_rcuscale" = yes 363then 364 primlist="`grep '\.name[ ]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" 365else 366 primlist= 367fi 368for prim in $primlist 369do 370 torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$HALF_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot" 371 torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make 372done 373 374if test "$do_kvfree" = "yes" 375then 376 torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" 377 torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 10 --kconfig "CONFIG_NR_CPUS=$HALF_ALLOTED_CPUS" --trust-make 378fi 379 380echo " --- " $scriptname $args 381echo " --- " Done `date` | tee -a $T/log 382ret=0 383nsuccesses=0 384echo SUCCESSES: | tee -a $T/log 385if test -s "$T/successes" 386then 387 cat "$T/successes" | tee -a $T/log 388 nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`" 389fi 390nfailures=0 391echo FAILURES: | tee -a $T/log 392if test -s "$T/failures" 393then 394 cat "$T/failures" | tee -a $T/log 395 nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`" 396 ret=2 397fi 398echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log 399echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log 400tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`" 401if test -n "$tdir" && test $compress_kasan_vmlinux -gt 0 402then 403 # KASAN vmlinux files can approach 1GB in size, so compress them. 404 echo Looking for KASAN files to compress: `date` > "$tdir/log-xz" 2>&1 405 find "$tdir" -type d -name '*-kasan' -print > $T/xz-todo 406 ncompresses=0 407 batchno=1 408 if test -s $T/xz-todo 409 then 410 echo Size before compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 411 for i in `cat $T/xz-todo` 412 do 413 echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1 414 for j in $i/*/vmlinux 415 do 416 xz "$j" >> "$tdir/log-xz" 2>&1 & 417 ncompresses=$((ncompresses+1)) 418 if test $ncompresses -ge $compress_kasan_vmlinux 419 then 420 echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log 421 wait 422 ncompresses=0 423 batchno=$((batchno+1)) 424 fi 425 done 426 done 427 if test $ncompresses -gt 0 428 then 429 echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log 430 fi 431 wait 432 echo Size after compressing: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 433 echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log 434 else 435 echo No compression needed: `date` >> "$tdir/log-xz" 2>&1 436 fi 437fi 438if test -n "$tdir" 439then 440 cp $T/log "$tdir" 441fi 442exit $ret 443