1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Run a series of torture tests, intended for overnight or 5# longer timeframes, and also for large systems. 6# 7# Usage: torture.sh [ options ] 8# 9# Copyright (C) 2020 Facebook, Inc. 10# 11# Authors: Paul E. McKenney <paulmck@kernel.org> 12 13scriptname=$0 14args="$*" 15 16RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE 17PATH=${RCUTORTURE}/bin:$PATH; export PATH 18. functions.sh 19 20TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`" 21MAKE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS*2)) 22SCALE_ALLOTED_CPUS=$((TORTURE_ALLOTED_CPUS/2)) 23if test "$SCALE_ALLOTED_CPUS" -lt 1 24then 25 SCALE_ALLOTED_CPUS=1 26fi 27VERBOSE_BATCH_CPUS=$((TORTURE_ALLOTED_CPUS/16)) 28if test "$VERBOSE_BATCH_CPUS" -lt 2 29then 30 VERBOSE_BATCH_CPUS=0 31fi 32 33# Configurations/scenarios. 34configs_rcutorture= 35configs_locktorture= 36configs_scftorture= 37kcsan_kmake_args= 38 39# Default compression, duration, and apportionment. 40compress_concurrency="`identify_qemu_vcpus`" 41duration_base=10 42duration_rcutorture_frac=7 43duration_locktorture_frac=1 44duration_scftorture_frac=2 45 46# "yes" or "no" parameters 47do_allmodconfig=yes 48do_rcutorture=yes 49do_locktorture=yes 50do_scftorture=yes 51do_rcuscale=yes 52do_refscale=yes 53do_kvfree=yes 54do_normal=yes 55explicit_normal=no 56do_kasan=yes 57do_kcsan=no 58do_clocksourcewd=yes 59do_rt=yes 60do_rcutasksflavors=yes 61do_srcu_lockdep=yes 62do_rcu_rust=no 63 64# doyesno - Helper function for yes/no arguments 65function doyesno () { 66 if test "$1" = "$2" 67 then 68 echo yes 69 else 70 echo no 71 fi 72} 73 74usage () { 75 echo "Usage: $scriptname optional arguments:" 76 echo " --compress-concurrency concurrency" 77 echo " --configs-rcutorture \"config-file list w/ repeat factor (3*TINY01)\"" 78 echo " --configs-locktorture \"config-file list w/ repeat factor (10*LOCK01)\"" 79 echo " --configs-scftorture \"config-file list w/ repeat factor (2*CFLIST)\"" 80 echo " --do-all" 81 echo " --do-allmodconfig / --do-no-allmodconfig / --no-allmodconfig" 82 echo " --do-clocksourcewd / --do-no-clocksourcewd / --no-clocksourcewd" 83 echo " --do-kasan / --do-no-kasan / --no-kasan" 84 echo " --do-kcsan / --do-no-kcsan / --no-kcsan" 85 echo " --do-kvfree / --do-no-kvfree / --no-kvfree" 86 echo " --do-locktorture / --do-no-locktorture / --no-locktorture" 87 echo " --do-none" 88 echo " --do-rcuscale / --do-no-rcuscale / --no-rcuscale" 89 echo " --do-rcutasksflavors / --do-no-rcutasksflavors / --no-rcutasksflavors" 90 echo " --do-rcutorture / --do-no-rcutorture / --no-rcutorture" 91 echo " --do-refscale / --do-no-refscale / --no-refscale" 92 echo " --do-rt / --do-no-rt / --no-rt" 93 echo " --do-rcu-rust / --do-no-rcu-rust / --no-rcu-rust" 94 echo " --do-scftorture / --do-no-scftorture / --no-scftorture" 95 echo " --do-srcu-lockdep / --do-no-srcu-lockdep / --no-srcu-lockdep" 96 echo " --duration [ <minutes> | <hours>h | <days>d ]" 97 echo " --guest-cpu-limit N" 98 echo " --kcsan-kmake-arg kernel-make-arguments" 99 exit 1 100} 101 102while test $# -gt 0 103do 104 case "$1" in 105 --compress-concurrency) 106 checkarg --compress-concurrency "(concurrency level)" $# "$2" '^[0-9][0-9]*$' '^error' 107 compress_concurrency=$2 108 shift 109 ;; 110 --config-rcutorture|--configs-rcutorture) 111 checkarg --configs-rcutorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 112 configs_rcutorture="$configs_rcutorture $2" 113 shift 114 ;; 115 --config-locktorture|--configs-locktorture) 116 checkarg --configs-locktorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 117 configs_locktorture="$configs_locktorture $2" 118 shift 119 ;; 120 --config-scftorture|--configs-scftorture) 121 checkarg --configs-scftorture "(list of config files)" "$#" "$2" '^[^/]\+$' '^--' 122 configs_scftorture="$configs_scftorture $2" 123 shift 124 ;; 125 --do-all|--doall) 126 do_allmodconfig=yes 127 do_rcutasksflavor=yes 128 do_rcutorture=yes 129 do_locktorture=yes 130 do_scftorture=yes 131 do_rcuscale=yes 132 do_refscale=yes 133 do_rt=yes 134 do_kvfree=yes 135 do_normal=yes 136 explicit_normal=no 137 do_kasan=yes 138 do_kcsan=yes 139 do_clocksourcewd=yes 140 do_srcu_lockdep=yes 141 ;; 142 --do-allmodconfig|--do-no-allmodconfig|--no-allmodconfig) 143 do_allmodconfig=`doyesno "$1" --do-allmodconfig` 144 ;; 145 --do-clocksourcewd|--do-no-clocksourcewd|--no-clocksourcewd) 146 do_clocksourcewd=`doyesno "$1" --do-clocksourcewd` 147 ;; 148 --do-kasan|--do-no-kasan|--no-kasan) 149 do_kasan=`doyesno "$1" --do-kasan` 150 ;; 151 --do-kcsan|--do-no-kcsan|--no-kcsan) 152 do_kcsan=`doyesno "$1" --do-kcsan` 153 ;; 154 --do-kvfree|--do-no-kvfree|--no-kvfree) 155 do_kvfree=`doyesno "$1" --do-kvfree` 156 ;; 157 --do-locktorture|--do-no-locktorture|--no-locktorture) 158 do_locktorture=`doyesno "$1" --do-locktorture` 159 ;; 160 --do-none|--donone) 161 do_allmodconfig=no 162 do_rcutasksflavors=no 163 do_rcutorture=no 164 do_locktorture=no 165 do_scftorture=no 166 do_rcuscale=no 167 do_refscale=no 168 do_rt=no 169 do_kvfree=no 170 do_normal=no 171 explicit_normal=no 172 do_kasan=no 173 do_kcsan=no 174 do_clocksourcewd=no 175 do_srcu_lockdep=no 176 ;; 177 --do-normal|--do-no-normal|--no-normal) 178 do_normal=`doyesno "$1" --do-normal` 179 explicit_normal=yes 180 ;; 181 --do-rcuscale|--do-no-rcuscale|--no-rcuscale) 182 do_rcuscale=`doyesno "$1" --do-rcuscale` 183 ;; 184 --do-rcutasksflavors|--do-no-rcutasksflavors|--no-rcutasksflavors) 185 do_rcutasksflavors=`doyesno "$1" --do-rcutasksflavors` 186 ;; 187 --do-rcutorture|--do-no-rcutorture|--no-rcutorture) 188 do_rcutorture=`doyesno "$1" --do-rcutorture` 189 ;; 190 --do-refscale|--do-no-refscale|--no-refscale) 191 do_refscale=`doyesno "$1" --do-refscale` 192 ;; 193 --do-rt|--do-no-rt|--no-rt) 194 do_rt=`doyesno "$1" --do-rt` 195 ;; 196 --do-rcu-rust|--do-no-rcu-rust|--no-rcu-rust) 197 do_rcu_rust=`doyesno "$1" --do-rcu-rust` 198 ;; 199 --do-scftorture|--do-no-scftorture|--no-scftorture) 200 do_scftorture=`doyesno "$1" --do-scftorture` 201 ;; 202 --do-srcu-lockdep|--do-no-srcu-lockdep|--no-srcu-lockdep) 203 do_srcu_lockdep=`doyesno "$1" --do-srcu-lockdep` 204 ;; 205 --duration) 206 checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error' 207 mult=1 208 if echo "$2" | grep -q 'm$' 209 then 210 mult=1 211 elif echo "$2" | grep -q 'h$' 212 then 213 mult=60 214 elif echo "$2" | grep -q 'd$' 215 then 216 mult=1440 217 fi 218 ts=`echo $2 | sed -e 's/[smhd]$//'` 219 duration_base=$(($ts*mult)) 220 shift 221 ;; 222 --guest-cpu-limit|--guest-cpu-lim) 223 checkarg --guest-cpu-limit "(number)" "$#" "$2" '^[0-9]*$' '^--' 224 if (("$2" <= "$TORTURE_ALLOTED_CPUS" / 2)) 225 then 226 SCALE_ALLOTED_CPUS="$2" 227 VERBOSE_BATCH_CPUS="$((SCALE_ALLOTED_CPUS/8))" 228 if (("$VERBOSE_BATCH_CPUS" < 2)) 229 then 230 VERBOSE_BATCH_CPUS=0 231 fi 232 else 233 echo "Ignoring value of $2 for --guest-cpu-limit which is greater than (("$TORTURE_ALLOTED_CPUS" / 2))." 234 fi 235 shift 236 ;; 237 --kcsan-kmake-arg|--kcsan-kmake-args) 238 checkarg --kcsan-kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' 239 kcsan_kmake_args="`echo "$kcsan_kmake_args $2" | sed -e 's/^ *//' -e 's/ *$//'`" 240 shift 241 ;; 242 *) 243 echo Unknown argument $1 244 usage 245 ;; 246 esac 247 shift 248done 249 250ds="`date +%Y.%m.%d-%H.%M.%S`-torture" 251startdate="`date`" 252starttime="`get_starttime`" 253 254T="`mktemp -d ${TMPDIR-/tmp}/torture.sh.XXXXXX`" 255trap 'rm -rf $T' 0 2 256 257echo " --- " $scriptname $args | tee -a $T/log 258echo " --- Results directory: " $ds | tee -a $T/log 259 260if test "$do_normal" = "no" && test "$do_kasan" = "no" && test "$do_kcsan" = "no" 261then 262 # Match old scripts so that "--do-none --do-rcutorture" does 263 # normal rcutorture testing, but no KASAN or KCSAN testing. 264 if test $explicit_normal = yes 265 then 266 echo " --- Everything disabled, so explicit --do-normal overridden" | tee -a $T/log 267 fi 268 do_normal=yes 269fi 270 271# Calculate rcutorture defaults and apportion time 272if test -z "$configs_rcutorture" 273then 274 configs_rcutorture=CFLIST 275fi 276duration_rcutorture=$((duration_base*duration_rcutorture_frac/10)) 277if test "$duration_rcutorture" -eq 0 278then 279 echo " --- Zero time for rcutorture, disabling" | tee -a $T/log 280 do_rcutorture=no 281fi 282 283# Calculate locktorture defaults and apportion time 284if test -z "$configs_locktorture" 285then 286 configs_locktorture=CFLIST 287fi 288duration_locktorture=$((duration_base*duration_locktorture_frac/10)) 289if test "$duration_locktorture" -eq 0 290then 291 echo " --- Zero time for locktorture, disabling" | tee -a $T/log 292 do_locktorture=no 293fi 294 295# Calculate scftorture defaults and apportion time 296if test -z "$configs_scftorture" 297then 298 configs_scftorture=CFLIST 299fi 300duration_scftorture=$((duration_base*duration_scftorture_frac/10)) 301if test "$duration_scftorture" -eq 0 302then 303 echo " --- Zero time for scftorture, disabling" | tee -a $T/log 304 do_scftorture=no 305fi 306 307touch $T/failures 308touch $T/successes 309 310# torture_one - Does a single kvm.sh run. 311# 312# Usage: 313# torture_bootargs="[ kernel boot arguments ]" 314# torture_one flavor [ kvm.sh arguments ] 315# 316# Note that "flavor" is an arbitrary string. Supply --torture if needed. 317# Note that quoting is problematic. So on the command line, pass multiple 318# values with multiple kvm.sh argument instances. 319function torture_one { 320 local cur_bootargs= 321 local boottag= 322 323 echo " --- $curflavor:" Start `date` | tee -a $T/log 324 if test -n "$torture_bootargs" 325 then 326 boottag="--bootargs" 327 cur_bootargs="$torture_bootargs" 328 fi 329 "$@" $boottag "$cur_bootargs" --datestamp "$ds/results-$curflavor" > $T/$curflavor.out 2>&1 330 retcode=$? 331 resdir="`grep '^Results directory: ' $T/$curflavor.out | tail -1 | sed -e 's/^Results directory: //'`" 332 if test -z "$resdir" 333 then 334 cat $T/$curflavor.out | tee -a $T/log 335 echo retcode=$retcode | tee -a $T/log 336 else 337 echo $resdir > $T/last-resdir 338 fi 339 if test "$retcode" == 0 340 then 341 echo "$curflavor($retcode)" $resdir >> $T/successes 342 else 343 echo "$curflavor($retcode)" $resdir >> $T/failures 344 fi 345} 346 347# torture_set - Does a set of tortures with and without KASAN and KCSAN. 348# 349# Usage: 350# torture_bootargs="[ kernel boot arguments ]" 351# torture_set flavor [ kvm.sh arguments ] 352# 353# Note that "flavor" is an arbitrary string that does not affect kvm.sh 354# in any way. So also supply --torture if you need something other than 355# the default. 356function torture_set { 357 local cur_kcsan_kmake_args= 358 local kcsan_kmake_tag= 359 local flavor=$1 360 shift 361 if test "$do_normal" = "yes" 362 then 363 curflavor=$flavor 364 torture_one "$@" 365 mv $T/last-resdir $T/last-resdir-nodebug || : 366 fi 367 if test "$do_kasan" = "yes" 368 then 369 curflavor=${flavor}-kasan 370 torture_one "$@" --kasan 371 mv $T/last-resdir $T/last-resdir-kasan || : 372 fi 373 if test "$do_kcsan" = "yes" 374 then 375 curflavor=${flavor}-kcsan 376 if test -n "$kcsan_kmake_args" 377 then 378 kcsan_kmake_tag="--kmake-args" 379 cur_kcsan_kmake_args="$kcsan_kmake_args" 380 fi 381 torture_one "$@" --kconfig "CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y" $kcsan_kmake_tag $cur_kcsan_kmake_args --kcsan 382 mv $T/last-resdir $T/last-resdir-kcsan || : 383 fi 384} 385 386# make allmodconfig 387if test "$do_allmodconfig" = "yes" 388then 389 echo " --- allmodconfig:" Start `date` | tee -a $T/log 390 amcdir="tools/testing/selftests/rcutorture/res/$ds/allmodconfig" 391 mkdir -p "$amcdir" 392 echo " --- make clean" | tee $amcdir/log > "$amcdir/Make.out" 2>&1 393 make -j$MAKE_ALLOTED_CPUS clean >> "$amcdir/Make.out" 2>&1 394 retcode=$? 395 buildphase='"make clean"' 396 if test "$retcode" -eq 0 397 then 398 echo " --- make allmodconfig" | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1 399 cp .config $amcdir 400 make -j$MAKE_ALLOTED_CPUS allmodconfig >> "$amcdir/Make.out" 2>&1 401 retcode=$? 402 buildphase='"make allmodconfig"' 403 fi 404 if test "$retcode" -eq 0 405 then 406 echo " --- make " | tee -a $amcdir/log >> "$amcdir/Make.out" 2>&1 407 make -j$MAKE_ALLOTED_CPUS >> "$amcdir/Make.out" 2>&1 408 retcode="$?" 409 echo $retcode > "$amcdir/Make.exitcode" 410 buildphase='"make"' 411 fi 412 if test "$retcode" -eq 0 413 then 414 echo "allmodconfig($retcode)" $amcdir >> $T/successes 415 echo Success >> $amcdir/log 416 else 417 echo "allmodconfig($retcode)" $amcdir >> $T/failures 418 echo " --- allmodconfig Test summary:" >> $amcdir/log 419 echo " --- Summary: Exit code $retcode from $buildphase, see Make.out" >> $amcdir/log 420 fi 421fi 422 423# Test building RCU Tasks flavors in isolation, both SMP and !SMP 424if test "$do_rcutasksflavors" = "yes" 425then 426 echo " --- rcutasksflavors:" Start `date` | tee -a $T/log 427 rtfdir="tools/testing/selftests/rcutorture/res/$ds/results-rcutasksflavors" 428 mkdir -p "$rtfdir" 429 cat > $T/rcutasksflavors << __EOF__ 430#CHECK#CONFIG_TASKS_RCU=n 431#CHECK#CONFIG_TASKS_RUDE_RCU=n 432#CHECK#CONFIG_TASKS_TRACE_RCU=n 433__EOF__ 434 for flavor in CONFIG_TASKS_RCU CONFIG_TASKS_RUDE_RCU CONFIG_TASKS_TRACE_RCU 435 do 436 forceflavor="`echo $flavor | sed -e 's/^CONFIG/CONFIG_FORCE/'`" 437 deselectedflavors="`grep -v $flavor $T/rcutasksflavors | tr '\012' ' ' | tr -s ' ' | sed -e 's/ *$//'`" 438 echo " --- Running RCU Tasks Trace flavor $flavor `date`" >> $rtfdir/log 439 tools/testing/selftests/rcutorture/bin/kvm.sh --datestamp "$ds/results-rcutasksflavors/$flavor" --buildonly --configs "TINY01 TREE04" --kconfig "CONFIG_RCU_EXPERT=y CONFIG_RCU_SCALE_TEST=y CONFIG_KPROBES=n CONFIG_RCU_TRACE=n CONFIG_TRACING=n CONFIG_BLK_DEV_IO_TRACE=n CONFIG_UPROBE_EVENTS=n $forceflavor=y $deselectedflavors" --trust-make > $T/$flavor.out 2>&1 440 retcode=$? 441 if test "$retcode" -ne 0 442 then 443 break 444 fi 445 done 446 if test "$retcode" -eq 0 447 then 448 echo "rcutasksflavors($retcode)" $rtfdir >> $T/successes 449 echo Success >> $rtfdir/log 450 else 451 echo "rcutasksflavors($retcode)" $rtfdir >> $T/failures 452 echo " --- rcutasksflavors Test summary:" >> $rtfdir/log 453 echo " --- Summary: Exit code $retcode from $flavor, see Make.out" >> $rtfdir/log 454 fi 455fi 456 457# --torture rcu 458if test "$do_rcutorture" = "yes" 459then 460 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000" 461 torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make 462fi 463 464if test "$do_locktorture" = "yes" 465then 466 torture_bootargs="torture.disable_onoff_at_boot" 467 torture_set "locktorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture lock --allcpus --duration "$duration_locktorture" --configs "$configs_locktorture" --trust-make 468fi 469 470if test "$do_scftorture" = "yes" 471then 472 # Scale memory based on the number of CPUs. 473 scfmem=$((3+SCALE_ALLOTED_CPUS/16)) 474 torture_bootargs="scftorture.nthreads=$SCALE_ALLOTED_CPUS torture.disable_onoff_at_boot csdlock_debug=1" 475 torture_set "scftorture" tools/testing/selftests/rcutorture/bin/kvm.sh --torture scf --allcpus --duration "$duration_scftorture" --configs "$configs_scftorture" --kconfig "CONFIG_NR_CPUS=$SCALE_ALLOTED_CPUS" --memory ${scfmem}G --trust-make 476fi 477 478if test "$do_rt" = "yes" 479then 480 # In both runs, disable testing of RCU priority boosting because 481 # -rt doesn't like its interaction with testing of callback 482 # flooding. 483 484 # With all post-boot grace periods forced to normal (default for PREEMPT_RT). 485 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcutorture.test_boost=0 rcutorture.preempt_duration=0" 486 torture_set "rcurttorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --kconfig "CONFIG_PREEMPT_RT=y CONFIG_EXPERT=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_RCU_NOCB_CPU=y" --trust-make 487 488 # With all post-boot grace periods forced to expedited. 489 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 rcutorture.test_boost=0 rcupdate.rcu_normal_after_boot=0 rcupdate.rcu_expedited=1 rcutorture.preempt_duration=0" 490 torture_set "rcurttorture-exp" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "TREE03" --kconfig "CONFIG_PREEMPT_RT=y CONFIG_EXPERT=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_FULL=y CONFIG_RCU_NOCB_CPU=y" --trust-make 491fi 492 493if test "$do_rcu_rust" = "yes" 494then 495 echo " --- do-rcu-rust:" Start `date` | tee -a $T/log 496 rrdir="tools/testing/selftests/rcutorture/res/$ds/results-rcu-rust" 497 mkdir -p "$rrdir" 498 echo " --- make LLVM=1 rustavailable " | tee -a $rrdir/log > $rrdir/rustavailable.out 499 make LLVM=1 rustavailable > $T/rustavailable.out 2>&1 500 retcode=$? 501 echo $retcode > $rrdir/rustavailable.exitcode 502 cat $T/rustavailable.out | tee -a $rrdir/log >> $rrdir/rustavailable.out 2>&1 503 buildphase=rustavailable 504 if test "$retcode" -eq 0 505 then 506 echo " --- Running 'make mrproper' in order to run kunit." | tee -a $rrdir/log > $rrdir/mrproper.out 507 make mrproper > $rrdir/mrproper.out 2>&1 508 retcode=$? 509 echo $retcode > $rrdir/mrproper.exitcode 510 buildphase=mrproper 511 fi 512 if test "$retcode" -eq 0 513 then 514 echo " --- Running rust_doctests_kernel." | tee -a $rrdir/log > $rrdir/rust_doctests_kernel.out 515 ./tools/testing/kunit/kunit.py run --make_options LLVM=1 --make_options CLIPPY=1 --arch arm64 --kconfig_add CONFIG_SMP=y --kconfig_add CONFIG_WERROR=y --kconfig_add CONFIG_RUST=y rust_doctests_kernel >> $rrdir/rust_doctests_kernel.out 2>&1 516 # @@@ Remove "--arch arm64" in order to test on native architecture? 517 # @@@ Analyze $rrdir/rust_doctests_kernel.out contents? 518 retcode=$? 519 echo $retcode > $rrdir/rust_doctests_kernel.exitcode 520 buildphase=rust_doctests_kernel 521 fi 522 if test "$retcode" -eq 0 523 then 524 echo "rcu-rust($retcode)" $rrdir >> $T/successes 525 echo Success >> $rrdir/log 526 else 527 echo "rcu-rust($retcode)" $rrdir >> $T/failures 528 echo " --- rcu-rust Test summary:" >> $rrdir/log 529 echo " --- Summary: Exit code $retcode from $buildphase, see $rrdir/$buildphase.out" >> $rrdir/log 530 fi 531fi 532 533if test "$do_srcu_lockdep" = "yes" 534then 535 echo " --- do-srcu-lockdep:" Start `date` | tee -a $T/log 536 tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh --datestamp "$ds/results-srcu-lockdep" > $T/srcu_lockdep.sh.out 2>&1 537 retcode=$? 538 cp $T/srcu_lockdep.sh.out "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" 539 if test "$retcode" -eq 0 540 then 541 echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/successes 542 echo Success >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" 543 else 544 echo "srcu_lockdep($retcode)" "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep" >> $T/failures 545 echo " --- srcu_lockdep Test Summary:" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" 546 echo " --- Summary: Exit code $retcode from srcu_lockdep.sh, see ds/results-srcu-lockdep" >> "tools/testing/selftests/rcutorture/res/$ds/results-srcu-lockdep/log" 547 fi 548fi 549 550if test "$do_refscale" = yes 551then 552 primlist="`grep '\.name[ ]*=' kernel/rcu/refscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" 553else 554 primlist= 555fi 556firsttime=1 557do_kasan_save="$do_kasan" 558do_kcsan_save="$do_kcsan" 559for prim in $primlist 560do 561 if test -n "$firsttime" 562 then 563 torture_bootargs="refscale.scale_type="$prim" refscale.nreaders=$SCALE_ALLOTED_CPUS refscale.loops=10000 refscale.holdoff=20 torture.disable_onoff_at_boot" 564 torture_set "refscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture refscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$SCALE_ALLOTED_CPUS" --bootargs "refscale.verbose_batched=$VERBOSE_BATCH_CPUS torture.verbose_sleep_frequency=8 torture.verbose_sleep_duration=$VERBOSE_BATCH_CPUS" --trust-make 565 mv $T/last-resdir-nodebug $T/first-resdir-nodebug || : 566 if test -f "$T/last-resdir-kasan" 567 then 568 mv $T/last-resdir-kasan $T/first-resdir-kasan || : 569 fi 570 if test -f "$T/last-resdir-kcsan" 571 then 572 mv $T/last-resdir-kcsan $T/first-resdir-kcsan || : 573 fi 574 firsttime= 575 do_kasan= 576 do_kcsan= 577 else 578 torture_bootargs= 579 for i in $T/first-resdir-* 580 do 581 case "$i" in 582 *-nodebug) 583 torture_suffix= 584 ;; 585 *-kasan) 586 torture_suffix="-kasan" 587 ;; 588 *-kcsan) 589 torture_suffix="-kcsan" 590 ;; 591 esac 592 torture_set "refscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "refscale.scale_type=$prim" 593 done 594 fi 595done 596do_kasan="$do_kasan_save" 597do_kcsan="$do_kcsan_save" 598 599if test "$do_rcuscale" = yes 600then 601 primlist="`grep '\.name[ ]*=' kernel/rcu/rcuscale.c | sed -e 's/^[^"]*"//' -e 's/".*$//'`" 602else 603 primlist= 604fi 605firsttime=1 606do_kasan_save="$do_kasan" 607do_kcsan_save="$do_kcsan" 608for prim in $primlist 609do 610 if test -n "$firsttime" 611 then 612 torture_bootargs="rcuscale.scale_type="$prim" rcuscale.nwriters=$SCALE_ALLOTED_CPUS rcuscale.holdoff=20 torture.disable_onoff_at_boot" 613 torture_set "rcuscale-$prim" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration 5 --kconfig "CONFIG_TASKS_TRACE_RCU=y CONFIG_NR_CPUS=$SCALE_ALLOTED_CPUS" --trust-make 614 mv $T/last-resdir-nodebug $T/first-resdir-nodebug || : 615 if test -f "$T/last-resdir-kasan" 616 then 617 mv $T/last-resdir-kasan $T/first-resdir-kasan || : 618 fi 619 if test -f "$T/last-resdir-kcsan" 620 then 621 mv $T/last-resdir-kcsan $T/first-resdir-kcsan || : 622 fi 623 firsttime= 624 do_kasan= 625 do_kcsan= 626 else 627 torture_bootargs= 628 for i in $T/first-resdir-* 629 do 630 case "$i" in 631 *-nodebug) 632 torture_suffix= 633 ;; 634 *-kasan) 635 torture_suffix="-kasan" 636 ;; 637 *-kcsan) 638 torture_suffix="-kcsan" 639 ;; 640 esac 641 torture_set "rcuscale-$prim$torture_suffix" tools/testing/selftests/rcutorture/bin/kvm-again.sh "`cat "$i"`" --duration 5 --bootargs "rcuscale.scale_type=$prim" 642 done 643 fi 644done 645do_kasan="$do_kasan_save" 646do_kcsan="$do_kcsan_save" 647 648if test "$do_kvfree" = "yes" 649then 650 torture_bootargs="rcuscale.kfree_rcu_test=1 rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 torture.disable_onoff_at_boot" 651 torture_set "rcuscale-kvfree" tools/testing/selftests/rcutorture/bin/kvm.sh --torture rcuscale --allcpus --duration $duration_rcutorture --kconfig "CONFIG_NR_CPUS=$SCALE_ALLOTED_CPUS" --memory 2G --trust-make 652fi 653 654if test "$do_clocksourcewd" = "yes" 655then 656 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog" 657 torture_set "clocksourcewd-1" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make 658 659 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog" 660 torture_set "clocksourcewd-2" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --kconfig "CONFIG_TEST_CLOCKSOURCE_WATCHDOG=y" --trust-make 661 662 # In case our work is already done... 663 if test "$do_rcutorture" != "yes" 664 then 665 torture_bootargs="rcupdate.rcu_cpu_stall_suppress_at_boot=1 torture.disable_onoff_at_boot rcupdate.rcu_task_stall_timeout=30000 tsc=watchdog" 666 torture_set "clocksourcewd-3" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 45s --configs TREE03 --trust-make 667 fi 668fi 669 670echo " --- " $scriptname $args 671echo " --- " Done `date` | tee -a $T/log 672ret=0 673nsuccesses=0 674echo SUCCESSES: | tee -a $T/log 675if test -s "$T/successes" 676then 677 cat "$T/successes" | tee -a $T/log 678 nsuccesses="`wc -l "$T/successes" | awk '{ print $1 }'`" 679fi 680nfailures=0 681echo FAILURES: | tee -a $T/log 682if test -s "$T/failures" 683then 684 awk < "$T/failures" -v sq="'" '{ print "echo " sq $0 sq; print "sed -e " sq "1,/^ --- .* Test summary:$/d" sq " " $2 "/log | grep Summary: | sed -e " sq "s/^[^S]*/ /" sq; }' | sh | tee -a $T/log | tee "$T/failuresum" 685 nfailures="`wc -l "$T/failures" | awk '{ print $1 }'`" 686 grep "^ Summary: " "$T/failuresum" | 687 grep -v '^ Summary: Bugs: [0-9]* (all bugs kcsan)$' > "$T/nonkcsan" 688 if test -s "$T/nonkcsan" 689 then 690 nonkcsanbug="yes" 691 fi 692 ret=2 693fi 694if test "$do_kcsan" = "yes" 695then 696 TORTURE_KCONFIG_KCSAN_ARG=1 tools/testing/selftests/rcutorture/bin/kcsan-collapse.sh tools/testing/selftests/rcutorture/res/$ds > tools/testing/selftests/rcutorture/res/$ds/kcsan.sum 697fi 698echo Started at $startdate, ended at `date`, duration `get_starttime_duration $starttime`. | tee -a $T/log 699echo Summary: Successes: $nsuccesses Failures: $nfailures. | tee -a $T/log 700tdir="`cat $T/successes $T/failures | head -1 | awk '{ print $NF }' | sed -e 's,/[^/]\+/*$,,'`" 701find "$tdir" -name 'ConfigFragment.diags' -print > $T/configerrors 702find "$tdir" -name 'Make.out.diags' -print > $T/builderrors 703if test -s "$T/configerrors" 704then 705 echo " Scenarios with .config errors: `wc -l "$T/configerrors" | awk '{ print $1 }'`" 706 nonkcsanbug="yes" 707fi 708if test -s "$T/builderrors" 709then 710 echo " Scenarios with build errors: `wc -l "$T/builderrors" | awk '{ print $1 }'`" 711 nonkcsanbug="yes" 712fi 713if test -z "$nonkcsanbug" && test -s "$T/failuresum" 714then 715 echo " All bugs were KCSAN failures." 716fi 717if test -n "$tdir" && test $compress_concurrency -gt 0 718then 719 # KASAN vmlinux files can approach 1GB in size, so compress them. 720 echo Looking for K[AC]SAN files to compress: `date` > "$tdir/log-xz" 2>&1 721 find "$tdir" -type d -name '*-k[ac]san' -print > $T/xz-todo-all 722 find "$tdir" -type f -name 're-run' -print | sed -e 's,/re-run,,' | 723 grep -e '-k[ac]san$' > $T/xz-todo-copy 724 sort $T/xz-todo-all $T/xz-todo-copy | uniq -u > $T/xz-todo 725 ncompresses=0 726 batchno=1 727 if test -s $T/xz-todo 728 then 729 for i in `cat $T/xz-todo` 730 do 731 find $i -name 'vmlinux*' -print 732 done | wc -l | awk '{ print $1 }' > $T/xz-todo-count 733 n2compress="`cat $T/xz-todo-count`" 734 echo Size before compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 735 for i in `cat $T/xz-todo` 736 do 737 echo Compressing vmlinux files in ${i}: `date` >> "$tdir/log-xz" 2>&1 738 for j in $i/*/vmlinux 739 do 740 xz "$j" >> "$tdir/log-xz" 2>&1 & 741 ncompresses=$((ncompresses+1)) 742 if test $ncompresses -ge $compress_concurrency 743 then 744 echo Waiting for batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log 745 wait 746 ncompresses=0 747 batchno=$((batchno+1)) 748 fi 749 done 750 done 751 if test $ncompresses -gt 0 752 then 753 echo Waiting for final batch $batchno of $ncompresses compressions `date` | tee -a "$tdir/log-xz" | tee -a $T/log 754 fi 755 wait 756 if test -s $T/xz-todo-copy 757 then 758 # The trick here is that we need corresponding 759 # vmlinux files from corresponding scenarios. 760 echo Linking vmlinux.xz files to re-use scenarios `date` | tee -a "$tdir/log-xz" | tee -a $T/log 761 dirstash="`pwd`" 762 for i in `cat $T/xz-todo-copy` 763 do 764 cd $i 765 find . -name vmlinux -print > $T/xz-todo-copy-vmlinux 766 for v in `cat $T/xz-todo-copy-vmlinux` 767 do 768 rm -f "$v" 769 cp -l `cat $i/re-run`/"$i/$v".xz "`dirname "$v"`" 770 done 771 cd "$dirstash" 772 done 773 fi 774 echo Size after compressing $n2compress files: `du -sh $tdir | awk '{ print $1 }'` `date` 2>&1 | tee -a "$tdir/log-xz" | tee -a $T/log 775 echo Total duration `get_starttime_duration $starttime`. | tee -a $T/log 776 else 777 echo No compression needed: `date` >> "$tdir/log-xz" 2>&1 778 fi 779fi 780if test -n "$tdir" 781then 782 cp $T/log "$tdir" 783fi 784exit $ret 785