1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Run a series of tests under KVM. By default, this series is specified 5# by the relevant CFLIST file, but can be overridden by the --configs 6# command-line argument. 7# 8# Usage: kvm.sh [ options ] 9# 10# Copyright (C) IBM Corporation, 2011 11# 12# Authors: Paul E. McKenney <paulmck@linux.ibm.com> 13 14scriptname=$0 15args="$*" 16 17T=${TMPDIR-/tmp}/kvm.sh.$$ 18trap 'rm -rf $T' 0 19mkdir $T 20 21cd `dirname $scriptname`/../../../../../ 22 23dur=$((30*60)) 24dryrun="" 25KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM 26PATH=${KVM}/bin:$PATH; export PATH 27. functions.sh 28 29TORTURE_ALLOTED_CPUS="`identify_qemu_vcpus`" 30TORTURE_DEFCONFIG=defconfig 31TORTURE_BOOT_IMAGE="" 32TORTURE_INITRD="$KVM/initrd"; export TORTURE_INITRD 33TORTURE_KCONFIG_ARG="" 34TORTURE_KCONFIG_KASAN_ARG="" 35TORTURE_KCONFIG_KCSAN_ARG="" 36TORTURE_KMAKE_ARG="" 37TORTURE_QEMU_MEM=512 38TORTURE_SHUTDOWN_GRACE=180 39TORTURE_SUITE=rcu 40TORTURE_TRUST_MAKE="" 41resdir="" 42configs="" 43cpus=0 44ds=`date +%Y.%m.%d-%H.%M.%S` 45jitter="-1" 46 47usage () { 48 echo "Usage: $scriptname optional arguments:" 49 echo " --bootargs kernel-boot-arguments" 50 echo " --bootimage relative-path-to-kernel-boot-image" 51 echo " --buildonly" 52 echo " --configs \"config-file list w/ repeat factor (3*TINY01)\"" 53 echo " --cpus N" 54 echo " --datestamp string" 55 echo " --defconfig string" 56 echo " --dryrun sched|script" 57 echo " --duration minutes" 58 echo " --interactive" 59 echo " --jitter N [ maxsleep (us) [ maxspin (us) ] ]" 60 echo " --kconfig Kconfig-options" 61 echo " --kmake-arg kernel-make-arguments" 62 echo " --mac nn:nn:nn:nn:nn:nn" 63 echo " --memory megabytes | nnnG" 64 echo " --no-initrd" 65 echo " --qemu-args qemu-arguments" 66 echo " --qemu-cmd qemu-system-..." 67 echo " --results absolute-pathname" 68 echo " --torture rcu" 69 echo " --trust-make" 70 exit 1 71} 72 73while test $# -gt 0 74do 75 case "$1" in 76 --allcpus) 77 cpus=$TORTURE_ALLOTED_CPUS 78 max_cpus=$TORTURE_ALLOTED_CPUS 79 ;; 80 --bootargs|--bootarg) 81 checkarg --bootargs "(list of kernel boot arguments)" "$#" "$2" '.*' '^--' 82 TORTURE_BOOTARGS="$2" 83 shift 84 ;; 85 --bootimage) 86 checkarg --bootimage "(relative path to kernel boot image)" "$#" "$2" '[a-zA-Z0-9][a-zA-Z0-9_]*' '^--' 87 TORTURE_BOOT_IMAGE="$2" 88 shift 89 ;; 90 --buildonly) 91 TORTURE_BUILDONLY=1 92 ;; 93 --configs|--config) 94 checkarg --configs "(list of config files)" "$#" "$2" '^[^/]*$' '^--' 95 configs="$2" 96 shift 97 ;; 98 --cpus) 99 checkarg --cpus "(number)" "$#" "$2" '^[0-9]*$' '^--' 100 cpus=$2 101 TORTURE_ALLOTED_CPUS="$2" 102 max_cpus="`identify_qemu_vcpus`" 103 if test "$TORTURE_ALLOTED_CPUS" -gt "$max_cpus" 104 then 105 TORTURE_ALLOTED_CPUS=$max_cpus 106 fi 107 shift 108 ;; 109 --datestamp) 110 checkarg --datestamp "(relative pathname)" "$#" "$2" '^[^/]*$' '^--' 111 ds=$2 112 shift 113 ;; 114 --defconfig) 115 checkarg --defconfig "defconfigtype" "$#" "$2" '^[^/][^/]*$' '^--' 116 TORTURE_DEFCONFIG=$2 117 shift 118 ;; 119 --dryrun) 120 checkarg --dryrun "sched|script" $# "$2" 'sched\|script' '^--' 121 dryrun=$2 122 shift 123 ;; 124 --duration) 125 checkarg --duration "(minutes)" $# "$2" '^[0-9]*$' '^error' 126 dur=$(($2*60)) 127 shift 128 ;; 129 --interactive) 130 TORTURE_QEMU_INTERACTIVE=1; export TORTURE_QEMU_INTERACTIVE 131 ;; 132 --jitter) 133 checkarg --jitter "(# threads [ sleep [ spin ] ])" $# "$2" '^-\{,1\}[0-9]\+\( \+[0-9]\+\)\{,2\} *$' '^error$' 134 jitter="$2" 135 shift 136 ;; 137 --kconfig) 138 checkarg --kconfig "(Kconfig options)" $# "$2" '^CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\( CONFIG_[A-Z0-9_]\+=\([ynm]\|[0-9]\+\)\)*$' '^error$' 139 TORTURE_KCONFIG_ARG="$2" 140 shift 141 ;; 142 --kasan) 143 TORTURE_KCONFIG_KASAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KASAN=y"; export TORTURE_KCONFIG_KASAN_ARG 144 ;; 145 --kcsan) 146 TORTURE_KCONFIG_KCSAN_ARG="CONFIG_DEBUG_INFO=y CONFIG_KCSAN=y CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY=n CONFIG_KCSAN_REPORT_ONCE_IN_MS=100000 CONFIG_KCSAN_VERBOSE=y CONFIG_KCSAN_INTERRUPT_WATCHER=y"; export TORTURE_KCONFIG_KCSAN_ARG 147 ;; 148 --kmake-arg) 149 checkarg --kmake-arg "(kernel make arguments)" $# "$2" '.*' '^error$' 150 TORTURE_KMAKE_ARG="$2" 151 shift 152 ;; 153 --mac) 154 checkarg --mac "(MAC address)" $# "$2" '^\([0-9a-fA-F]\{2\}:\)\{5\}[0-9a-fA-F]\{2\}$' error 155 TORTURE_QEMU_MAC=$2 156 shift 157 ;; 158 --memory) 159 checkarg --memory "(memory size)" $# "$2" '^[0-9]\+[MG]\?$' error 160 TORTURE_QEMU_MEM=$2 161 shift 162 ;; 163 --no-initrd) 164 TORTURE_INITRD=""; export TORTURE_INITRD 165 ;; 166 --qemu-args|--qemu-arg) 167 checkarg --qemu-args "(qemu arguments)" $# "$2" '^-' '^error' 168 TORTURE_QEMU_ARG="$2" 169 shift 170 ;; 171 --qemu-cmd) 172 checkarg --qemu-cmd "(qemu-system-...)" $# "$2" 'qemu-system-' '^--' 173 TORTURE_QEMU_CMD="$2" 174 shift 175 ;; 176 --results) 177 checkarg --results "(absolute pathname)" "$#" "$2" '^/' '^error' 178 resdir=$2 179 shift 180 ;; 181 --shutdown-grace) 182 checkarg --shutdown-grace "(seconds)" "$#" "$2" '^[0-9]*$' '^error' 183 TORTURE_SHUTDOWN_GRACE=$2 184 shift 185 ;; 186 --torture) 187 checkarg --torture "(suite name)" "$#" "$2" '^\(lock\|rcu\|rcuperf\|refscale\)$' '^--' 188 TORTURE_SUITE=$2 189 shift 190 if test "$TORTURE_SUITE" = rcuperf || test "$TORTURE_SUITE" = refscale 191 then 192 # If you really want jitter for refscale or 193 # rcuperf, specify it after specifying the rcuperf 194 # or the refscale. (But why jitter in these cases?) 195 jitter=0 196 fi 197 ;; 198 --trust-make) 199 TORTURE_TRUST_MAKE="y" 200 ;; 201 *) 202 echo Unknown argument $1 203 usage 204 ;; 205 esac 206 shift 207done 208 209if test -z "$TORTURE_INITRD" || tools/testing/selftests/rcutorture/bin/mkinitrd.sh 210then 211 : 212else 213 echo No initrd and unable to create one, aborting test >&2 214 exit 1 215fi 216 217CONFIGFRAG=${KVM}/configs/${TORTURE_SUITE}; export CONFIGFRAG 218 219defaultconfigs="`tr '\012' ' ' < $CONFIGFRAG/CFLIST`" 220if test -z "$configs" 221then 222 configs=$defaultconfigs 223fi 224 225if test -z "$resdir" 226then 227 resdir=$KVM/res 228fi 229 230# Create a file of test-name/#cpus pairs, sorted by decreasing #cpus. 231configs_derep= 232for CF in $configs 233do 234 case $CF in 235 [0-9]\**|[0-9][0-9]\**|[0-9][0-9][0-9]\**) 236 config_reps=`echo $CF | sed -e 's/\*.*$//'` 237 CF1=`echo $CF | sed -e 's/^[^*]*\*//'` 238 ;; 239 *) 240 config_reps=1 241 CF1=$CF 242 ;; 243 esac 244 for ((cur_rep=0;cur_rep<$config_reps;cur_rep++)) 245 do 246 configs_derep="$configs_derep $CF1" 247 done 248done 249touch $T/cfgcpu 250configs_derep="`echo $configs_derep | sed -e "s/\<CFLIST\>/$defaultconfigs/g"`" 251for CF1 in $configs_derep 252do 253 if test -f "$CONFIGFRAG/$CF1" 254 then 255 cpu_count=`configNR_CPUS.sh $CONFIGFRAG/$CF1` 256 cpu_count=`configfrag_boot_cpus "$TORTURE_BOOTARGS" "$CONFIGFRAG/$CF1" "$cpu_count"` 257 cpu_count=`configfrag_boot_maxcpus "$TORTURE_BOOTARGS" "$CONFIGFRAG/$CF1" "$cpu_count"` 258 echo $CF1 $cpu_count >> $T/cfgcpu 259 else 260 echo "The --configs file $CF1 does not exist, terminating." 261 exit 1 262 fi 263done 264sort -k2nr $T/cfgcpu -T="$T" > $T/cfgcpu.sort 265 266# Use a greedy bin-packing algorithm, sorting the list accordingly. 267awk < $T/cfgcpu.sort > $T/cfgcpu.pack -v ncpus=$cpus ' 268BEGIN { 269 njobs = 0; 270} 271 272{ 273 # Read file of tests and corresponding required numbers of CPUs. 274 cf[njobs] = $1; 275 cpus[njobs] = $2; 276 njobs++; 277} 278 279END { 280 batch = 0; 281 nc = -1; 282 283 # Each pass through the following loop creates on test batch 284 # that can be executed concurrently given ncpus. Note that a 285 # given test that requires more than the available CPUs will run in 286 # their own batch. Such tests just have to make do with what 287 # is available. 288 while (nc != ncpus) { 289 batch++; 290 nc = ncpus; 291 292 # Each pass through the following loop considers one 293 # test for inclusion in the current batch. 294 for (i = 0; i < njobs; i++) { 295 if (done[i]) 296 continue; # Already part of a batch. 297 if (nc >= cpus[i] || nc == ncpus) { 298 299 # This test fits into the current batch. 300 done[i] = batch; 301 nc -= cpus[i]; 302 if (nc <= 0) 303 break; # Too-big test in its own batch. 304 } 305 } 306 } 307 308 # Dump out the tests in batch order. 309 for (b = 1; b <= batch; b++) 310 for (i = 0; i < njobs; i++) 311 if (done[i] == b) 312 print cf[i], cpus[i]; 313}' 314 315# Generate a script to execute the tests in appropriate batches. 316cat << ___EOF___ > $T/script 317CONFIGFRAG="$CONFIGFRAG"; export CONFIGFRAG 318KVM="$KVM"; export KVM 319PATH="$PATH"; export PATH 320TORTURE_ALLOTED_CPUS="$TORTURE_ALLOTED_CPUS"; export TORTURE_ALLOTED_CPUS 321TORTURE_BOOT_IMAGE="$TORTURE_BOOT_IMAGE"; export TORTURE_BOOT_IMAGE 322TORTURE_BUILDONLY="$TORTURE_BUILDONLY"; export TORTURE_BUILDONLY 323TORTURE_DEFCONFIG="$TORTURE_DEFCONFIG"; export TORTURE_DEFCONFIG 324TORTURE_INITRD="$TORTURE_INITRD"; export TORTURE_INITRD 325TORTURE_KCONFIG_ARG="$TORTURE_KCONFIG_ARG"; export TORTURE_KCONFIG_ARG 326TORTURE_KCONFIG_KASAN_ARG="$TORTURE_KCONFIG_KASAN_ARG"; export TORTURE_KCONFIG_KASAN_ARG 327TORTURE_KCONFIG_KCSAN_ARG="$TORTURE_KCONFIG_KCSAN_ARG"; export TORTURE_KCONFIG_KCSAN_ARG 328TORTURE_KMAKE_ARG="$TORTURE_KMAKE_ARG"; export TORTURE_KMAKE_ARG 329TORTURE_QEMU_CMD="$TORTURE_QEMU_CMD"; export TORTURE_QEMU_CMD 330TORTURE_QEMU_INTERACTIVE="$TORTURE_QEMU_INTERACTIVE"; export TORTURE_QEMU_INTERACTIVE 331TORTURE_QEMU_MAC="$TORTURE_QEMU_MAC"; export TORTURE_QEMU_MAC 332TORTURE_QEMU_MEM="$TORTURE_QEMU_MEM"; export TORTURE_QEMU_MEM 333TORTURE_SHUTDOWN_GRACE="$TORTURE_SHUTDOWN_GRACE"; export TORTURE_SHUTDOWN_GRACE 334TORTURE_SUITE="$TORTURE_SUITE"; export TORTURE_SUITE 335TORTURE_TRUST_MAKE="$TORTURE_TRUST_MAKE"; export TORTURE_TRUST_MAKE 336if ! test -e $resdir 337then 338 mkdir -p "$resdir" || : 339fi 340mkdir $resdir/$ds 341TORTURE_RESDIR="$resdir/$ds"; export TORTURE_RESDIR 342TORTURE_STOPFILE="$resdir/$ds/STOP"; export TORTURE_STOPFILE 343echo Results directory: $resdir/$ds 344echo $scriptname $args 345touch $resdir/$ds/log 346echo $scriptname $args >> $resdir/$ds/log 347echo ${TORTURE_SUITE} > $resdir/$ds/TORTURE_SUITE 348pwd > $resdir/$ds/testid.txt 349if test -d .git 350then 351 git status >> $resdir/$ds/testid.txt 352 git rev-parse HEAD >> $resdir/$ds/testid.txt 353 git diff HEAD >> $resdir/$ds/testid.txt 354fi 355___EOF___ 356awk < $T/cfgcpu.pack \ 357 -v TORTURE_BUILDONLY="$TORTURE_BUILDONLY" \ 358 -v CONFIGDIR="$CONFIGFRAG/" \ 359 -v KVM="$KVM" \ 360 -v ncpus=$cpus \ 361 -v jitter="$jitter" \ 362 -v rd=$resdir/$ds/ \ 363 -v dur=$dur \ 364 -v TORTURE_QEMU_ARG="$TORTURE_QEMU_ARG" \ 365 -v TORTURE_BOOTARGS="$TORTURE_BOOTARGS" \ 366'BEGIN { 367 i = 0; 368} 369 370{ 371 cf[i] = $1; 372 cpus[i] = $2; 373 i++; 374} 375 376# Dump out the scripting required to run one test batch. 377function dump(first, pastlast, batchnum) 378{ 379 print "echo ----Start batch " batchnum ": `date` | tee -a " rd "log"; 380 print "needqemurun=" 381 jn=1 382 for (j = first; j < pastlast; j++) { 383 builddir=KVM "/b" j - first + 1 384 cpusr[jn] = cpus[j]; 385 if (cfrep[cf[j]] == "") { 386 cfr[jn] = cf[j]; 387 cfrep[cf[j]] = 1; 388 } else { 389 cfrep[cf[j]]++; 390 cfr[jn] = cf[j] "." cfrep[cf[j]]; 391 } 392 if (cpusr[jn] > ncpus && ncpus != 0) 393 ovf = "-ovf"; 394 else 395 ovf = ""; 396 print "echo ", cfr[jn], cpusr[jn] ovf ": Starting build. `date` | tee -a " rd "log"; 397 print "rm -f " builddir ".*"; 398 print "touch " builddir ".wait"; 399 print "mkdir " rd cfr[jn] " || :"; 400 print "kvm-test-1-run.sh " CONFIGDIR cf[j], builddir, rd cfr[jn], dur " \"" TORTURE_QEMU_ARG "\" \"" TORTURE_BOOTARGS "\" > " rd cfr[jn] "/kvm-test-1-run.sh.out 2>&1 &" 401 print "echo ", cfr[jn], cpusr[jn] ovf ": Waiting for build to complete. `date` | tee -a " rd "log"; 402 print "while test -f " builddir ".wait" 403 print "do" 404 print "\tsleep 1" 405 print "done" 406 print "echo ", cfr[jn], cpusr[jn] ovf ": Build complete. `date` | tee -a " rd "log"; 407 jn++; 408 } 409 for (j = 1; j < jn; j++) { 410 builddir=KVM "/b" j 411 print "rm -f " builddir ".ready" 412 print "if test -f \"" rd cfr[j] "/builtkernel\"" 413 print "then" 414 print "\techo ----", cfr[j], cpusr[j] ovf ": Kernel present. `date` | tee -a " rd "log"; 415 print "\tneedqemurun=1" 416 print "fi" 417 } 418 njitter = 0; 419 split(jitter, ja); 420 if (ja[1] == -1 && ncpus == 0) 421 njitter = 1; 422 else if (ja[1] == -1) 423 njitter = ncpus; 424 else 425 njitter = ja[1]; 426 if (TORTURE_BUILDONLY && njitter != 0) { 427 njitter = 0; 428 print "echo Build-only run, so suppressing jitter | tee -a " rd "log" 429 } 430 if (TORTURE_BUILDONLY) { 431 print "needqemurun=" 432 } 433 print "if test -n \"$needqemurun\"" 434 print "then" 435 print "\techo ---- Starting kernels. `date` | tee -a " rd "log"; 436 for (j = 0; j < njitter; j++) 437 print "\tjitter.sh " j " " dur " " ja[2] " " ja[3] "&" 438 print "\twait" 439 print "\techo ---- All kernel runs complete. `date` | tee -a " rd "log"; 440 print "else" 441 print "\twait" 442 print "\techo ---- No kernel runs. `date` | tee -a " rd "log"; 443 print "fi" 444 for (j = 1; j < jn; j++) { 445 builddir=KVM "/b" j 446 print "echo ----", cfr[j], cpusr[j] ovf ": Build/run results: | tee -a " rd "log"; 447 print "cat " rd cfr[j] "/kvm-test-1-run.sh.out | tee -a " rd "log"; 448 } 449} 450 451END { 452 njobs = i; 453 nc = ncpus; 454 first = 0; 455 batchnum = 1; 456 457 # Each pass through the following loop considers one test. 458 for (i = 0; i < njobs; i++) { 459 if (ncpus == 0) { 460 # Sequential test specified, each test its own batch. 461 dump(i, i + 1, batchnum); 462 first = i; 463 batchnum++; 464 } else if (nc < cpus[i] && i != 0) { 465 # Out of CPUs, dump out a batch. 466 dump(first, i, batchnum); 467 first = i; 468 nc = ncpus; 469 batchnum++; 470 } 471 # Account for the CPUs needed by the current test. 472 nc -= cpus[i]; 473 } 474 # Dump the last batch. 475 if (ncpus != 0) 476 dump(first, i, batchnum); 477}' >> $T/script 478 479cat << ___EOF___ >> $T/script 480echo 481echo 482echo " --- `date` Test summary:" 483echo Results directory: $resdir/$ds 484kcsan-collapse.sh $resdir/$ds 485kvm-recheck.sh $resdir/$ds 486___EOF___ 487 488if test "$dryrun" = script 489then 490 cat $T/script 491 exit 0 492elif test "$dryrun" = sched 493then 494 # Extract the test run schedule from the script. 495 egrep 'Start batch|Starting build\.' $T/script | 496 grep -v ">>" | 497 sed -e 's/:.*$//' -e 's/^echo //' 498 exit 0 499else 500 # Not a dryrun, so run the script. 501 sh $T/script 502fi 503 504# Tracing: trace_event=rcu:rcu_grace_period,rcu:rcu_future_grace_period,rcu:rcu_grace_period_init,rcu:rcu_nocb_wake,rcu:rcu_preempt_task,rcu:rcu_unlock_preempted_task,rcu:rcu_quiescent_state_report,rcu:rcu_fqs,rcu:rcu_callback,rcu:rcu_kfree_callback,rcu:rcu_batch_start,rcu:rcu_invoke_callback,rcu:rcu_invoke_kfree_callback,rcu:rcu_batch_end,rcu:rcu_torture_read,rcu:rcu_barrier 505# Function-graph tracing: ftrace=function_graph ftrace_graph_filter=sched_setaffinity,migration_cpu_stop 506# Also --kconfig "CONFIG_FUNCTION_TRACER=y CONFIG_FUNCTION_GRAPH_TRACER=y" 507# Control buffer size: --bootargs trace_buf_size=3k 508# Get trace-buffer dumps on all oopses: --bootargs ftrace_dump_on_oops 509# Ditto, but dump only the oopsing CPU: --bootargs ftrace_dump_on_oops=orig_cpu 510# Heavy-handed way to also dump on warnings: --bootargs panic_on_warn 511