1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Run a series of tests on remote systems under KVM. 5# 6# Usage: kvm-remote.sh "systems" [ <kvm.sh args> ] 7# kvm-remote.sh "systems" /path/to/old/run [ <kvm-again.sh args> ] 8# 9# Copyright (C) 2021 Facebook, Inc. 10# 11# Authors: Paul E. McKenney <paulmck@kernel.org> 12 13scriptname=$0 14args="$*" 15 16if ! test -d tools/testing/selftests/rcutorture/bin 17then 18 echo $scriptname must be run from top-level directory of kernel source tree. 19 exit 1 20fi 21 22RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE 23PATH=${RCUTORTURE}/bin:$PATH; export PATH 24. functions.sh 25 26starttime="`get_starttime`" 27 28systems="$1" 29if test -z "$systems" 30then 31 echo $scriptname: Empty list of systems will go nowhere good, giving up. 32 exit 1 33fi 34shift 35 36# Pathnames: 37# T: /tmp/kvm-remote.sh.NNNNNN where "NNNNNN" is set by mktemp 38# resdir: /tmp/kvm-remote.sh.NNNNNN/res 39# rundir: /tmp/kvm-remote.sh.NNNNNN/res/$ds ("-remote" suffix) 40# oldrun: `pwd`/tools/testing/.../res/$otherds 41# 42# Pathname segments: 43# TD: kvm-remote.sh.NNNNNN 44# ds: yyyy.mm.dd-hh.mm.ss-remote 45 46T="`mktemp -d ${TMPDIR-/tmp}/kvm-remote.sh.XXXXXX`" 47trap 'rm -rf $T' 0 48TD="`basename "$T"`" 49 50resdir="$T/res" 51ds=`date +%Y.%m.%d-%H.%M.%S`-remote 52rundir=$resdir/$ds 53echo Results directory: $rundir 54echo $scriptname $args 55if echo $1 | grep -q '^--' 56then 57 # Fresh build. Create a datestamp unless the caller supplied one. 58 datestamp="`echo "$@" | awk -v ds="$ds" '{ 59 for (i = 1; i < NF; i++) { 60 if ($i == "--datestamp") { 61 ds = ""; 62 break; 63 } 64 } 65 if (ds != "") 66 print "--datestamp " ds; 67 }'`" 68 kvm.sh --remote "$@" $datestamp --buildonly > $T/kvm.sh.out 2>&1 69 ret=$? 70 if test "$ret" -ne 0 71 then 72 echo $scriptname: kvm.sh failed exit code $? 73 cat $T/kvm.sh.out 74 exit 2 75 fi 76 oldrun="`grep -m 1 "^Results directory: " $T/kvm.sh.out | awk '{ print $3 }'`" 77 touch "$oldrun/remote-log" 78 echo $scriptname $args >> "$oldrun/remote-log" 79 echo | tee -a "$oldrun/remote-log" 80 echo " ----" kvm.sh output: "(`date`)" | tee -a "$oldrun/remote-log" 81 cat $T/kvm.sh.out | tee -a "$oldrun/remote-log" 82 # We are going to run this, so remove the buildonly files. 83 rm -f "$oldrun"/*/buildonly 84 kvm-again.sh $oldrun --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&1 85 ret=$? 86 if test "$ret" -ne 0 87 then 88 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log" 89 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log" 90 exit 2 91 fi 92else 93 # Re-use old run. 94 oldrun="$1" 95 if ! echo $oldrun | grep -q '^/' 96 then 97 oldrun="`pwd`/$oldrun" 98 fi 99 shift 100 touch "$oldrun/remote-log" 101 echo $scriptname $args >> "$oldrun/remote-log" 102 kvm-again.sh "$oldrun" "$@" --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&1 103 ret=$? 104 if test "$ret" -ne 0 105 then 106 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log" 107 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log" 108 exit 2 109 fi 110 cp -a "$rundir" "$RCUTORTURE/res/" 111 oldrun="$RCUTORTURE/res/$ds" 112fi 113echo | tee -a "$oldrun/remote-log" 114echo " ----" kvm-again.sh output: "(`date`)" | tee -a "$oldrun/remote-log" 115cat $T/kvm-again.sh.out 116echo | tee -a "$oldrun/remote-log" 117echo Remote run directory: $rundir | tee -a "$oldrun/remote-log" 118echo Local build-side run directory: $oldrun | tee -a "$oldrun/remote-log" 119 120# Create the kvm-remote-N.sh scripts in the bin directory. 121awk < "$rundir"/scenarios -v dest="$T/bin" -v rundir="$rundir" ' 122{ 123 n = $1; 124 sub(/\./, "", n); 125 fn = dest "/kvm-remote-" n ".sh" 126 print "kvm-remote-noreap.sh " rundir " &" > fn; 127 scenarios = ""; 128 for (i = 2; i <= NF; i++) 129 scenarios = scenarios " " $i; 130 print "kvm-test-1-run-batch.sh" scenarios >> fn; 131 print "sync" >> fn; 132 print "rm " rundir "/remote.run" >> fn; 133}' 134chmod +x $T/bin/kvm-remote-*.sh 135( cd "`dirname $T`"; tar -chzf $T/binres.tgz "$TD/bin" "$TD/res" ) 136 137# Check first to avoid the need for cleanup for system-name typos 138for i in $systems 139do 140 ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN > $T/ssh.stdout 2> $T/ssh.stderr 141 ret=$? 142 if test "$ret" -ne 0 143 then 144 echo "System $i unreachable ($ret), giving up." | tee -a "$oldrun/remote-log" 145 echo ' --- ssh stdout: vvv' | tee -a "$oldrun/remote-log" 146 cat $T/ssh.stdout | tee -a "$oldrun/remote-log" 147 echo ' --- ssh stdout: ^^^' | tee -a "$oldrun/remote-log" 148 echo ' --- ssh stderr: vvv' | tee -a "$oldrun/remote-log" 149 cat $T/ssh.stderr | tee -a "$oldrun/remote-log" 150 echo ' --- ssh stderr: ^^^' | tee -a "$oldrun/remote-log" 151 exit 4 152 fi 153 echo $i: `cat $T/ssh.stdout` CPUs " " `date` | tee -a "$oldrun/remote-log" 154done 155 156# Download and expand the tarball on all systems. 157echo Build-products tarball: `du -h $T/binres.tgz` | tee -a "$oldrun/remote-log" 158for i in $systems 159do 160 echo Downloading tarball to $i `date` | tee -a "$oldrun/remote-log" 161 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -" 162 ret=$? 163 tries=0 164 while test "$ret" -ne 0 165 do 166 echo Unable to download $T/binres.tgz to system $i, waiting and then retrying. $tries prior retries. | tee -a "$oldrun/remote-log" 167 sleep 60 168 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -" 169 ret=$? 170 if test "$ret" -ne 0 171 then 172 if test "$tries" > 5 173 then 174 echo Unable to download $T/binres.tgz to system $i, giving up. | tee -a "$oldrun/remote-log" 175 exit 10 176 fi 177 fi 178 tries=$((tries+1)) 179 done 180done 181 182# Function to check for presence of a file on the specified system. 183# Complain if the system cannot be reached, and retry after a wait. 184# Currently just waits forever if a machine disappears. 185# 186# Usage: checkremotefile system pathname 187checkremotefile () { 188 local ret 189 local sleeptime=60 190 191 while : 192 do 193 ssh -o BatchMode=yes $1 "test -f \"$2\"" 194 ret=$? 195 if test "$ret" -eq 255 196 then 197 echo " ---" ssh failure to $1 checking for file $2, retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log" 198 elif test "$ret" -eq 0 199 then 200 return 0 201 elif test "$ret" -eq 1 202 then 203 echo " ---" File \"$2\" not found: ssh $1 test -f \"$2\" | tee -a "$oldrun/remote-log" 204 return 1 205 else 206 echo " ---" Exit code $ret: ssh $1 test -f \"$2\", retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log" 207 return $ret 208 fi 209 sleep $sleeptime 210 done 211} 212 213# Function to start batches on idle remote $systems 214# 215# Usage: startbatches curbatch nbatches 216# 217# Batches are numbered starting at 1. Returns the next batch to start. 218# Be careful to redirect all debug output to FD 2 (stderr). 219startbatches () { 220 local curbatch="$1" 221 local nbatches="$2" 222 local ret 223 224 # Each pass through the following loop examines one system. 225 for i in $systems 226 do 227 if test "$curbatch" -gt "$nbatches" 228 then 229 echo $((nbatches + 1)) 230 return 0 231 fi 232 if checkremotefile "$i" "$resdir/$ds/remote.run" 1>&2 233 then 234 continue # System still running last test, skip. 235 fi 236 ssh -o BatchMode=yes "$i" "cd \"$resdir/$ds\"; touch remote.run; PATH=\"$T/bin:$PATH\" nohup kvm-remote-$curbatch.sh > kvm-remote-$curbatch.sh.out 2>&1 &" 1>&2 237 ret=$? 238 if test "$ret" -ne 0 239 then 240 echo ssh $i failed: exitcode $ret 1>&2 241 exit 11 242 fi 243 echo " ----" System $i Batch `head -n $curbatch < "$rundir"/scenarios | tail -1` `date` 1>&2 244 curbatch=$((curbatch + 1)) 245 done 246 echo $curbatch 247} 248 249# Launch all the scenarios. 250nbatches="`wc -l "$rundir"/scenarios | awk '{ print $1 }'`" 251curbatch=1 252while test "$curbatch" -le "$nbatches" 253do 254 startbatches $curbatch $nbatches > $T/curbatch 2> $T/startbatches.stderr 255 curbatch="`cat $T/curbatch`" 256 if test -s "$T/startbatches.stderr" 257 then 258 cat "$T/startbatches.stderr" | tee -a "$oldrun/remote-log" 259 fi 260 if test "$curbatch" -le "$nbatches" 261 then 262 sleep 30 263 fi 264done 265echo All batches started. `date` | tee -a "$oldrun/remote-log" 266 267# Wait for all remaining scenarios to complete and collect results. 268for i in $systems 269do 270 echo " ---" Waiting for $i `date` | tee -a "$oldrun/remote-log" 271 while checkremotefile "$i" "$resdir/$ds/remote.run" 272 do 273 sleep 30 274 done 275 echo " ---" Collecting results from $i `date` | tee -a "$oldrun/remote-log" 276 ( cd "$oldrun"; ssh -o BatchMode=yes $i "cd $rundir; tar -czf - kvm-remote-*.sh.out */console.log */kvm-test-1-run*.sh.out */qemu[_-]pid */qemu-retval */qemu-affinity; rm -rf $T > /dev/null 2>&1" | tar -xzf - ) 277done 278 279( kvm-end-run-stats.sh "$oldrun" "$starttime"; echo $? > $T/exitcode ) | tee -a "$oldrun/remote-log" 280exit "`cat $T/exitcode`" 281