2 # SPDX-License-Identifier: GPL-2.0+
4 # Run a series of tests on remote systems under KVM.
6 # Usage: kvm-remote.sh "systems" [ <kvm.sh args> ]
7 # kvm-remote.sh "systems" /path/to/old/run [ <kvm-again.sh args> ]
9 # Copyright (C) 2021 Facebook, Inc.
11 # Authors: Paul E. McKenney <paulmck@kernel.org>
16 if ! test -d tools/testing/selftests/rcutorture/bin
18 echo $scriptname must be run from top-level directory of kernel source tree.
22 RCUTORTURE="`pwd`/tools/testing/selftests/rcutorture"; export RCUTORTURE
23 PATH=${RCUTORTURE}/bin:$PATH; export PATH
26 starttime="`get_starttime`"
31 echo $scriptname: Empty list of systems will go nowhere good, giving up.
37 # T: /tmp/kvm-remote.sh.$$
38 # resdir: /tmp/kvm-remote.sh.$$/res
39 # rundir: /tmp/kvm-remote.sh.$$/res/$ds ("-remote" suffix)
40 # oldrun: `pwd`/tools/testing/.../res/$otherds
43 # TD: kvm-remote.sh.$$
44 # ds: yyyy.mm.dd-hh.mm.ss-remote
52 ds=`date +%Y.%m.%d-%H.%M.%S`-remote
54 echo Results directory: $rundir
55 echo $scriptname $args
56 if echo $1 | grep -q '^--'
58 # Fresh build. Create a datestamp unless the caller supplied one.
59 datestamp="`echo "$@" | awk -v ds="$ds" '{
60 for (i = 1; i < NF; i++) {
61 if ($i == "--datestamp") {
67 print "--datestamp " ds;
69 kvm.sh --remote "$@" $datestamp --buildonly > $T/kvm.sh.out 2>&1
73 echo $scriptname: kvm.sh failed exit code $?
77 oldrun="`grep -m 1 "^Results directory: " $T/kvm.sh.out | awk '{ print $3 }'`"
78 touch "$oldrun/remote-log"
79 echo $scriptname $args >> "$oldrun/remote-log"
80 echo | tee -a "$oldrun/remote-log"
81 echo " ----" kvm.sh output: "(`date`)" | tee -a "$oldrun/remote-log"
82 cat $T/kvm.sh.out | tee -a "$oldrun/remote-log"
83 # We are going to run this, so remove the buildonly files.
84 rm -f "$oldrun"/*/buildonly
85 kvm-again.sh $oldrun --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&1
89 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log"
90 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log"
96 if ! echo $oldrun | grep -q '^/'
98 oldrun="`pwd`/$oldrun"
101 touch "$oldrun/remote-log"
102 echo $scriptname $args >> "$oldrun/remote-log"
103 kvm-again.sh "$oldrun" "$@" --dryrun --remote --rundir "$rundir" > $T/kvm-again.sh.out 2>&1
107 echo $scriptname: kvm-again.sh failed exit code $? | tee -a "$oldrun/remote-log"
108 cat $T/kvm-again.sh.out | tee -a "$oldrun/remote-log"
111 cp -a "$rundir" "$RCUTORTURE/res/"
112 oldrun="$RCUTORTURE/res/$ds"
114 echo | tee -a "$oldrun/remote-log"
115 echo " ----" kvm-again.sh output: "(`date`)" | tee -a "$oldrun/remote-log"
116 cat $T/kvm-again.sh.out
117 echo | tee -a "$oldrun/remote-log"
118 echo Remote run directory: $rundir | tee -a "$oldrun/remote-log"
119 echo Local build-side run directory: $oldrun | tee -a "$oldrun/remote-log"
121 # Create the kvm-remote-N.sh scripts in the bin directory.
122 awk < "$rundir"/scenarios -v dest="$T/bin" -v rundir="$rundir" '
126 fn = dest "/kvm-remote-" n ".sh"
127 print "kvm-remote-noreap.sh " rundir " &" > fn;
129 for (i = 2; i <= NF; i++)
130 scenarios = scenarios " " $i;
131 print "kvm-test-1-run-batch.sh" scenarios >> fn;
133 print "rm " rundir "/remote.run" >> fn;
135 chmod +x $T/bin/kvm-remote-*.sh
136 ( cd "`dirname $T`"; tar -chzf $T/binres.tgz "$TD/bin" "$TD/res" )
138 # Check first to avoid the need for cleanup for system-name typos
141 ncpus="`ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN 2> /dev/null`"
145 echo System $i unreachable, giving up. | tee -a "$oldrun/remote-log"
148 echo $i: $ncpus CPUs " " `date` | tee -a "$oldrun/remote-log"
151 # Download and expand the tarball on all systems.
152 echo Build-products tarball: `du -h $T/binres.tgz` | tee -a "$oldrun/remote-log"
155 echo Downloading tarball to $i `date` | tee -a "$oldrun/remote-log"
156 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -"
159 while test "$ret" -ne 0
161 echo Unable to download $T/binres.tgz to system $i, waiting and then retrying. $tries prior retries. | tee -a "$oldrun/remote-log"
163 cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -"
169 echo Unable to download $T/binres.tgz to system $i, giving up. | tee -a "$oldrun/remote-log"
177 # Function to check for presence of a file on the specified system.
178 # Complain if the system cannot be reached, and retry after a wait.
179 # Currently just waits forever if a machine disappears.
181 # Usage: checkremotefile system pathname
188 ssh -o BatchMode=yes $1 "test -f \"$2\""
190 if test "$ret" -eq 255
192 echo " ---" ssh failure to $1 checking for file $2, retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log"
193 elif test "$ret" -eq 0
196 elif test "$ret" -eq 1
198 echo " ---" File \"$2\" not found: ssh $1 test -f \"$2\" | tee -a "$oldrun/remote-log"
201 echo " ---" Exit code $ret: ssh $1 test -f \"$2\", retry after $sleeptime seconds. `date` | tee -a "$oldrun/remote-log"
208 # Function to start batches on idle remote $systems
210 # Usage: startbatches curbatch nbatches
212 # Batches are numbered starting at 1. Returns the next batch to start.
213 # Be careful to redirect all debug output to FD 2 (stderr).
219 # Each pass through the following loop examines one system.
222 if test "$curbatch" -gt "$nbatches"
224 echo $((nbatches + 1))
227 if checkremotefile "$i" "$resdir/$ds/remote.run" 1>&2
229 continue # System still running last test, skip.
231 ssh -o BatchMode=yes "$i" "cd \"$resdir/$ds\"; touch remote.run; PATH=\"$T/bin:$PATH\" nohup kvm-remote-$curbatch.sh > kvm-remote-$curbatch.sh.out 2>&1 &" 1>&2
235 echo ssh $i failed: exitcode $ret 1>&2
238 echo " ----" System $i Batch `head -n $curbatch < "$rundir"/scenarios | tail -1` `date` 1>&2
239 curbatch=$((curbatch + 1))
244 # Launch all the scenarios.
245 nbatches="`wc -l "$rundir"/scenarios | awk '{ print $1 }'`"
247 while test "$curbatch" -le "$nbatches"
249 startbatches $curbatch $nbatches > $T/curbatch 2> $T/startbatches.stderr
250 curbatch="`cat $T/curbatch`"
251 if test -s "$T/startbatches.stderr"
253 cat "$T/startbatches.stderr" | tee -a "$oldrun/remote-log"
255 if test "$curbatch" -le "$nbatches"
260 echo All batches started. `date` | tee -a "$oldrun/remote-log"
262 # Wait for all remaining scenarios to complete and collect results.
265 while checkremotefile "$i" "$resdir/$ds/remote.run"
269 echo " ---" Collecting results from $i `date` | tee -a "$oldrun/remote-log"
270 ( cd "$oldrun"; ssh -o BatchMode=yes $i "cd $rundir; tar -czf - kvm-remote-*.sh.out */console.log */kvm-test-1-run*.sh.out */qemu[_-]pid */qemu-retval */qemu-affinity; rm -rf $T > /dev/null 2>&1" | tar -xzf - )
273 ( kvm-end-run-stats.sh "$oldrun" "$starttime"; echo $? > $T/exitcode ) | tee -a "$oldrun/remote-log"
274 exit "`cat $T/exitcode`"