1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0 3# 4# Run the objtool klp tests. Each test-*.sh prints one TAP result line. 5# 6# Tests live in generic/ and in a directory per architecture. A run executes 7# generic/ plus the one matching this architecture, so a test which cannot 8# apply here is not run rather than reporting a skip; what was left out is 9# reported once, as a comment, so differing coverage is still visible. 10# 11# A run covers one compiler and one architecture; CI runs the combinations. 12# The harness checks the environment once up front and fails the run if the 13# suite cannot execute, rather than letting every test skip and exit 0. 14 15set -u 16 17# Determinism: the order test-*.sh expands in, and how grep's character ranges 18# and sort's collation behave inside the tests, are all locale-dependent. A 19# suite whose results depend on the invoking shell's locale is a suite whose 20# failures cannot be reproduced. 21export LC_ALL=C 22 23usage() 24{ 25 cat <<EOF 26usage: $(basename "$0") [-k|--keep] [test...] 27 28Run the objtool klp tests for this architecture: everything in generic/, plus 29everything in the directory named for it. With no arguments, runs all of them. 30A test may be named with or without its "test-" prefix and ".sh" suffix, and is 31looked for in both directories. 32 33Options: 34 -k, --keep same as KEEP=all (see below) 35 36Environment: 37 OBJTOOL objtool binary to test (default ../objtool) 38 CC compiler used to build fixtures (default gcc) 39 ARCH architecture the tests are for (default: uname -m) 40 KEEP failed keep only failing tests (default) 41 all keep every test's working directory 42 none remove all working directories 43 44A test which needs something of its own says so in its skip message. 45EOF 46 exit "${1:-0}" 47} 48 49cd "$(dirname "$0")" || exit 1 50 51keep_from_args= 52while [ $# -gt 0 ]; do 53 case "$1" in 54 -h|--help) usage ;; 55 -k|--keep) keep_from_args=all; shift ;; 56 --) shift; break ;; 57 -*) echo "unknown option: $1" >&2; usage 1 ;; 58 *) break ;; 59 esac 60done 61 62KLP_TEST_KEEP="${KEEP:-failed}" 63[ -n "$keep_from_args" ] && KLP_TEST_KEEP="$keep_from_args" 64case "$KLP_TEST_KEEP" in 65all|none|failed) ;; 66*) 67 echo "invalid KEEP=$KLP_TEST_KEEP (want failed, all, or none)" >&2 68 exit 1 69 ;; 70esac 71export KLP_TEST_KEEP 72 73echo "TAP version 13" 74 75# Sourcing the harness runs its preflight, which decides which architecture 76# this run is for -- so the test list cannot be built before it has, and the 77# tests inherit the answers rather than working them out again. 78. ./lib.sh 79 80dirs=( generic ) 81[ -d "$KLP_TEST_ARCH" ] && dirs+=( "$KLP_TEST_ARCH" ) 82 83if [ $# -gt 0 ]; then 84 tests=() 85 for arg in "$@"; do 86 name="test-${arg#test-}"; name="${name%.sh}.sh" 87 found= 88 for d in "${dirs[@]}"; do 89 [ -f "$d/$name" ] || continue 90 [ -x "$d/$name" ] || 91 { echo "not executable: $d/$name" >&2; exit 1; } 92 tests+=( "$d/$name" ); found=y 93 done 94 [ -n "$found" ] || 95 { echo "no such test for $KLP_TEST_ARCH: $arg" >&2; exit 1; } 96 done 97else 98 tests=() 99 for d in "${dirs[@]}"; do 100 for t in "$d"/test-*.sh; do 101 [ -f "$t" ] && tests+=( "$t" ) 102 done 103 done 104 [ "${#tests[@]}" -gt 0 ] || 105 { echo "1..0 # SKIP no tests found"; exit 0; } 106 107 # Tests for another architecture are absent from this run entirely. Say 108 # how many, so a run which covers less than the tree holds does not look 109 # like one that covers all of it. 110 for d in */; do 111 d="${d%/}" 112 case "$d" in generic|"$KLP_TEST_ARCH") continue ;; esac 113 n=$(ls "$d"/test-*.sh 2>/dev/null | wc -l) 114 [ "$n" -gt 0 ] || continue 115 echo "# not run: $n test$( [ "$n" = 1 ] || echo s ) in $d/" \ 116 "(this run is $KLP_TEST_ARCH)" 117 done 118fi 119 120# One directory for the whole run, one per test inside it, mirroring the 121# source layout. A run then leaves a single thing behind instead of 39 122# scattered among everything else using mktemp. 123rundir="$(mktemp -d "${TMPDIR:-/tmp}/klp-tests.XXXXXXXX")" || 124 { echo "Bail out! cannot create a working directory" >&2; exit 1; } 125 126# The run's own two levels: each test's directory, and the one per source 127# directory holding them. Take them away if the tests left them empty, and 128# say so if they did not. Either rmdir may fail -- the glob stays unexpanded 129# when nothing was created -- so ask the directory itself rather than trusting 130# the status. Never rm -rf: what to keep is the tests' decision, made in 131# cleanup() as each one exits, and this must not overrule it. 132reap_rundir() 133{ 134 rmdir "$rundir"/*/ 2>/dev/null 135 rmdir "$rundir" 2>/dev/null 136 [ -d "$rundir" ] 137} 138 139# An interrupted run has the same directory to answer for, and the tests it 140# never reached will not clean up on their way out. The one it was running 141# has, and under the default it kept what it had built, so say where. 142interrupted() 143{ 144 reap_rundir && echo "# interrupted; what was built is in $rundir" 145 exit 130 146} 147trap interrupted INT TERM HUP 148 149echo "1..${#tests[@]}" 150 151pass=0 fail=0 static_skip=0 probe_skip=0 xfail=0 xpass=0 152failed_dirs=() 153 154for t in "${tests[@]}"; do 155 out="$(KLP_TEST_WORKDIR="$rundir/${t%.sh}" ./"$t" 2>&1)" 156 rc=$? 157 158 # A test prints one result line, but it is not necessarily the only 159 # thing it prints: objtool warns on stderr, and the runner captures 160 # that. Classify the result line itself rather than the whole of the 161 # output, or a stray line ahead of it makes every pattern below miss and 162 # the exit status decide -- which would count an expected failure, which 163 # exits 0, as a pass. 164 result="$(printf '%s\n' "$out" | grep -E '^(ok|not ok)' | tail -1)" 165 rest="$(printf '%s\n' "$out" | grep -Ev '^(ok|not ok)')" 166 167 # Classify from the result line, not the exit status: a skip and a pass 168 # both exit 0, and telling them apart is the point of counting. 169 # 170 # The two skip kinds differ in what they promise. A static skip was 171 # declared before the test ran ("clang does not do this"), so it is 172 # expected indefinitely. A probe skip means the construct did not turn 173 # up this time, which is weaker and worth watching: one that becomes 174 # permanent is a fixture that quietly stopped testing anything. 175 case "$result" in 176 *"# SKIP (declared)"*) static_skip=$((static_skip + 1)) ;; 177 *"# SKIP (probe)"*) probe_skip=$((probe_skip + 1)) ;; 178 *"# SKIP"*) 179 # An undeclared skip: the test gave up for a reason it never 180 # said it might. That is a hole, not an expected outcome. 181 # 182 # Replace the line rather than adding one. Every test owes the 183 # plan exactly one result, and a consumer counting them is 184 # entitled to say so when the totals disagree. 185 rest="$rest${rest:+$'\n'}was: $result" 186 result="not ok - $(basename "$t" .sh): undeclared skip" 187 result="$result (use gcc_only/clang_only or require_input_*)" 188 fail=$((fail + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;; 189 "not ok"*"# TODO"*) xfail=$((xfail + 1)) ;; 190 "ok"*"# TODO"*) xpass=$((xpass + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;; 191 "not ok"*) fail=$((fail + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;; 192 "ok"*) pass=$((pass + 1)) ;; 193 *) 194 # No result line at all: the test died before reporting. 195 rest="$rest${rest:+$'\n'}exited $rc without a result line" 196 result="not ok - $(basename "$t" .sh): no TAP result" 197 fail=$((fail + 1)); failed_dirs+=( "$rundir/${t%.sh}" ) ;; 198 esac 199 200 echo "$result" 201 [ -n "$rest" ] && printf '%s\n' "$rest" | sed 's/^[^#]/# &/' 202 203done 204 205echo "# pass:$pass fail:$fail static-skip:$static_skip" \ 206 "probe-skip:$probe_skip xfail:$xfail xpass:$xpass" 207 208case "$KLP_TEST_KEEP" in 209all) 210 echo "# keep=all: workdirs kept in $rundir" 211 echo "# inspect: diff.log, readelf -S out.o under each test-* subdirectory" 212 echo "# cleanup: rm -rf $rundir" 213 ;; 214failed) 215 if [ "${#failed_dirs[@]}" -gt 0 ]; then 216 echo "# keep=failed: ${#failed_dirs[@]} failing test(s) kept under $rundir:" 217 for d in "${failed_dirs[@]}"; do 218 echo "# ${d#"$rundir"/}/" 219 done 220 echo "# inspect: diff.log readelf -S out.o" 221 echo "# one test: $PWD/run-tests.sh <name>" 222 echo "# cleanup: rm -rf $rundir" 223 elif reap_rundir; then 224 echo "# $rundir was not empty;" \ 225 "a test did not clean up after itself" 226 fi 227 ;; 228none) 229 if reap_rundir; then 230 echo "# $rundir was not empty;" \ 231 "a test did not clean up after itself" 232 elif [ "$fail" != 0 ] || [ "$xpass" != 0 ]; then 233 echo "# keep=none: artifacts were removed" \ 234 "(re-run with KEEP=failed or KEEP=all)" 235 fi 236 ;; 237esac 238 239[ "$fail" = 0 ] && [ "$xpass" = 0 ] 240