1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0 3# 4# Run the objtool klp tests. Each test-*.sh prints one TAP result line. 5# 6# Tests live in generic/ and in a directory per architecture. A run executes 7# generic/ plus the one matching this architecture, so a test which cannot 8# apply here is not run rather than reporting a skip; what was left out is 9# reported once, as a comment, so differing coverage is still visible. 10# 11# A run covers one compiler and one architecture; CI runs the combinations. 12# The harness checks the environment once up front and fails the run if the 13# suite cannot execute, rather than letting every test skip and exit 0. 14 15set -u 16 17# Determinism: the order test-*.sh expands in, and how grep's character ranges 18# and sort's collation behave inside the tests, are all locale-dependent. A 19# suite whose results depend on the invoking shell's locale is a suite whose 20# failures cannot be reproduced. 21export LC_ALL=C 22 23usage() 24{ 25 cat <<EOF 26usage: $(basename "$0") [-k|--keep] [test...] 27 28Run the objtool klp tests for this architecture: everything in generic/, plus 29everything in the directory named for it. With no arguments, runs all of them. 30A test may be named with or without its "test-" prefix and ".sh" suffix, and is 31looked for in both directories. 32 33Options: 34 -k, --keep do not delete each test's working directory; print its path, 35 so the objects a failing test built can be looked at 36 37Environment: 38 OBJTOOL objtool binary to test (default ../objtool) 39 CC compiler used to build fixtures (default gcc) 40 ARCH architecture the tests are for (default: uname -m) 41 42A test which needs something of its own says so in its skip message. 43EOF 44 exit "${1:-0}" 45} 46 47cd "$(dirname "$0")" || exit 1 48 49while [ $# -gt 0 ]; do 50 case "$1" in 51 -h|--help) usage ;; 52 -k|--keep) export KLP_TEST_KEEP=1; shift ;; 53 --) shift; break ;; 54 -*) echo "unknown option: $1" >&2; usage 1 ;; 55 *) break ;; 56 esac 57done 58 59echo "TAP version 13" 60 61# Sourcing the harness runs its preflight, which decides which architecture 62# this run is for -- so the test list cannot be built before it has, and the 63# tests inherit the answers rather than working them out again. 64. ./lib.sh 65 66dirs=( generic ) 67[ -d "$KLP_TEST_ARCH" ] && dirs+=( "$KLP_TEST_ARCH" ) 68 69if [ $# -gt 0 ]; then 70 tests=() 71 for arg in "$@"; do 72 name="test-${arg#test-}"; name="${name%.sh}.sh" 73 found= 74 for d in "${dirs[@]}"; do 75 [ -f "$d/$name" ] || continue 76 [ -x "$d/$name" ] || 77 { echo "not executable: $d/$name" >&2; exit 1; } 78 tests+=( "$d/$name" ); found=y 79 done 80 [ -n "$found" ] || 81 { echo "no such test for $KLP_TEST_ARCH: $arg" >&2; exit 1; } 82 done 83else 84 tests=() 85 for d in "${dirs[@]}"; do 86 for t in "$d"/test-*.sh; do 87 [ -f "$t" ] && tests+=( "$t" ) 88 done 89 done 90 [ "${#tests[@]}" -gt 0 ] || 91 { echo "1..0 # SKIP no tests found"; exit 0; } 92 93 # Tests for another architecture are absent from this run entirely. Say 94 # how many, so a run which covers less than the tree holds does not look 95 # like one that covers all of it. 96 for d in */; do 97 d="${d%/}" 98 case "$d" in generic|"$KLP_TEST_ARCH") continue ;; esac 99 n=$(ls "$d"/test-*.sh 2>/dev/null | wc -l) 100 [ "$n" -gt 0 ] || continue 101 echo "# not run: $n test$( [ "$n" = 1 ] || echo s ) in $d/" \ 102 "(this run is $KLP_TEST_ARCH)" 103 done 104fi 105 106# One directory for the whole run, one per test inside it, mirroring the 107# source layout. A run then leaves a single thing behind instead of 39 108# scattered among everything else using mktemp. 109rundir="$(mktemp -d "${TMPDIR:-/tmp}/klp-tests.XXXXXXXX")" || 110 { echo "Bail out! cannot create a working directory" >&2; exit 1; } 111 112echo "1..${#tests[@]}" 113 114pass=0 fail=0 static_skip=0 probe_skip=0 xfail=0 xpass=0 115 116for t in "${tests[@]}"; do 117 out="$(KLP_TEST_WORKDIR="$rundir/${t%.sh}" ./"$t" 2>&1)" 118 rc=$? 119 120 # A test prints one result line, but it is not necessarily the only 121 # thing it prints: objtool warns on stderr, and the runner captures 122 # that. Classify the result line itself rather than the whole of the 123 # output, or a stray line ahead of it makes every pattern below miss and 124 # the exit status decide -- which would count an expected failure, which 125 # exits 0, as a pass. 126 result="$(printf '%s\n' "$out" | grep -E '^(ok|not ok)' | tail -1)" 127 rest="$(printf '%s\n' "$out" | grep -Ev '^(ok|not ok)')" 128 129 # Classify from the result line, not the exit status: a skip and a pass 130 # both exit 0, and telling them apart is the point of counting. 131 # 132 # The two skip kinds differ in what they promise. A static skip was 133 # declared before the test ran ("clang does not do this"), so it is 134 # expected indefinitely. A probe skip means the construct did not turn 135 # up this time, which is weaker and worth watching: one that becomes 136 # permanent is a fixture that quietly stopped testing anything. 137 case "$result" in 138 *"# SKIP (declared)"*) static_skip=$((static_skip + 1)) ;; 139 *"# SKIP (probe)"*) probe_skip=$((probe_skip + 1)) ;; 140 *"# SKIP"*) 141 # An undeclared skip: the test gave up for a reason it never 142 # said it might. That is a hole, not an expected outcome. 143 # 144 # Replace the line rather than adding one. Every test owes the 145 # plan exactly one result, and a consumer counting them is 146 # entitled to say so when the totals disagree. 147 rest="$rest${rest:+$'\n'}was: $result" 148 result="not ok - $(basename "$t" .sh): undeclared skip" 149 result="$result (use gcc_only/clang_only or require_input_*)" 150 fail=$((fail + 1)) ;; 151 "not ok"*"# TODO"*) xfail=$((xfail + 1)) ;; 152 "ok"*"# TODO"*) xpass=$((xpass + 1)) ;; 153 "not ok"*) fail=$((fail + 1)) ;; 154 "ok"*) pass=$((pass + 1)) ;; 155 *) 156 # No result line at all: the test died before reporting. 157 rest="$rest${rest:+$'\n'}exited $rc without a result line" 158 result="not ok - $(basename "$t" .sh): no TAP result" 159 fail=$((fail + 1)) ;; 160 esac 161 162 echo "$result" 163 [ -n "$rest" ] && printf '%s\n' "$rest" | sed 's/^[^#]/# &/' 164 165done 166 167echo "# pass:$pass fail:$fail static-skip:$static_skip" \ 168 "probe-skip:$probe_skip xfail:$xfail xpass:$xpass" 169 170# A failure is the one time the objects matter, and by default they are 171# already gone. Say so then rather than in the usage text nobody reads while 172# something is broken. 173if [ -n "${KLP_TEST_KEEP:-}" ]; then 174 echo "# working directories kept in $rundir -- inspect, then rm -rf it" 175elif ! rmdir "$rundir"/*/ "$rundir" 2>/dev/null; then 176 echo "# $rundir was not empty; a test did not clean up after itself" 177elif [ "$fail" != 0 ] || [ "$xpass" != 0 ]; then 178 echo "# re-run with --keep to hold on to what a failing test built" 179fi 180 181[ "$fail" = 0 ] && [ "$xpass" = 0 ] 182