1#!/bin/sh 2 3#set -x 4 5cd $(dirname "$0") 6 7default_path=".." 8default_cmd="sparse \$file" 9default_args="$SPARSE_TEST_ARGS" 10tests_list="" 11prog_name=`basename $0` 12 13if [ ! -x "$default_path/sparse-llvm" ]; then 14 disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis" 15fi 16 17# flags: 18# - some tests gave an unexpected result 19failed=0 20 21# counts: 22# - tests that have not been converted to test-suite format 23# - tests that are disabled 24# - tests that passed 25# - tests that failed 26# - tests that failed but are known to fail 27unhandled_tests=0 28disabled_tests=0 29ok_tests=0 30ko_tests=0 31known_ko_tests=0 32 33# defaults to not verbose 34[ -z "$V" ] && V=0 35vquiet="" 36quiet=0 37abort=0 38 39 40## 41# verbose(string) - prints string if we are in verbose mode 42verbose() 43{ 44 [ "$V" -eq "1" ] && echo " $1" 45 return 0 46} 47 48## 49# warning(string) - prints a warning 50warning() 51{ 52 [ "$quiet" -ne 1 ] && echo "warning: $1" 53 return 0 54} 55 56## 57# error(string[, die]) - prints an error and exits with value die if given 58error() 59{ 60 [ "$quiet" -ne 1 ] && echo "error: $1" 61 [ -n "$2" ] && exit $2 62 return 0 63} 64 65 66## 67# get_tag_value(file) - get the 'check-<...>' tags & values 68get_tag_value() 69{ 70 check_name="" 71 check_command="$default_cmd" 72 check_exit_value=0 73 check_timeout=0 74 check_known_to_fail=0 75 check_error_ignore=0 76 check_output_ignore=0 77 check_output_contains=0 78 check_output_excludes=0 79 check_output_pattern=0 80 check_arch_ignore="" 81 check_arch_only="" 82 check_assert="" 83 check_cpp_if="" 84 85 lines=$(grep 'check-[a-z-]*' $1 | \ 86 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/') 87 88 while read tag val; do 89 #echo "-> tag: '$tag'" 90 #echo "-> val: '$val'" 91 case $tag in 92 check-name:) check_name="$val" ;; 93 check-command:) check_command="$val" ;; 94 check-exit-value:) check_exit_value="$val" ;; 95 check-timeout:) [ -z "$val" ] && val=1 96 check_timeout="$val" ;; 97 check-known-to-fail) check_known_to_fail=1 ;; 98 check-error-ignore) check_error_ignore=1 ;; 99 check-output-ignore) check_output_ignore=1 ;; 100 check-output-contains:) check_output_contains=1 ;; 101 check-output-excludes:) check_output_excludes=1 ;; 102 check-output-pattern) check_output_pattern=1 ;; 103 check-arch-ignore:) arch=$(uname -m) 104 check_arch_ignore="$val" ;; 105 check-arch-only:) arch=$(uname -m) 106 check_arch_only="$val" ;; 107 check-assert:) check_assert="$val" ;; 108 check-cpp-if:) check_cpp_if="$val" ;; 109 110 check-description:) ;; # ignore 111 check-note:) ;; # ignore 112 check-warning:) ;; # ignore 113 check-error-start) ;; # ignore 114 check-error-end) ;; # ignore 115 check-output-start) ;; # ignore 116 check-output-end) ;; # ignore 117 check-should-pass) ;; # ignore, unused annotation 118 check-should-fail) ;; # ignore, unused annotation 119 check-should-warn) ;; # ignore, unused annotation 120 check-*) error "$1: unknown tag '$tag'" 1 ;; 121 esac 122 done << EOT 123 $lines 124EOT 125} 126 127## 128# helper for has_(each|none)_patterns() 129has_patterns() 130{ 131 ifile="$1" 132 patt="$2" 133 ofile="$3" 134 cmp="$4" 135 msg="$5" 136 grep "$patt:" "$ifile" | \ 137 sed -e "s/^.*$patt: *\(.*\)$/\1/" | \ 138 while read val; do 139 grep -s -q "$val" "$ofile" 140 if [ "$?" $cmp 0 ]; then 141 error " Pattern '$val' unexpectedly $msg" 142 return 1 143 fi 144 done 145 146 return $? 147} 148 149## 150# has_each_patterns(ifile tag ofile) - does ofile contains some 151# of the patterns given by ifile's tags? 152# 153# returns 0 if all present, 1 otherwise 154has_each_patterns() 155{ 156 has_patterns "$1" "$2" "$4" -ne "$3" 157} 158 159## 160# has_none_patterns(ifile tag ofile) - does ofile contains some 161# of the patterns given by ifile's tags? 162# 163# returns 1 if any present, 0 otherwise 164has_none_patterns() 165{ 166 has_patterns "$1" "$2" "$4" -eq "$3" 167} 168 169## 170# minmax_patterns(ifile tag ofile) - does ofile contains the 171# the patterns given by ifile's tags 172# the right number of time? 173minmax_patterns() 174{ 175 ifile="$1" 176 patt="$2" 177 ofile="$3" 178 grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \ 179 sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \ 180 -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \ 181 while read min max pat; do 182 n=$(grep -s "$pat" "$ofile" | wc -l) 183 if [ "$max" = "eq" ]; then 184 if [ "$n" -ne "$min" ]; then 185 error " Pattern '$pat' expected $min times but got $n times" 186 return 1 187 fi 188 continue 189 fi 190 if [ "$min" != '-' ]; then 191 if [ "$n" -lt "$min" ]; then 192 error " Pattern '$pat' expected min $min times but got $n times" 193 return 1 194 fi 195 fi 196 if [ "$max" != '-' ]; then 197 if [ "$n" -gt "$max" ]; then 198 error " Pattern '$pat' expected max $max times but got $n times" 199 return 1 200 fi 201 fi 202 done 203 204 return $? 205} 206 207## 208# arg_file(filename) - checks if filename exists 209arg_file() 210{ 211 [ -z "$1" ] && { 212 do_usage 213 exit 1 214 } 215 [ -e "$1" ] || { 216 error "Can't open file $1" 217 exit 1 218 } 219 return 0 220} 221 222 223## 224do_usage() 225{ 226echo "$prog_name - a tiny automatic testing script" 227echo "Usage: $prog_name [option(s)] [command] [arguments]" 228echo 229echo "options:" 230echo " -a|--abort Abort the tests as soon as one fails." 231echo " -q|--quiet Be extra quiet while running the tests." 232echo " --args='...' Add these options to the test command." 233echo 234echo "commands:" 235echo " [file ...] Runs the test suite on the given file(s)." 236echo " If a directory is given, run only those files." 237echo " If no file is given, run the whole testsuite." 238echo " single file Run the test in 'file'." 239echo " format file [name [cmd]] Help writing a new test case using cmd." 240echo 241echo " [command] help Print usage." 242} 243 244disable() 245{ 246 disabled_tests=$(($disabled_tests + 1)) 247 if [ -z "$vquiet" ]; then 248 echo " SKIP $1 ($2)" 249 fi 250} 251 252## 253# do_test(file) - tries to validate a test case 254# 255# it "parses" file, looking for check-* tags and tries to validate 256# the test against an expected result 257# returns: 258# - 0 if the test passed, 259# - 1 if it failed, 260# - 2 if it is not a "test-suite" test. 261# - 3 if the test is disabled. 262do_test() 263{ 264 test_failed=0 265 file="$1" 266 quiet=0 267 268 get_tag_value $file 269 270 # can this test be handled by test-suite ? 271 # (it has to have a check-name key in it) 272 if [ "$check_name" = "" ]; then 273 warning "$file: test unhandled" 274 unhandled_tests=$(($unhandled_tests + 1)) 275 return 2 276 fi 277 test_name="$check_name" 278 279 # does the test provide a specific command ? 280 if [ "$check_command" = "" ]; then 281 check_command="$defaut_command" 282 fi 283 284 # check for disabled commands 285 set -- $check_command 286 base_cmd=$1 287 for i in $disabled_cmds; do 288 if [ "$i" = "$base_cmd" ] ; then 289 disable "$test_name" "$file" 290 return 3 291 fi 292 done 293 if [ "$check_arch_ignore" != "" ]; then 294 if echo $arch | egrep -q -w "$check_arch_ignore"; then 295 disable "$test_name" "$file" 296 return 3 297 fi 298 fi 299 if [ "$check_arch_only" != "" ]; then 300 if ! (echo $arch | egrep -q -w "$check_arch_only"); then 301 disable "$test_name" "$file" 302 return 3 303 fi 304 fi 305 if [ "$check_assert" != "" ]; then 306 res=$(../sparse - 2>&1 >/dev/null <<- EOF 307 _Static_assert($check_assert, "$check_assert"); 308 EOF 309 ) 310 if [ "$res" != "" ]; then 311 disable "$test_name" "$file" 312 return 3 313 fi 314 fi 315 if [ "$check_cpp_if" != "" ]; then 316 res=$(../sparse -E - 2>/dev/null <<- EOF 317 #if !($check_cpp_if) 318 fail 319 #endif 320 EOF 321 ) 322 if [ "$res" != "" ]; then 323 disable "$test_name" "$file" 324 return 3 325 fi 326 fi 327 328 if [ -z "$vquiet" ]; then 329 echo " TEST $test_name ($file)" 330 fi 331 332 verbose "Using command : $(echo "$@")" 333 334 # grab the expected exit value 335 expected_exit_value=$check_exit_value 336 verbose "Expecting exit value: $expected_exit_value" 337 338 # do we want a timeout? 339 pre_cmd="" 340 if [ $check_timeout -ne 0 ]; then 341 pre_cmd="timeout -k 1s $check_timeout" 342 fi 343 344 shift 345 # launch the test command and 346 # grab the actual output & exit value 347 eval $pre_cmd $default_path/$base_cmd $default_args "$@" \ 348 1> $file.output.got 2> $file.error.got 349 actual_exit_value=$? 350 351 must_fail=$check_known_to_fail 352 [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1 353 known_ko_tests=$(($known_ko_tests + $must_fail)) 354 355 for stream in error output; do 356 eval ignore=\$check_${stream}_ignore 357 [ $ignore -eq 1 ] && continue 358 359 # grab the expected output 360 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \ 361 | grep -v check-$stream > "$file".$stream.expected 362 363 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff 364 if [ "$?" -ne "0" ]; then 365 error "actual $stream text does not match expected $stream text." 366 error "see $file.$stream.* for further investigation." 367 [ $quiet -ne 1 ] && cat "$file".$stream.diff 368 test_failed=1 369 fi 370 done 371 372 if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then 373 error "Actual exit value does not match the expected one." 374 error "expected $expected_exit_value, got $actual_exit_value." 375 test_failed=1 376 fi 377 378 # verify the 'check-output-contains/excludes' tags 379 if [ $check_output_contains -eq 1 ]; then 380 has_each_patterns "$file" 'check-output-contains' absent $file.output.got 381 if [ "$?" -ne "0" ]; then 382 test_failed=1 383 fi 384 fi 385 if [ $check_output_excludes -eq 1 ]; then 386 has_none_patterns "$file" 'check-output-excludes' present $file.output.got 387 if [ "$?" -ne "0" ]; then 388 test_failed=1 389 fi 390 fi 391 if [ $check_output_pattern -eq 1 ]; then 392 # verify the 'check-output-pattern(...)' tags 393 minmax_patterns "$file" 'check-output-pattern' $file.output.got 394 if [ "$?" -ne "0" ]; then 395 test_failed=1 396 fi 397 fi 398 399 if [ "$must_fail" -eq "1" ]; then 400 if [ "$test_failed" -eq "1" ]; then 401 [ -z "$vquiet" ] && \ 402 echo "info: XFAIL: test '$file' is known to fail" 403 else 404 echo "error: XPASS: test '$file' is known to fail but succeed!" 405 fi 406 else 407 if [ "$test_failed" -eq "1" ]; then 408 echo "error: FAIL: test '$file' failed" 409 else 410 [ "$V" -ne "0" ] && \ 411 echo "info: PASS: test '$file' passed" 412 fi 413 fi 414 415 if [ "$test_failed" -ne "$must_fail" ]; then 416 [ $abort -eq 1 ] && exit 1 417 test_failed=1 418 failed=1 419 fi 420 421 if [ "$test_failed" -eq "1" ]; then 422 ko_tests=$(($ko_tests + 1)) 423 else 424 ok_tests=$(($ok_tests + 1)) 425 rm -f $file.{error,output}.{expected,got,diff} 426 fi 427 return $test_failed 428} 429 430do_test_suite() 431{ 432 for i in $tests_list; do 433 do_test "$i" 434 done 435 436 OK=OK 437 [ $failed -eq 0 ] || OK=KO 438 439 # prints some numbers 440 tests_nr=$(($ok_tests + $ko_tests)) 441 echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed" 442 if [ "$known_ko_tests" -ne 0 ]; then 443 echo " $known_ko_tests of them are known to fail" 444 fi 445 if [ "$unhandled_tests" -ne "0" ]; then 446 echo " $unhandled_tests tests could not be handled by $prog_name" 447 fi 448 if [ "$disabled_tests" -ne "0" ]; then 449 echo " $disabled_tests tests were disabled" 450 fi 451} 452 453## 454do_format_help() { 455echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]" 456echo 457echo "options:" 458echo " -a append the created test to the input file" 459echo " -f write a test known to fail" 460echo " -l write a test for linearized code" 461echo 462echo "argument(s):" 463echo " file file containing the test case(s)" 464echo " name name for the test case (defaults to file)" 465echo " cmd command to be used (defaults to 'sparse \$file')" 466} 467 468## 469# do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags 470do_format() 471{ 472 def_cmd="$default_cmd" 473 append=0 474 linear=0 475 fail=0 476 477 while [ $# -gt 1 ] ; do 478 case "$1" in 479 -a) 480 append=1 ;; 481 -f) 482 fail=1 ;; 483 -l) 484 def_cmd='test-linearize -Wno-decl $file' 485 linear=1 ;; 486 help|-*) 487 do_format_help 488 return 0 489 ;; 490 *) break ;; 491 esac 492 shift 493 continue 494 done 495 496 arg_file "$1" || return 1 497 498 file="$1" 499 fname="$2" 500 [ -z "$fname" ] && fname="$(basename "$1" .c)" 501 fcmd="$3" 502 [ -z "$fcmd" ] && fcmd="$def_cmd" 503 504 cmd=`eval echo $default_path/$fcmd` 505 $cmd 1> $file.output.got 2> $file.error.got 506 fexit_value=$? 507 [ $append != 0 ] && exec >> $file 508 cat <<_EOF 509 510/* 511 * check-name: $fname 512_EOF 513 if [ "$fcmd" != "$default_cmd" ]; then 514 echo " * check-command: $fcmd" 515 fi 516 if [ "$fexit_value" -ne "0" ]; then 517 echo " * check-exit-value: $fexit_value" 518 fi 519 if [ $fail != 0 ]; then 520 echo " * check-known-to-fail" 521 fi 522 if [ $linear != 0 ]; then 523 echo ' *' 524 echo ' * check-output-ignore' 525 echo ' * check-output-contains: xyz\\\\.' 526 echo ' * check-output-excludes: \\\\.' 527 fi 528 for stream in output error; do 529 if [ -s "$file.$stream.got" ]; then 530 echo " *" 531 echo " * check-$stream-start" 532 cat "$file.$stream.got" 533 echo " * check-$stream-end" 534 fi 535 done 536 echo " */" 537 return 0 538} 539 540## allow flags from environment 541set -- $SPARSE_TEST_FLAGS "$@" 542 543## process the flags 544while [ "$#" -gt "0" ]; do 545 case "$1" in 546 -a|--abort) 547 abort=1 548 ;; 549 -q|--quiet) 550 vquiet=1 551 ;; 552 --args=*) 553 default_args="${1#--args=}"; 554 ;; 555 556 single|--single) 557 arg_file "$2" 558 do_test "$2" 559 case "$?" in 560 0) echo "$2 passed !";; 561 1) echo "$2 failed !";; 562 2) echo "$2 can't be handled by $prog_name";; 563 esac 564 exit $failed 565 ;; 566 format|--format) 567 shift 568 do_format "$@" 569 exit 0 570 ;; 571 help) 572 do_usage 573 exit 1 574 ;; 575 576 *.c|*.cdoc) 577 tests_list="$tests_list $1" 578 ;; 579 *) 580 if [ ! -d "$1" ]; then 581 do_usage 582 exit 1 583 fi 584 tests_list="$tests_list $(find "$1" -name '*.c' | sort)" 585 ;; 586 esac 587 shift 588done 589 590if [ -z "$tests_list" ]; then 591 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort` 592fi 593 594do_test_suite 595exit $failed 596 597