1#!/usr/local/bin/ruby 2# -------+---------+---------+-------- + --------+---------+---------+---------+ 3# Copyright (c) 2005 - Garance Alistair Drosehn <gad@FreeBSD.org>. 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25# SUCH DAMAGE. 26# -------+---------+---------+-------- + --------+---------+---------+---------+ 27# $FreeBSD$ 28# -------+---------+---------+-------- + --------+---------+---------+---------+ 29# This script was written to provide a battery of regression-tests for some 30# changes I am making to the `env' command. I wrote a new script for this 31# for several reasons. 1) I needed to test all kinds of special-character 32# combinations, and I wanted to be able to type those in exactly as they would 33# would be in real-life situations. 2) I wanted to set environment variables 34# before executing a test, 3) I had many different details to test, so I wanted 35# to write up dozens of tests, without needing to create a hundred separate 36# little tiny files, 4) I wanted to test *failure* conditions, where I expected 37# the test would fail but I wanted to be sure that it failed the way I intended 38# it to fail. 39# This script was written for the special "shebang-line" testing that I 40# wanted for my changes to `env', but I expect it could be turned into a 41# general-purpose test-suite with a little more work. 42# Garance/June 12/2005 43# -------+---------+---------+-------- + --------+---------+---------+---------+ 44 45 46# -------+---------+---------+-------- + --------+---------+---------+---------+ 47class ExpectedResult 48 attr_writer :cmdvalue, :shebang_args, :user_args 49 @@gbl_envs = Hash.new 50 51 def ExpectedResult.add_gblenv(avar, avalue) 52 @@gbl_envs[avar] = avalue 53 end 54 55 def initialize 56 @shebang_args = "" 57 @cmdvalue = 0 58 @clear_envs = Hash.new 59 @new_envs = Hash.new 60 @old_envs = Hash.new 61 @script_lines = "" 62 @expect_err = Array.new 63 @expect_out = Array.new 64 @symlinks = Array.new 65 @user_args = nil 66 end 67 68 def add_expecterr(aline) 69 @expect_err << aline 70 end 71 72 def add_expectout(aline) 73 @expect_out << aline 74 end 75 76 def add_script(aline) 77 @script_lines += aline 78 @script_lines += "\n" if aline[-1] != "\n" 79 end 80 81 def add_clearenv(avar) 82 @clear_envs[avar] = true 83 end 84 85 def add_setenv(avar, avalue) 86 @new_envs[avar] = avalue 87 end 88 89 def add_symlink(srcf, newf) 90 @symlinks << Array.[](srcf, newf) 91 end 92 93 def check_out(name, fname, expect_arr) 94 idx = -1 95 all_matched = true 96 extra_lines = 0 97 rdata = File.open(fname) 98 rdata.each_line { |rline| 99 rline.chomp! 100 idx += 1 101 if idx > expect_arr.length - 1 102 if extra_lines == 0 and $verbose >= 1 103 printf "-- Extra line(s) on %s:\n", name 104 end 105 printf "-- [%d] > %s\n", idx, rline if $verbose >= 1 106 extra_lines += 1 107 elsif rline != expect_arr[idx] 108 if all_matched and $verbose >= 1 109 printf "-- Mismatched line(s) on %s:\n", name 110 end 111 printf "-- [%d] < %s\n", idx, expect_arr[idx] if $verbose >= 2 112 printf "-- > %s\n", rline if $verbose >= 1 113 all_matched = false 114 else 115 printf "-- %s[%d] = %s\n", name, idx, rline if $verbose >= 5 116 end 117 } 118 rdata.close 119 if extra_lines > 0 120 printf "-- %d extra line(s) found on %s\n", extra_lines, 121 name if $verbose == 0 122 return false 123 end 124 if not all_matched 125 printf "-- Mismatched line(s) found on %s\n", 126 name if $verbose == 0 127 return false 128 end 129 return true 130 end 131 132 def create_links 133 @symlinks.each { |fnames| 134 if $verbose >= 2 135 printf "-- Creating: symlink %s %s\n", fnames[0], fnames[1] 136 end 137 symres = File.symlink(fnames[0], fnames[1]) 138 return false if symres == nil 139 return false unless File.symlink?(fnames[1]) 140 } 141 return true 142 end 143 144 def destroy_links 145 @symlinks.each { |fnames| 146 if $verbose >= 2 147 printf "-- Removing: %s (symlink)\n", fnames[1] 148 end 149 if File.symlink?(fnames[1]) 150 if File.delete(fnames[1]) != 1 151 $stderr.printf "Warning: problem removing symlink '%s'\n", 152 fnames[1] 153 end 154 else 155 $stderr.printf "Warning: Symlink '%s' does not exist?!?\n", 156 fnames[1] 157 end 158 } 159 return true 160 end 161 162 def init_io_files 163 @stderr = $scriptfile + ".stderr" 164 @stdout = $scriptfile + ".stdout" 165 File.delete(@stderr) if File.exists?(@stderr) 166 File.delete(@stdout) if File.exists?(@stdout) 167 @stdin = "/dev/null" 168 169 @redirs = " <" + @stdin 170 @redirs += " >" + @stdout 171 @redirs += " 2>" + @stderr 172 173 end 174 175 def pop_envs 176 @new_envs.each_key { |evar| 177 if @old_envs.has_key?(evar) 178 ENV[evar] = @old_envs[evar] 179 else 180 ENV.delete(evar) 181 end 182 } 183 end 184 185 def push_envs 186 @@gbl_envs.each_pair { |evar, eval| 187 ENV[evar] = eval 188 } 189 @new_envs.each_pair { |evar, eval| 190 if ENV.has_key?(evar) 191 @old_envs[evar] = ENV[evar] 192 end 193 ENV[evar] = eval 194 } 195 end 196 197 def run_test 198 tscript = File.open($scriptfile, "w") 199 tscript.printf "#!%s", $testpgm 200 tscript.printf " %s", @shebang_args if @shebang_args != "" 201 tscript.printf "\n" 202 tscript.printf "%s", @script_lines if @script_lines != "" 203 tscript.close 204 File.chmod(0755, $scriptfile) 205 206 usercmd = $scriptfile 207 usercmd += " " + @user_args if @user_args != nil 208 init_io_files 209 210 push_envs 211 return 0 unless create_links 212 printf "- Executing: %s\n", usercmd if $verbose >= 1 213 printf "----- with: %s\n", @redirs if $verbose >= 6 214 sys_ok = system(usercmd + @redirs) 215 if sys_ok 216 @sav_cmdvalue = 0 217 elsif $?.exited? 218 @sav_cmdvalue = $?.exitstatus 219 else 220 @sav_cmdvalue = 125 221 end 222 destroy_links 223 pop_envs 224 sys_ok = true 225 if @sav_cmdvalue != @cmdvalue 226 printf "-- Expecting cmdvalue of %d, but $? == %d\n", @cmdvalue, 227 @sav_cmdvalue 228 sys_ok = false 229 end 230 sys_ok = false unless check_out("stdout", @stdout, @expect_out) 231 sys_ok = false unless check_out("stderr", @stderr, @expect_err) 232 return 1 if sys_ok 233 return 0 234 end 235end 236 237# -------+---------+---------+-------- + --------+---------+---------+---------+ 238# Processing of the command-line options given to the regress-sb.rb script. 239# 240class CommandOptions 241 def CommandOptions.parse(command_args) 242 parse_ok = true 243 command_args.each { |userarg| 244 case userarg 245 when /^--rgdata=(\S+)$/ 246 parse_ok = false unless set_rgdatafile($1) 247 when /^--testpgm=(\S+)$/ 248 parse_ok = false unless set_testpgm($1) 249 $cmdopt_testpgm = $testpgm 250 when "--stop-on-error", "--stop_on_error" 251 $stop_on_error = true 252 when /^--/ 253 $stderr.printf "Error: Invalid long option: %s\n", userarg 254 parse_ok = false 255 when /^-/ 256 userarg = userarg[1...userarg.length] 257 userarg.each_byte { |byte| 258 char = byte.chr 259 case char 260 when "v" 261 $verbose += 1 262 else 263 $stderr.printf "Error: Invalid short option: -%s\n", char 264 parse_ok = false 265 end 266 } 267 else 268 $stderr.printf "Error: Invalid request: %s\n", userarg 269 parse_ok = false 270 end 271 } 272 if $rgdatafile == nil 273 rgmatch = Dir.glob("regress*.rgdata") 274 if rgmatch.length == 1 275 $rgdatafile = rgmatch[0] 276 printf "Assuming --rgdata=%s\n", $rgdatafile 277 else 278 $stderr.printf "Error: The --rgdata file was not specified\n" 279 parse_ok = false 280 end 281 end 282 return parse_ok 283 end 284 285 def CommandOptions.set_rgdatafile(fname) 286 if not File.exists?(fname) 287 $stderr.printf "Error: Rgdata file '%s' does not exist\n", fname 288 return false 289 elsif not File.readable?(fname) 290 $stderr.printf "Error: Rgdata file '%s' is not readable\n", fname 291 return false 292 end 293 $rgdatafile = File.expand_path(fname) 294 return true 295 end 296 297 def CommandOptions.set_testpgm(fname) 298 if not File.exists?(fname) 299 $stderr.printf "Error: Testpgm file '%s' does not exist\n", fname 300 return false 301 elsif not File.executable?(fname) 302 $stderr.printf "Error: Testpgm file '%s' is not executable\n", fname 303 return false 304 end 305 $testpgm = File.expand_path(fname) 306 return true 307 end 308end 309 310# -------+---------+---------+-------- + --------+---------+---------+---------+ 311# Processing of the test-specific options specifed in each [test]/[run] 312# section of the regression-data file. This will set values in the 313# global $testdata object. 314# 315class RGTestOptions 316 @@rgtest_opts = nil; 317 318 def RGTestOptions.init_rgtopts 319 @@rgtest_opts = Hash.new 320 @@rgtest_opts["$?"] = true 321 @@rgtest_opts["clearenv"] = true 322 @@rgtest_opts["sb_args"] = true 323 @@rgtest_opts["script"] = true 324 @@rgtest_opts["setenv"] = true 325 @@rgtest_opts["stderr"] = true 326 @@rgtest_opts["stdout"] = true 327 @@rgtest_opts["symlink"] = true 328 @@rgtest_opts["user_args"] = true 329 end 330 331 def RGTestOptions.parse(optname, optval) 332 init_rgtopts unless @@rgtest_opts 333 334 if not @@rgtest_opts.has_key?(optname) 335 $stderr.printf "Error: Invalid test-option in rgdata file: %s\n", 336 optname 337 return false 338 end 339 340 # Support a few very specific substitutions in values specified 341 # for test data. Format of all recognized values should be: 342 # [%-object.value-%] 343 # which is hopefully distinctive-enough that they will never 344 # conflict with any naturally-occurring string. Also note that 345 # we only match the specific values that we recognize, and not 346 # "just anything" that matches the general pattern. There are 347 # no blanks in the recognized values, but I use an x-tended 348 # regexp and then add blanks to make it more readable. 349 optval.gsub!(/\[%- testpgm\.basename -%\]/x, File.basename($testpgm)) 350 optval.gsub!(/\[%- script\.pathname -%\]/x, $scriptfile) 351 352 invalid_value = false 353 case optname 354 when "$?" 355 if optval =~ /^\d+$/ 356 $testdata.cmdvalue = optval.to_i 357 else 358 invalid_value = true 359 end 360 when "clearenv" 361 if optval =~ /^\s*([A-Za-z]\w*)\s*$/ 362 $testdata.add_clearenv($1) 363 else 364 invalid_value = true 365 end 366 when "sb_args" 367 $testdata.shebang_args = optval 368 when "script" 369 $testdata.add_script(optval) 370 when "setenv" 371 if optval =~ /^\s*([A-Za-z]\w*)=(.*)$/ 372 $testdata.add_setenv($1, $2) 373 else 374 invalid_value = true 375 end 376 when "stderr" 377 $testdata.add_expecterr(optval) 378 when "stdout" 379 $testdata.add_expectout(optval) 380 when "symlink" 381 if optval =~ /^\s*(\S+)\s+(\S+)\s*$/ 382 srcfile = $1 383 newfile = $2 384 if not File.exists?(srcfile) 385 $stderr.printf "Error: source file '%s' does not exist.\n", 386 srcfile 387 invalid_value = true 388 elsif File.exists?(newfile) 389 $stderr.printf "Error: new file '%s' already exists.\n", 390 newfile 391 invalid_value = true 392 else 393 $testdata.add_symlink(srcfile, newfile) 394 end 395 else 396 invalid_value = true 397 end 398 when "user_args" 399 $testdata.user_args = optval 400 else 401 $stderr.printf "InternalError: Invalid test-option in rgdata file: %s\n", 402 optname 403 return false 404 end 405 406 if invalid_value 407 $stderr.printf "Error: Invalid value(s) for %s: %s\n", 408 optname, optval 409 return false 410 end 411 return true 412 end 413end 414 415# -------+---------+---------+-------- + --------+---------+---------+---------+ 416# Here's where the "main" routine begins... 417# 418 419$cmdopt_testpgm = nil 420$testpgm = nil 421$rgdatafile = nil 422$scriptfile = "/tmp/env-regress" 423$stop_on_error = false 424$verbose = 0 425 426exit 1 unless CommandOptions.parse(ARGV) 427 428errline = nil 429test_count = 0 430testok_count = 0 431test_lineno = -1 432max_test = -1 433regress_data = File.open($rgdatafile) 434regress_data.each_line { |dline| 435 case dline 436 when /^\s*#/, /^\s*$/ 437 # Just a comment line, ignore it. 438 when /^\s*gblenv=\s*(.+)$/ 439 if test_lineno > 0 440 $stderr.printf "Error: Cannot define a global-value in the middle of a test (#5d)\n", test_lineno 441 errline = regress_data.lineno 442 break; 443 end 444 tempval = $1 445 if tempval !~ /^([A-Za-z]\w*)=(.*)$/ 446 $stderr.printf "Error: Invalid value for 'gblenv=' request: %s\n", 447 tempval 448 errline = regress_data.lineno 449 break; 450 end 451 ExpectedResult.add_gblenv($1, $2) 452 when /^testpgm=\s*(\S+)\s*/ 453 # Set the location of the program to be tested, if it wasn't set 454 # on the command-line processing. 455 if $cmdopt_testpgm == nil 456 if not CommandOptions.set_testpgm($1) 457 errline = regress_data.lineno 458 break; 459 end 460 end 461 when /^\[test\]$/ 462 if test_lineno > 0 463 $stderr.printf "Error: Request to define a [test], but we are still defining\n" 464 $stderr.printf " the [test] at line #%s\n", test_lineno 465 errline = regress_data.lineno 466 break; 467 end 468 test_lineno = regress_data.lineno 469 max_test = test_lineno 470 printf "- Defining test at line #%s\n", test_lineno if $verbose >= 6 471 $testdata = ExpectedResult.new 472 when /^\[end\]$/ 473 # User wants us to ignore the remainder of the rgdata file... 474 break; 475 when /^\[run\]$/ 476 if test_lineno < 0 477 $stderr.printf "Error: Request to [run] a test, but no test is presently defined\n" 478 errline = regress_data.lineno 479 break; 480 end 481 printf "- Running test at line #%s\n", test_lineno if $verbose >= 1 482 run_result = $testdata.run_test 483 test_count += 1 484 printf "[Test #%3d: ", test_count 485 case run_result 486 when 0 487 # Test failed 488 printf "Failed! (line %4d)]\n", test_lineno 489 break if $stop_on_error 490 when 1 491 # Test ran as expected 492 testok_count += 1 493 printf "OK]\n" 494 else 495 # Internal error of some sort 496 printf "InternalError! (line %4d)]\n", test_lineno 497 errline = regress_data.lineno 498 break; 499 end 500 test_lineno = -1 501 502 when /^(\s*)([^\s:]+)\s*:(.+)$/ 503 blankpfx = $1 504 test_lhs = $2 505 test_rhs = $3 506 if test_lineno < 0 507 $stderr.printf "Error: No test is presently being defined\n" 508 errline = regress_data.lineno 509 break; 510 end 511 # All the real work happens in RGTestOptions.parse 512 if not RGTestOptions.parse(test_lhs, test_rhs) 513 errline = regress_data.lineno 514 break; 515 end 516 if blankpfx.length == 0 517 $stderr.printf "Note: You should at least one blank before:%s\n", 518 dline.chomp 519 $stderr.printf " at line %d of rgdata file %s\n", 520 regress_data.lineno, $rgdatafile 521 end 522 523 else 524 $stderr.printf "Error: Invalid line: %s\n", dline.chomp 525 errline = regress_data.lineno 526 break; 527 end 528} 529regress_data.close 530if errline != nil 531 $stderr.printf " at line %d of rgdata file %s\n", errline, $rgdatafile 532 exit 2 533end 534if testok_count != test_count 535 printf "%d of %d tests were successful.\n", testok_count, test_count 536 exit 1 537end 538 539printf "All %d tests were successful!\n", testok_count 540exit 0 541