1#!/usr/local/bin/ruby 2# -------+---------+---------+-------- + --------+---------+---------+---------+ 3# Copyright (c) 2005 - Garance Alistair Drosehn <gad@FreeBSD.org>. 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25# SUCH DAMAGE. 26# -------+---------+---------+-------- + --------+---------+---------+---------+ 27# $FreeBSD$ 28# -------+---------+---------+-------- + --------+---------+---------+---------+ 29# This script was written to provide a battery of regression-tests for some 30# changes I am making to the `env' command. I wrote a new script for this 31# for several reasons. 1) I needed to test all kinds of special-character 32# combinations, and I wanted to be able to type those in exactly as they would 33# would be in real-life situations. 2) I wanted to set environment variables 34# before executing a test, 3) I had many different details to test, so I wanted 35# to write up dozens of tests, without needing to create a hundred separate 36# little tiny files, 4) I wanted to test *failure* conditions, where I expected 37# the test would fail but I wanted to be sure that it failed the way I intended 38# it to fail. 39# This script was written for the special "shebang-line" testing that I 40# wanted for my changes to `env', but I expect it could be turned into a 41# general-purpose test-suite with a little more work. 42# Garance/June 12/2005 43# -------+---------+---------+-------- + --------+---------+---------+---------+ 44 45 46# -------+---------+---------+-------- + --------+---------+---------+---------+ 47class ExpectedResult 48 attr_writer :cmdvalue, :shebang_args, :user_args 49 @@gbl_envs = Hash.new 50 51 def ExpectedResult.add_gblenv(avar, avalue) 52 @@gbl_envs[avar] = avalue 53 end 54 55 def initialize 56 @shebang_args = "" 57 @cmdvalue = 0 58 @clear_envs = Hash.new 59 @new_envs = Hash.new 60 @old_envs = Hash.new 61 @script_lines = "" 62 @expect_err = Array.new 63 @expect_out = Array.new 64 @symlinks = Array.new 65 @user_args = nil 66 end 67 68 def add_expecterr(aline) 69 @expect_err << aline 70 end 71 72 def add_expectout(aline) 73 @expect_out << aline 74 end 75 76 def add_script(aline) 77 @script_lines += aline 78 @script_lines += "\n" if aline[-1] != "\n" 79 end 80 81 def add_clearenv(avar) 82 @clear_envs[avar] = true 83 end 84 85 def add_setenv(avar, avalue) 86 @new_envs[avar] = avalue 87 end 88 89 def add_symlink(srcf, newf) 90 @symlinks << Array.[](srcf, newf) 91 end 92 93 def check_out(name, fname, expect_arr) 94 idx = -1 95 all_matched = true 96 extra_lines = 0 97 rdata = File.open(fname) 98 rdata.each_line { |rline| 99 rline.chomp! 100 idx += 1 101 if idx > expect_arr.length - 1 102 if extra_lines == 0 and $verbose >= 1 103 printf "-- Extra line(s) on %s:\n", name 104 end 105 printf "-- [%d] > %s\n", idx, rline if $verbose >= 1 106 extra_lines += 1 107 elsif rline != expect_arr[idx] 108 if all_matched and $verbose >= 1 109 printf "-- Mismatched line(s) on %s:\n", name 110 end 111 printf "-- [%d] < %s\n", idx, expect_arr[idx] if $verbose >= 2 112 printf "-- > %s\n", rline if $verbose >= 1 113 all_matched = false 114 else 115 printf "-- %s[%d] = %s\n", name, idx, rline if $verbose >= 5 116 end 117 } 118 rdata.close 119 if extra_lines > 0 120 printf "-- %d extra line(s) found on %s\n", extra_lines, 121 name if $verbose == 0 122 return false 123 end 124 if not all_matched 125 printf "-- Mismatched line(s) found on %s\n", 126 name if $verbose == 0 127 return false 128 end 129 return true 130 end 131 132 def create_links 133 @symlinks.each { |fnames| 134 if $verbose >= 2 135 printf "-- Creating: symlink %s %s\n", fnames[0], fnames[1] 136 end 137 symres = File.symlink(fnames[0], fnames[1]) 138 return false if symres == nil 139 return false unless File.symlink?(fnames[1]) 140 } 141 return true 142 end 143 144 def destroy_links 145 @symlinks.each { |fnames| 146 if $verbose >= 2 147 printf "-- Removing: %s (symlink)\n", fnames[1] 148 end 149 if File.symlink?(fnames[1]) 150 if File.delete(fnames[1]) != 1 151 $stderr.printf "Warning: problem removing symlink '%s'\n", 152 fnames[1] 153 end 154 else 155 $stderr.printf "Warning: Symlink '%s' does not exist?!?\n", 156 fnames[1] 157 end 158 } 159 return true 160 end 161 162 def init_io_files 163 @stderr = $scriptfile + ".stderr" 164 @stdout = $scriptfile + ".stdout" 165 File.delete(@stderr) if File.exists?(@stderr) 166 File.delete(@stdout) if File.exists?(@stdout) 167 @stdin = "/dev/null" 168 169 @redirs = " <" + @stdin 170 @redirs += " >" + @stdout 171 @redirs += " 2>" + @stderr 172 173 end 174 175 def pop_envs 176 @new_envs.each_key { |evar| 177 if @old_envs.has_key?(evar) 178 ENV[evar] = @old_envs[evar] 179 else 180 ENV.delete(evar) 181 end 182 } 183 end 184 185 def push_envs 186 @@gbl_envs.each_pair { |evar, eval| 187 ENV[evar] = eval 188 } 189 @new_envs.each_pair { |evar, eval| 190 if ENV.has_key?(evar) 191 @old_envs[evar] = ENV[evar] 192 end 193 ENV[evar] = eval 194 } 195 end 196 197 def run_test 198 tscript = File.open($scriptfile, "w") 199 tscript.printf "#!%s", $testpgm 200 tscript.printf " %s", @shebang_args if @shebang_args != "" 201 tscript.printf "\n" 202 tscript.printf "%s", @script_lines if @script_lines != "" 203 tscript.close 204 File.chmod(0755, $scriptfile) 205 206 usercmd = $scriptfile 207 usercmd += " " + @user_args if @user_args != nil 208 init_io_files 209 210 push_envs 211 return 0 unless create_links 212 printf "- Executing: %s\n", usercmd if $verbose >= 1 213 printf "----- with: %s\n", @redirs if $verbose >= 6 214 sys_ok = system(usercmd + @redirs) 215 if sys_ok 216 @sav_cmdvalue = 0 217 elsif $?.exited? 218 @sav_cmdvalue = $?.exitstatus 219 else 220 @sav_cmdvalue = 125 221 end 222 destroy_links 223 pop_envs 224 sys_ok = true 225 if @sav_cmdvalue != @cmdvalue 226 printf "-- Expecting cmdvalue of %d, but $? == %d\n", @cmdvalue, 227 @sav_cmdvalue 228 sys_ok = false 229 end 230 sys_ok = false unless check_out("stdout", @stdout, @expect_out) 231 sys_ok = false unless check_out("stderr", @stderr, @expect_err) 232 return 1 if sys_ok 233 return 0 234 end 235end 236 237# -------+---------+---------+-------- + --------+---------+---------+---------+ 238# Processing of the command-line options given to the regress-sb.rb script. 239# 240class CommandOptions 241 def CommandOptions.parse(command_args) 242 parse_ok = true 243 command_args.each { |userarg| 244 case userarg 245 when /^--rgdata=(\S+)$/ 246 parse_ok = false unless set_rgdatafile($1) 247 when /^--testpgm=(\S+)$/ 248 parse_ok = false unless set_testpgm($1) 249 $cmdopt_testpgm = $testpgm 250 when "--stop-on-error", "--stop_on_error" 251 $stop_on_error = true 252 when /^--/ 253 $stderr.printf "Error: Invalid long option: %s\n", userarg 254 parse_ok = false 255 when /^-/ 256 userarg = userarg[1...userarg.length] 257 userarg.each_byte { |byte| 258 char = byte.chr 259 case char 260 when "v" 261 $verbose += 1 262 else 263 $stderr.printf "Error: Invalid short option: -%s\n", char 264 parse_ok = false 265 end 266 } 267 else 268 $stderr.printf "Error: Invalid request: %s\n", userarg 269 parse_ok = false 270 end 271 } 272 if $rgdatafile == nil 273 rgmatch = Dir.glob("regress*.rgdata") 274 if rgmatch.length == 1 275 $rgdatafile = rgmatch[0] 276 printf "Assuming --rgdata=%s\n", $rgdatafile 277 else 278 $stderr.printf "Error: The --rgdata file was not specified\n" 279 parse_ok = false 280 end 281 end 282 return parse_ok 283 end 284 285 def CommandOptions.set_rgdatafile(fname) 286 if not File.exists?(fname) 287 $stderr.printf "Error: Rgdata file '%s' does not exist\n", fname 288 return false 289 elsif not File.readable?(fname) 290 $stderr.printf "Error: Rgdata file '%s' is not readable\n", fname 291 return false 292 end 293 $rgdatafile = File.expand_path(fname) 294 return true 295 end 296 297 def CommandOptions.set_testpgm(fname) 298 if not File.exists?(fname) 299 $stderr.printf "Error: Testpgm file '%s' does not exist\n", fname 300 return false 301 elsif not File.executable?(fname) 302 $stderr.printf "Error: Testpgm file '%s' is not executable\n", fname 303 return false 304 end 305 $testpgm = File.expand_path(fname) 306 return true 307 end 308end 309 310# -------+---------+---------+-------- + --------+---------+---------+---------+ 311# Processing of the test-specific options specifed in each [test]/[run] 312# section of the regression-data file. This will set values in the 313# global $testdata object. 314# 315class RGTestOptions 316 @@rgtest_opts = nil; 317 318 def RGTestOptions.init_rgtopts 319 @@rgtest_opts = Hash.new 320 @@rgtest_opts["$?"] = true 321 @@rgtest_opts["clearenv"] = true 322 @@rgtest_opts["sb_args"] = true 323 @@rgtest_opts["script"] = true 324 @@rgtest_opts["setenv"] = true 325 @@rgtest_opts["stderr"] = true 326 @@rgtest_opts["stdout"] = true 327 @@rgtest_opts["symlink"] = true 328 @@rgtest_opts["user_args"] = true 329 end 330 331 def RGTestOptions.parse(optname, optval) 332 init_rgtopts unless @@rgtest_opts 333 334 if not @@rgtest_opts.has_key?(optname) 335 $stderr.printf "Error: Invalid test-option in rgdata file: %s\n", 336 optname 337 return false 338 end 339 340 # Support a few very specific substitutions in values specified 341 # for test data. Format of all recognized values should be: 342 # [%-object.value-%] 343 # which is hopefully distinctive-enough that they will never 344 # conflict with any naturally-occurring string. Also note that 345 # we only match the specific values that we recognize, and not 346 # "just anything" that matches the general pattern. There are 347 # no blanks in the recognized values, but I use an x-tended 348 # regexp and then add blanks to make it more readable. 349 optval.gsub!(/\[%- testpgm\.pathname -%\]/x, $testpgm) 350 optval.gsub!(/\[%- testpgm\.basename -%\]/x, File.basename($testpgm)) 351 optval.gsub!(/\[%- script\.pathname -%\]/x, $scriptfile) 352 353 invalid_value = false 354 case optname 355 when "$?" 356 if optval =~ /^\d+$/ 357 $testdata.cmdvalue = optval.to_i 358 else 359 invalid_value = true 360 end 361 when "clearenv" 362 if optval =~ /^\s*([A-Za-z]\w*)\s*$/ 363 $testdata.add_clearenv($1) 364 else 365 invalid_value = true 366 end 367 when "sb_args" 368 $testdata.shebang_args = optval 369 when "script" 370 $testdata.add_script(optval) 371 when "setenv" 372 if optval =~ /^\s*([A-Za-z]\w*)=(.*)$/ 373 $testdata.add_setenv($1, $2) 374 else 375 invalid_value = true 376 end 377 when "stderr" 378 $testdata.add_expecterr(optval) 379 when "stdout" 380 $testdata.add_expectout(optval) 381 when "symlink" 382 if optval =~ /^\s*(\S+)\s+(\S+)\s*$/ 383 srcfile = $1 384 newfile = $2 385 if not File.exists?(srcfile) 386 $stderr.printf "Error: source file '%s' does not exist.\n", 387 srcfile 388 invalid_value = true 389 elsif File.exists?(newfile) 390 $stderr.printf "Error: new file '%s' already exists.\n", 391 newfile 392 invalid_value = true 393 else 394 $testdata.add_symlink(srcfile, newfile) 395 end 396 else 397 invalid_value = true 398 end 399 when "user_args" 400 $testdata.user_args = optval 401 else 402 $stderr.printf "InternalError: Invalid test-option in rgdata file: %s\n", 403 optname 404 return false 405 end 406 407 if invalid_value 408 $stderr.printf "Error: Invalid value(s) for %s: %s\n", 409 optname, optval 410 return false 411 end 412 return true 413 end 414end 415 416# -------+---------+---------+-------- + --------+---------+---------+---------+ 417# Here's where the "main" routine begins... 418# 419 420$cmdopt_testpgm = nil 421$testpgm = nil 422$rgdatafile = nil 423$scriptfile = "/tmp/env-regress" 424$stop_on_error = false 425$verbose = 0 426 427exit 1 unless CommandOptions.parse(ARGV) 428 429errline = nil 430test_count = 0 431testok_count = 0 432test_lineno = -1 433max_test = -1 434regress_data = File.open($rgdatafile) 435regress_data.each_line { |dline| 436 case dline 437 when /^\s*#/, /^\s*$/ 438 # Just a comment line, ignore it. 439 when /^\s*gblenv=\s*(.+)$/ 440 if test_lineno > 0 441 $stderr.printf "Error: Cannot define a global-value in the middle of a test (#5d)\n", test_lineno 442 errline = regress_data.lineno 443 break; 444 end 445 tempval = $1 446 if tempval !~ /^([A-Za-z]\w*)=(.*)$/ 447 $stderr.printf "Error: Invalid value for 'gblenv=' request: %s\n", 448 tempval 449 errline = regress_data.lineno 450 break; 451 end 452 ExpectedResult.add_gblenv($1, $2) 453 when /^testpgm=\s*(\S+)\s*/ 454 # Set the location of the program to be tested, if it wasn't set 455 # on the command-line processing. 456 if $cmdopt_testpgm == nil 457 if not CommandOptions.set_testpgm($1) 458 errline = regress_data.lineno 459 break; 460 end 461 end 462 when /^\[test\]$/ 463 if test_lineno > 0 464 $stderr.printf "Error: Request to define a [test], but we are still defining\n" 465 $stderr.printf " the [test] at line #%s\n", test_lineno 466 errline = regress_data.lineno 467 break; 468 end 469 test_lineno = regress_data.lineno 470 max_test = test_lineno 471 printf "- Defining test at line #%s\n", test_lineno if $verbose >= 6 472 $testdata = ExpectedResult.new 473 when /^\[end\]$/ 474 # User wants us to ignore the remainder of the rgdata file... 475 break; 476 when /^\[run\]$/ 477 if test_lineno < 0 478 $stderr.printf "Error: Request to [run] a test, but no test is presently defined\n" 479 errline = regress_data.lineno 480 break; 481 end 482 printf "- Running test at line #%s\n", test_lineno if $verbose >= 1 483 run_result = $testdata.run_test 484 test_count += 1 485 printf "[Test #%3d: ", test_count 486 case run_result 487 when 0 488 # Test failed 489 printf "Failed! (line %4d)]\n", test_lineno 490 break if $stop_on_error 491 when 1 492 # Test ran as expected 493 testok_count += 1 494 printf "OK]\n" 495 else 496 # Internal error of some sort 497 printf "InternalError! (line %4d)]\n", test_lineno 498 errline = regress_data.lineno 499 break; 500 end 501 test_lineno = -1 502 503 when /^(\s*)([^\s:]+)\s*:(.+)$/ 504 blankpfx = $1 505 test_lhs = $2 506 test_rhs = $3 507 if test_lineno < 0 508 $stderr.printf "Error: No test is presently being defined\n" 509 errline = regress_data.lineno 510 break; 511 end 512 # All the real work happens in RGTestOptions.parse 513 if not RGTestOptions.parse(test_lhs, test_rhs) 514 errline = regress_data.lineno 515 break; 516 end 517 if blankpfx.length == 0 518 $stderr.printf "Note: You should at least one blank before:%s\n", 519 dline.chomp 520 $stderr.printf " at line %d of rgdata file %s\n", 521 regress_data.lineno, $rgdatafile 522 end 523 524 else 525 $stderr.printf "Error: Invalid line: %s\n", dline.chomp 526 errline = regress_data.lineno 527 break; 528 end 529} 530regress_data.close 531if errline != nil 532 $stderr.printf " at line %d of rgdata file %s\n", errline, $rgdatafile 533 exit 2 534end 535if testok_count != test_count 536 printf "%d of %d tests were successful.\n", testok_count, test_count 537 exit 1 538end 539 540printf "All %d tests were successful!\n", testok_count 541exit 0 542