1#!/usr/local/bin/ruby 2# -------+---------+---------+-------- + --------+---------+---------+---------+ 3# Copyright (c) 2005 - Garance Alistair Drosehn <gad@FreeBSD.org>. 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25# SUCH DAMAGE. 26# -------+---------+---------+-------- + --------+---------+---------+---------+ 27# -------+---------+---------+-------- + --------+---------+---------+---------+ 28# This script was written to provide a battery of regression-tests for some 29# changes I am making to the `env' command. I wrote a new script for this 30# for several reasons. 1) I needed to test all kinds of special-character 31# combinations, and I wanted to be able to type those in exactly as they would 32# would be in real-life situations. 2) I wanted to set environment variables 33# before executing a test, 3) I had many different details to test, so I wanted 34# to write up dozens of tests, without needing to create a hundred separate 35# little tiny files, 4) I wanted to test *failure* conditions, where I expected 36# the test would fail but I wanted to be sure that it failed the way I intended 37# it to fail. 38# This script was written for the special "shebang-line" testing that I 39# wanted for my changes to `env', but I expect it could be turned into a 40# general-purpose test-suite with a little more work. 41# Garance/June 12/2005 42# -------+---------+---------+-------- + --------+---------+---------+---------+ 43 44 45# -------+---------+---------+-------- + --------+---------+---------+---------+ 46class ExpectedResult 47 attr_writer :cmdvalue, :shebang_args, :user_args 48 @@gbl_envs = Hash.new 49 50 def ExpectedResult.add_gblenv(avar, avalue) 51 @@gbl_envs[avar] = avalue 52 end 53 54 def initialize 55 @shebang_args = "" 56 @cmdvalue = 0 57 @clear_envs = Hash.new 58 @new_envs = Hash.new 59 @old_envs = Hash.new 60 @script_lines = "" 61 @expect_err = Array.new 62 @expect_out = Array.new 63 @symlinks = Array.new 64 @user_args = nil 65 end 66 67 def add_expecterr(aline) 68 @expect_err << aline 69 end 70 71 def add_expectout(aline) 72 @expect_out << aline 73 end 74 75 def add_script(aline) 76 @script_lines += aline 77 @script_lines += "\n" if aline[-1] != "\n" 78 end 79 80 def add_clearenv(avar) 81 @clear_envs[avar] = true 82 end 83 84 def add_setenv(avar, avalue) 85 @new_envs[avar] = avalue 86 end 87 88 def add_symlink(srcf, newf) 89 @symlinks << Array.[](srcf, newf) 90 end 91 92 def check_out(name, fname, expect_arr) 93 idx = -1 94 all_matched = true 95 extra_lines = 0 96 rdata = File.open(fname) 97 rdata.each_line { |rline| 98 rline.chomp! 99 idx += 1 100 if idx > expect_arr.length - 1 101 if extra_lines == 0 and $verbose >= 1 102 printf "-- Extra line(s) on %s:\n", name 103 end 104 printf "-- [%d] > %s\n", idx, rline if $verbose >= 1 105 extra_lines += 1 106 elsif rline != expect_arr[idx] 107 if all_matched and $verbose >= 1 108 printf "-- Mismatched line(s) on %s:\n", name 109 end 110 printf "-- [%d] < %s\n", idx, expect_arr[idx] if $verbose >= 2 111 printf "-- > %s\n", rline if $verbose >= 1 112 all_matched = false 113 else 114 printf "-- %s[%d] = %s\n", name, idx, rline if $verbose >= 5 115 end 116 } 117 rdata.close 118 if extra_lines > 0 119 printf "-- %d extra line(s) found on %s\n", extra_lines, 120 name if $verbose == 0 121 return false 122 end 123 if not all_matched 124 printf "-- Mismatched line(s) found on %s\n", 125 name if $verbose == 0 126 return false 127 end 128 return true 129 end 130 131 def create_links 132 @symlinks.each { |fnames| 133 if $verbose >= 2 134 printf "-- Creating: symlink %s %s\n", fnames[0], fnames[1] 135 end 136 symres = File.symlink(fnames[0], fnames[1]) 137 return false if symres == nil 138 return false unless File.symlink?(fnames[1]) 139 } 140 return true 141 end 142 143 def destroy_links 144 @symlinks.each { |fnames| 145 if $verbose >= 2 146 printf "-- Removing: %s (symlink)\n", fnames[1] 147 end 148 if File.symlink?(fnames[1]) 149 if File.delete(fnames[1]) != 1 150 $stderr.printf "Warning: problem removing symlink '%s'\n", 151 fnames[1] 152 end 153 else 154 $stderr.printf "Warning: Symlink '%s' does not exist?!?\n", 155 fnames[1] 156 end 157 } 158 return true 159 end 160 161 def init_io_files 162 @stderr = $scriptfile + ".stderr" 163 @stdout = $scriptfile + ".stdout" 164 File.delete(@stderr) if File.exists?(@stderr) 165 File.delete(@stdout) if File.exists?(@stdout) 166 @stdin = "/dev/null" 167 168 @redirs = " <" + @stdin 169 @redirs += " >" + @stdout 170 @redirs += " 2>" + @stderr 171 172 end 173 174 def pop_envs 175 @new_envs.each_key { |evar| 176 if @old_envs.has_key?(evar) 177 ENV[evar] = @old_envs[evar] 178 else 179 ENV.delete(evar) 180 end 181 } 182 end 183 184 def push_envs 185 @@gbl_envs.each_pair { |evar, eval| 186 ENV[evar] = eval 187 } 188 @new_envs.each_pair { |evar, eval| 189 if ENV.has_key?(evar) 190 @old_envs[evar] = ENV[evar] 191 end 192 ENV[evar] = eval 193 } 194 end 195 196 def run_test 197 tscript = File.open($scriptfile, "w") 198 tscript.printf "#!%s", $testpgm 199 tscript.printf " %s", @shebang_args if @shebang_args != "" 200 tscript.printf "\n" 201 tscript.printf "%s", @script_lines if @script_lines != "" 202 tscript.close 203 File.chmod(0755, $scriptfile) 204 205 usercmd = $scriptfile 206 usercmd += " " + @user_args if @user_args != nil 207 init_io_files 208 209 push_envs 210 return 0 unless create_links 211 printf "- Executing: %s\n", usercmd if $verbose >= 1 212 printf "----- with: %s\n", @redirs if $verbose >= 6 213 sys_ok = system(usercmd + @redirs) 214 if sys_ok 215 @sav_cmdvalue = 0 216 elsif $?.exited? 217 @sav_cmdvalue = $?.exitstatus 218 else 219 @sav_cmdvalue = 125 220 end 221 destroy_links 222 pop_envs 223 sys_ok = true 224 if @sav_cmdvalue != @cmdvalue 225 printf "-- Expecting cmdvalue of %d, but $? == %d\n", @cmdvalue, 226 @sav_cmdvalue 227 sys_ok = false 228 end 229 sys_ok = false unless check_out("stdout", @stdout, @expect_out) 230 sys_ok = false unless check_out("stderr", @stderr, @expect_err) 231 return 1 if sys_ok 232 return 0 233 end 234end 235 236# -------+---------+---------+-------- + --------+---------+---------+---------+ 237# Processing of the command-line options given to the regress-sb.rb script. 238# 239class CommandOptions 240 def CommandOptions.parse(command_args) 241 parse_ok = true 242 command_args.each { |userarg| 243 case userarg 244 when /^--rgdata=(\S+)$/ 245 parse_ok = false unless set_rgdatafile($1) 246 when /^--testpgm=(\S+)$/ 247 parse_ok = false unless set_testpgm($1) 248 $cmdopt_testpgm = $testpgm 249 when "--stop-on-error", "--stop_on_error" 250 $stop_on_error = true 251 when /^--/ 252 $stderr.printf "Error: Invalid long option: %s\n", userarg 253 parse_ok = false 254 when /^-/ 255 userarg = userarg[1...userarg.length] 256 userarg.each_byte { |byte| 257 char = byte.chr 258 case char 259 when "v" 260 $verbose += 1 261 else 262 $stderr.printf "Error: Invalid short option: -%s\n", char 263 parse_ok = false 264 end 265 } 266 else 267 $stderr.printf "Error: Invalid request: %s\n", userarg 268 parse_ok = false 269 end 270 } 271 if $rgdatafile == nil 272 rgmatch = Dir.glob("regress*.rgdata") 273 if rgmatch.length == 1 274 $rgdatafile = rgmatch[0] 275 printf "Assuming --rgdata=%s\n", $rgdatafile 276 else 277 $stderr.printf "Error: The --rgdata file was not specified\n" 278 parse_ok = false 279 end 280 end 281 return parse_ok 282 end 283 284 def CommandOptions.set_rgdatafile(fname) 285 if not File.exists?(fname) 286 $stderr.printf "Error: Rgdata file '%s' does not exist\n", fname 287 return false 288 elsif not File.readable?(fname) 289 $stderr.printf "Error: Rgdata file '%s' is not readable\n", fname 290 return false 291 end 292 $rgdatafile = File.expand_path(fname) 293 return true 294 end 295 296 def CommandOptions.set_testpgm(fname) 297 if not File.exists?(fname) 298 $stderr.printf "Error: Testpgm file '%s' does not exist\n", fname 299 return false 300 elsif not File.executable?(fname) 301 $stderr.printf "Error: Testpgm file '%s' is not executable\n", fname 302 return false 303 end 304 $testpgm = File.expand_path(fname) 305 return true 306 end 307end 308 309# -------+---------+---------+-------- + --------+---------+---------+---------+ 310# Processing of the test-specific options specifed in each [test]/[run] 311# section of the regression-data file. This will set values in the 312# global $testdata object. 313# 314class RGTestOptions 315 @@rgtest_opts = nil; 316 317 def RGTestOptions.init_rgtopts 318 @@rgtest_opts = Hash.new 319 @@rgtest_opts["$?"] = true 320 @@rgtest_opts["clearenv"] = true 321 @@rgtest_opts["sb_args"] = true 322 @@rgtest_opts["script"] = true 323 @@rgtest_opts["setenv"] = true 324 @@rgtest_opts["stderr"] = true 325 @@rgtest_opts["stdout"] = true 326 @@rgtest_opts["symlink"] = true 327 @@rgtest_opts["user_args"] = true 328 end 329 330 def RGTestOptions.parse(optname, optval) 331 init_rgtopts unless @@rgtest_opts 332 333 if not @@rgtest_opts.has_key?(optname) 334 $stderr.printf "Error: Invalid test-option in rgdata file: %s\n", 335 optname 336 return false 337 end 338 339 # Support a few very specific substitutions in values specified 340 # for test data. Format of all recognized values should be: 341 # [%-object.value-%] 342 # which is hopefully distinctive-enough that they will never 343 # conflict with any naturally-occurring string. Also note that 344 # we only match the specific values that we recognize, and not 345 # "just anything" that matches the general pattern. There are 346 # no blanks in the recognized values, but I use an x-tended 347 # regexp and then add blanks to make it more readable. 348 optval.gsub!(/\[%- testpgm\.pathname -%\]/x, $testpgm) 349 optval.gsub!(/\[%- testpgm\.basename -%\]/x, File.basename($testpgm)) 350 optval.gsub!(/\[%- script\.pathname -%\]/x, $scriptfile) 351 352 invalid_value = false 353 case optname 354 when "$?" 355 if optval =~ /^\d+$/ 356 $testdata.cmdvalue = optval.to_i 357 else 358 invalid_value = true 359 end 360 when "clearenv" 361 if optval =~ /^\s*([A-Za-z]\w*)\s*$/ 362 $testdata.add_clearenv($1) 363 else 364 invalid_value = true 365 end 366 when "sb_args" 367 $testdata.shebang_args = optval 368 when "script" 369 $testdata.add_script(optval) 370 when "setenv" 371 if optval =~ /^\s*([A-Za-z]\w*)=(.*)$/ 372 $testdata.add_setenv($1, $2) 373 else 374 invalid_value = true 375 end 376 when "stderr" 377 $testdata.add_expecterr(optval) 378 when "stdout" 379 $testdata.add_expectout(optval) 380 when "symlink" 381 if optval =~ /^\s*(\S+)\s+(\S+)\s*$/ 382 srcfile = $1 383 newfile = $2 384 if not File.exists?(srcfile) 385 $stderr.printf "Error: source file '%s' does not exist.\n", 386 srcfile 387 invalid_value = true 388 elsif File.exists?(newfile) 389 $stderr.printf "Error: new file '%s' already exists.\n", 390 newfile 391 invalid_value = true 392 else 393 $testdata.add_symlink(srcfile, newfile) 394 end 395 else 396 invalid_value = true 397 end 398 when "user_args" 399 $testdata.user_args = optval 400 else 401 $stderr.printf "InternalError: Invalid test-option in rgdata file: %s\n", 402 optname 403 return false 404 end 405 406 if invalid_value 407 $stderr.printf "Error: Invalid value(s) for %s: %s\n", 408 optname, optval 409 return false 410 end 411 return true 412 end 413end 414 415# -------+---------+---------+-------- + --------+---------+---------+---------+ 416# Here's where the "main" routine begins... 417# 418 419$cmdopt_testpgm = nil 420$testpgm = nil 421$rgdatafile = nil 422$scriptfile = "/tmp/env-regress" 423$stop_on_error = false 424$verbose = 0 425 426exit 1 unless CommandOptions.parse(ARGV) 427 428errline = nil 429test_count = 0 430testok_count = 0 431test_lineno = -1 432max_test = -1 433regress_data = File.open($rgdatafile) 434regress_data.each_line { |dline| 435 case dline 436 when /^\s*#/, /^\s*$/ 437 # Just a comment line, ignore it. 438 when /^\s*gblenv=\s*(.+)$/ 439 if test_lineno > 0 440 $stderr.printf "Error: Cannot define a global-value in the middle of a test (#5d)\n", test_lineno 441 errline = regress_data.lineno 442 break; 443 end 444 tempval = $1 445 if tempval !~ /^([A-Za-z]\w*)=(.*)$/ 446 $stderr.printf "Error: Invalid value for 'gblenv=' request: %s\n", 447 tempval 448 errline = regress_data.lineno 449 break; 450 end 451 ExpectedResult.add_gblenv($1, $2) 452 when /^testpgm=\s*(\S+)\s*/ 453 # Set the location of the program to be tested, if it wasn't set 454 # on the command-line processing. 455 if $cmdopt_testpgm == nil 456 if not CommandOptions.set_testpgm($1) 457 errline = regress_data.lineno 458 break; 459 end 460 end 461 when /^\[test\]$/ 462 if test_lineno > 0 463 $stderr.printf "Error: Request to define a [test], but we are still defining\n" 464 $stderr.printf " the [test] at line #%s\n", test_lineno 465 errline = regress_data.lineno 466 break; 467 end 468 test_lineno = regress_data.lineno 469 max_test = test_lineno 470 printf "- Defining test at line #%s\n", test_lineno if $verbose >= 6 471 $testdata = ExpectedResult.new 472 when /^\[end\]$/ 473 # User wants us to ignore the remainder of the rgdata file... 474 break; 475 when /^\[run\]$/ 476 if test_lineno < 0 477 $stderr.printf "Error: Request to [run] a test, but no test is presently defined\n" 478 errline = regress_data.lineno 479 break; 480 end 481 printf "- Running test at line #%s\n", test_lineno if $verbose >= 1 482 run_result = $testdata.run_test 483 test_count += 1 484 printf "[Test #%3d: ", test_count 485 case run_result 486 when 0 487 # Test failed 488 printf "Failed! (line %4d)]\n", test_lineno 489 break if $stop_on_error 490 when 1 491 # Test ran as expected 492 testok_count += 1 493 printf "OK]\n" 494 else 495 # Internal error of some sort 496 printf "InternalError! (line %4d)]\n", test_lineno 497 errline = regress_data.lineno 498 break; 499 end 500 test_lineno = -1 501 502 when /^(\s*)([^\s:]+)\s*:(.+)$/ 503 blankpfx = $1 504 test_lhs = $2 505 test_rhs = $3 506 if test_lineno < 0 507 $stderr.printf "Error: No test is presently being defined\n" 508 errline = regress_data.lineno 509 break; 510 end 511 # All the real work happens in RGTestOptions.parse 512 if not RGTestOptions.parse(test_lhs, test_rhs) 513 errline = regress_data.lineno 514 break; 515 end 516 if blankpfx.length == 0 517 $stderr.printf "Note: You should at least one blank before:%s\n", 518 dline.chomp 519 $stderr.printf " at line %d of rgdata file %s\n", 520 regress_data.lineno, $rgdatafile 521 end 522 523 else 524 $stderr.printf "Error: Invalid line: %s\n", dline.chomp 525 errline = regress_data.lineno 526 break; 527 end 528} 529regress_data.close 530if errline != nil 531 $stderr.printf " at line %d of rgdata file %s\n", errline, $rgdatafile 532 exit 2 533end 534if testok_count != test_count 535 printf "%d of %d tests were successful.\n", testok_count, test_count 536 exit 1 537end 538 539printf "All %d tests were successful!\n", testok_count 540exit 0 541