1#!@PYTHON@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2012, 2016 by Delphix. All rights reserved. 16# Copyright (c) 2017, Chris Fraire <cfraire@me.com>. 17# Copyright 2019 Joyent, Inc. 18# Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 19# 20 21from __future__ import print_function 22import sys 23PY3 = sys.version_info[0] == 3 24 25if PY3: 26 import configparser 27else: 28 import ConfigParser as configparser 29 30import io 31import os 32import logging 33import platform 34from logging.handlers import WatchedFileHandler 35from datetime import datetime 36from optparse import OptionParser 37from pwd import getpwnam 38from pwd import getpwuid 39from select import select 40from subprocess import PIPE 41from subprocess import Popen 42from sys import argv 43from sys import exit 44from sys import maxsize 45from threading import Timer 46from time import time 47 48BASEDIR = '/var/tmp/test_results' 49KILL = '/usr/bin/kill' 50TRUE = '/usr/bin/true' 51SUDO = '/usr/bin/sudo' 52 53retcode = 0 54 55# Custom class to reopen the log file in case it is forcibly closed by a test. 56class WatchedFileHandlerClosed(WatchedFileHandler): 57 """Watch files, including closed files. 58 Similar to (and inherits from) logging.handler.WatchedFileHandler, 59 except that IOErrors are handled by reopening the stream and retrying. 60 This will be retried up to a configurable number of times before 61 giving up, default 5. 62 """ 63 64 def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): 65 self.max_tries = max_tries 66 self.tries = 0 67 WatchedFileHandler.__init__(self, filename, mode, encoding, delay) 68 69 def emit(self, record): 70 while True: 71 try: 72 WatchedFileHandler.emit(self, record) 73 self.tries = 0 74 return 75 except IOError as err: 76 if self.tries == self.max_tries: 77 raise 78 self.stream.close() 79 self.stream = self._open() 80 self.tries += 1 81 82class Result(object): 83 total = 0 84 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} 85 86 def __init__(self): 87 self.starttime = None 88 self.returncode = None 89 self.runtime = '' 90 self.stdout = [] 91 self.stderr = [] 92 self.result = '' 93 94 def done(self, proc, killed): 95 """ 96 Finalize the results of this Cmd. 97 Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED) 98 as defined in ../stf/include/stf.shlib 99 """ 100 global retcode 101 102 Result.total += 1 103 m, s = divmod(time() - self.starttime, 60) 104 self.runtime = '%02d:%02d' % (m, s) 105 self.returncode = proc.returncode 106 if killed: 107 self.result = 'KILLED' 108 Result.runresults['KILLED'] += 1 109 retcode = 2; 110 elif self.returncode == 0: 111 self.result = 'PASS' 112 Result.runresults['PASS'] += 1 113 elif self.returncode == 3 or self.returncode == 4: 114 self.result = 'SKIP' 115 Result.runresults['SKIP'] += 1 116 elif self.returncode != 0: 117 self.result = 'FAIL' 118 Result.runresults['FAIL'] += 1 119 retcode = 1; 120 121 122class Output(object): 123 """ 124 This class is a slightly modified version of the 'Stream' class found 125 here: http://goo.gl/aSGfv 126 """ 127 def __init__(self, stream): 128 self.stream = stream 129 self._buf = '' 130 self.lines = [] 131 132 def fileno(self): 133 return self.stream.fileno() 134 135 def read(self, drain=0): 136 """ 137 Read from the file descriptor. If 'drain' set, read until EOF. 138 """ 139 while self._read() is not None: 140 if not drain: 141 break 142 143 def _read(self): 144 """ 145 Read up to 4k of data from this output stream. Collect the output 146 up to the last newline, and append it to any leftover data from a 147 previous call. The lines are stored as a (timestamp, data) tuple 148 for easy sorting/merging later. 149 """ 150 fd = self.fileno() 151 buf = os.read(fd, 4096).decode('utf-8', errors='ignore') 152 if not buf: 153 return None 154 if '\n' not in buf: 155 self._buf += buf 156 return [] 157 158 buf = self._buf + buf 159 tmp, rest = buf.rsplit('\n', 1) 160 self._buf = rest 161 now = datetime.now() 162 rows = tmp.split('\n') 163 self.lines += [(now, r) for r in rows] 164 165 166class Cmd(object): 167 verified_users = [] 168 169 def __init__(self, pathname, outputdir=None, timeout=None, user=None): 170 self.pathname = pathname 171 self.outputdir = outputdir or 'BASEDIR' 172 self.timeout = timeout 173 self.user = user or '' 174 self.killed = False 175 self.result = Result() 176 177 if self.timeout is None: 178 self.timeout = 60 179 180 def __str__(self): 181 return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nUser: %s\n" % \ 182 (self.pathname, self.outputdir, self.timeout, self.user) 183 184 def kill_cmd(self, proc): 185 """ 186 Kill a running command due to timeout, or ^C from the keyboard. If 187 sudo is required, this user was verified previously. 188 """ 189 self.killed = True 190 do_sudo = len(self.user) != 0 191 signal = '-TERM' 192 193 cmd = [SUDO, KILL, signal, str(proc.pid)] 194 if not do_sudo: 195 del cmd[0] 196 197 try: 198 kp = Popen(cmd) 199 kp.wait() 200 except: 201 pass 202 203 def update_cmd_privs(self, cmd, user): 204 """ 205 If a user has been specified to run this Cmd and we're not already 206 running as that user, prepend the appropriate sudo command to run 207 as that user. 208 """ 209 me = getpwuid(os.getuid()) 210 211 if not user or user == me: 212 return cmd 213 214 ret = '%s -E -u %s %s' % (SUDO, user, cmd) 215 return ret.split(' ') 216 217 def collect_output(self, proc): 218 """ 219 Read from stdout/stderr as data becomes available, until the 220 process is no longer running. Return the lines from the stdout and 221 stderr Output objects. 222 """ 223 out = Output(proc.stdout) 224 err = Output(proc.stderr) 225 res = [] 226 while proc.returncode is None: 227 proc.poll() 228 res = select([out, err], [], [], .1) 229 for fd in res[0]: 230 fd.read() 231 for fd in res[0]: 232 fd.read(drain=1) 233 234 return out.lines, err.lines 235 236 def run(self, options): 237 """ 238 This is the main function that runs each individual test. 239 Determine whether or not the command requires sudo, and modify it 240 if needed. Run the command, and update the result object. 241 """ 242 if options.dryrun is True: 243 print(self) 244 return 245 246 privcmd = self.update_cmd_privs(self.pathname, self.user) 247 try: 248 old = os.umask(0) 249 if not os.path.isdir(self.outputdir): 250 os.makedirs(self.outputdir, mode=0o777) 251 os.umask(old) 252 except OSError as e: 253 fail('%s' % e) 254 255 try: 256 self.result.starttime = time() 257 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, 258 universal_newlines=True) 259 proc.stdin.close() 260 261 # Allow a special timeout value of 0 to mean infinity 262 if int(self.timeout) == 0: 263 self.timeout = maxsize 264 t = Timer(int(self.timeout), self.kill_cmd, [proc]) 265 t.start() 266 self.result.stdout, self.result.stderr = self.collect_output(proc) 267 except KeyboardInterrupt: 268 self.kill_cmd(proc) 269 fail('\nRun terminated at user request.') 270 finally: 271 t.cancel() 272 273 self.result.done(proc, self.killed) 274 275 def skip(self): 276 """ 277 Initialize enough of the test result that we can log a skipped 278 command. 279 """ 280 Result.total += 1 281 Result.runresults['SKIP'] += 1 282 self.result.stdout = self.result.stderr = [] 283 self.result.starttime = time() 284 m, s = divmod(time() - self.result.starttime, 60) 285 self.result.runtime = '%02d:%02d' % (m, s) 286 self.result.result = 'SKIP' 287 288 def log(self, logger, options): 289 """ 290 This function is responsible for writing all output. This includes 291 the console output, the logfile of all results (with timestamped 292 merged stdout and stderr), and for each test, the unmodified 293 stdout/stderr/merged in it's own file. 294 """ 295 if logger is None: 296 return 297 298 logname = getpwuid(os.getuid()).pw_name 299 user = ' (run as %s)' % (self.user if len(self.user) else logname) 300 msga = 'Test: %s%s ' % (self.pathname, user) 301 msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) 302 pad = ' ' * (80 - (len(msga) + len(msgb))) 303 304 # If -q is specified, only print a line for tests that didn't pass. 305 # This means passing tests need to be logged as DEBUG, or the one 306 # line summary will only be printed in the logfile for failures. 307 if not options.quiet: 308 logger.info('%s%s%s' % (msga, pad, msgb)) 309 elif self.result.result != 'PASS': 310 logger.info('%s%s%s' % (msga, pad, msgb)) 311 else: 312 logger.debug('%s%s%s' % (msga, pad, msgb)) 313 314 lines = sorted(self.result.stdout + self.result.stderr, 315 key=lambda x: x[0]) 316 317 for dt, line in lines: 318 logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) 319 320 if len(self.result.stdout): 321 with io.open(os.path.join(self.outputdir, 'stdout'), 322 encoding='utf-8', 323 errors='surrogateescape', 324 mode='w') as out: 325 for _, line in self.result.stdout: 326 out.write('%s\n' % line) 327 if len(self.result.stderr): 328 with io.open(os.path.join(self.outputdir, 'stderr'), 329 encoding='utf-8', 330 errors='surrogateescape', 331 mode='w') as err: 332 for _, line in self.result.stderr: 333 err.write('%s\n' % line) 334 if len(self.result.stdout) and len(self.result.stderr): 335 with io.open(os.path.join(self.outputdir, 'merged'), 336 encoding='utf-8', 337 errors='surrogateescape', 338 mode='w') as merged: 339 for _, line in lines: 340 merged.write('%s\n' % line) 341 342 343class Test(Cmd): 344 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 345 'post_user'] 346 347 def __init__(self, pathname, outputdir=None, timeout=None, user=None, 348 pre=None, pre_user=None, post=None, post_user=None): 349 super(Test, self).__init__(pathname, outputdir, timeout, user) 350 self.pre = pre or '' 351 self.pre_user = pre_user or '' 352 self.post = post or '' 353 self.post_user = post_user or '' 354 355 def __str__(self): 356 post_user = pre_user = '' 357 if len(self.pre_user): 358 pre_user = ' (as %s)' % (self.pre_user) 359 if len(self.post_user): 360 post_user = ' (as %s)' % (self.post_user) 361 return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \ 362 "%s%s\nUser: %s\n" % \ 363 (self.pathname, self.outputdir, self.timeout, self.pre, 364 pre_user, self.post, post_user, self.user) 365 366 def verify(self, logger): 367 """ 368 Check the pre/post scripts, user and Test. Omit the Test from this 369 run if there are any problems. 370 """ 371 files = [self.pre, self.pathname, self.post] 372 users = [self.pre_user, self.user, self.post_user] 373 374 for f in [f for f in files if len(f)]: 375 if not verify_file(f): 376 logger.info("Warning: Test '%s' not added to this run because" 377 " it failed verification." % f) 378 return False 379 380 for user in [user for user in users if len(user)]: 381 if not verify_user(user, logger): 382 logger.info("Not adding Test '%s' to this run." % 383 self.pathname) 384 return False 385 386 return True 387 388 def run(self, logger, options): 389 """ 390 Create Cmd instances for the pre/post scripts. If the pre script 391 doesn't pass, skip this Test. Run the post script regardless. 392 """ 393 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 394 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 395 user=self.pre_user) 396 test = Cmd(self.pathname, outputdir=self.outputdir, 397 timeout=self.timeout, user=self.user) 398 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 399 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 400 user=self.post_user) 401 402 cont = True 403 if len(pretest.pathname): 404 pretest.run(options) 405 cont = pretest.result.result == 'PASS' 406 pretest.log(logger, options) 407 408 if cont: 409 test.run(options) 410 else: 411 test.skip() 412 413 test.log(logger, options) 414 415 if len(posttest.pathname): 416 posttest.run(options) 417 posttest.log(logger, options) 418 419 420class TestGroup(Test): 421 props = Test.props + ['tests'] 422 423 def __init__(self, pathname, outputdir=None, timeout=None, user=None, 424 pre=None, pre_user=None, post=None, post_user=None, 425 tests=None): 426 super(TestGroup, self).__init__(pathname, outputdir, timeout, user, 427 pre, pre_user, post, post_user) 428 self.tests = tests or [] 429 430 def __str__(self): 431 post_user = pre_user = '' 432 if len(self.pre_user): 433 pre_user = ' (as %s)' % (self.pre_user) 434 if len(self.post_user): 435 post_user = ' (as %s)' % (self.post_user) 436 return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \ 437 "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \ 438 (self.pathname, self.outputdir, self.tests, self.timeout, 439 self.pre, pre_user, self.post, post_user, self.user) 440 441 def verify(self, logger): 442 """ 443 Check the pre/post scripts, user and tests in this TestGroup. Omit 444 the TestGroup entirely, or simply delete the relevant tests in the 445 group, if that's all that's required. 446 """ 447 # If the pre or post scripts are relative pathnames, convert to 448 # absolute, so they stand a chance of passing verification. 449 if len(self.pre) and not os.path.isabs(self.pre): 450 self.pre = os.path.join(self.pathname, self.pre) 451 if len(self.post) and not os.path.isabs(self.post): 452 self.post = os.path.join(self.pathname, self.post) 453 454 auxfiles = [self.pre, self.post] 455 users = [self.pre_user, self.user, self.post_user] 456 457 for f in [f for f in auxfiles if len(f)]: 458 if self.pathname != os.path.dirname(f): 459 logger.info("Warning: TestGroup '%s' not added to this run. " 460 "Auxiliary script '%s' exists in a different " 461 "directory." % (self.pathname, f)) 462 return False 463 464 if not verify_file(f): 465 logger.info("Warning: TestGroup '%s' not added to this run. " 466 "Auxiliary script '%s' failed verification." % 467 (self.pathname, f)) 468 return False 469 470 for user in [user for user in users if len(user)]: 471 if not verify_user(user, logger): 472 logger.info("Not adding TestGroup '%s' to this run." % 473 self.pathname) 474 return False 475 476 # If one of the tests is invalid, delete it, log it, and drive on. 477 self.tests[:] = [f for f in self.tests if 478 verify_file(os.path.join(self.pathname, f))] 479 480 return len(self.tests) != 0 481 482 def run(self, logger, options): 483 """ 484 Create Cmd instances for the pre/post scripts. If the pre script 485 doesn't pass, skip all the tests in this TestGroup. Run the post 486 script regardless. 487 """ 488 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 489 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 490 user=self.pre_user) 491 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 492 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 493 user=self.post_user) 494 495 cont = True 496 if len(pretest.pathname): 497 pretest.run(options) 498 cont = pretest.result.result == 'PASS' 499 pretest.log(logger, options) 500 501 for fname in self.tests: 502 test = Cmd(os.path.join(self.pathname, fname), 503 outputdir=os.path.join(self.outputdir, fname), 504 timeout=self.timeout, user=self.user) 505 if cont: 506 test.run(options) 507 else: 508 test.skip() 509 510 test.log(logger, options) 511 512 if len(posttest.pathname): 513 posttest.run(options) 514 posttest.log(logger, options) 515 516 517class TestRun(object): 518 props = ['quiet', 'outputdir'] 519 520 def __init__(self, options): 521 self.tests = {} 522 self.testgroups = {} 523 self.starttime = time() 524 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 525 self.outputdir = os.path.join(options.outputdir, self.timestamp) 526 self.logger = self.setup_logging(options) 527 self.defaults = [ 528 ('outputdir', BASEDIR), 529 ('quiet', False), 530 ('timeout', 60), 531 ('user', ''), 532 ('pre', ''), 533 ('pre_user', ''), 534 ('post', ''), 535 ('post_user', '') 536 ] 537 538 def __str__(self): 539 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 540 s += 'TESTS:\n' 541 for key in sorted(self.tests.keys()): 542 s += '%s%s' % (self.tests[key].__str__(), '\n') 543 s += 'TESTGROUPS:\n' 544 for key in sorted(self.testgroups.keys()): 545 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 546 return s 547 548 def addtest(self, pathname, options): 549 """ 550 Create a new Test, and apply any properties that were passed in 551 from the command line. If it passes verification, add it to the 552 TestRun. 553 """ 554 test = Test(pathname) 555 for prop in Test.props: 556 setattr(test, prop, getattr(options, prop)) 557 558 if test.verify(self.logger): 559 self.tests[pathname] = test 560 561 def addtestgroup(self, dirname, filenames, options): 562 """ 563 Create a new TestGroup, and apply any properties that were passed 564 in from the command line. If it passes verification, add it to the 565 TestRun. 566 """ 567 if dirname not in self.testgroups: 568 testgroup = TestGroup(dirname) 569 for prop in Test.props: 570 setattr(testgroup, prop, getattr(options, prop)) 571 572 # Prevent pre/post scripts from running as regular tests 573 for f in [testgroup.pre, testgroup.post]: 574 if f in filenames: 575 del filenames[filenames.index(f)] 576 577 self.testgroups[dirname] = testgroup 578 self.testgroups[dirname].tests = sorted(filenames) 579 580 testgroup.verify(self.logger) 581 582 def read(self, logger, options): 583 """ 584 Read in the specified runfile, and apply the TestRun properties 585 listed in the 'DEFAULT' section to our TestRun. Then read each 586 section, and apply the appropriate properties to the Test or 587 TestGroup. Properties from individual sections override those set 588 in the 'DEFAULT' section. If the Test or TestGroup passes 589 verification, add it to the TestRun. 590 """ 591 config = configparser.RawConfigParser() 592 if not len(config.read(options.runfile)): 593 fail("Coulnd't read config file %s" % options.runfile) 594 595 for opt in TestRun.props: 596 if config.has_option('DEFAULT', opt): 597 setattr(self, opt, config.get('DEFAULT', opt)) 598 self.outputdir = os.path.join(self.outputdir, self.timestamp) 599 600 for section in config.sections(): 601 if ('arch' in config.options(section) and 602 platform.machine() != config.get(section, 'arch')): 603 continue 604 605 if 'tests' in config.options(section): 606 testgroup = TestGroup(section) 607 for prop in TestGroup.props: 608 for sect in ['DEFAULT', section]: 609 if config.has_option(sect, prop): 610 setattr(testgroup, prop, config.get(sect, prop)) 611 612 # Repopulate tests using eval to convert the string to a list 613 testgroup.tests = eval(config.get(section, 'tests')) 614 615 if testgroup.verify(logger): 616 self.testgroups[section] = testgroup 617 618 elif 'autotests' in config.options(section): 619 testgroup = TestGroup(section) 620 for prop in TestGroup.props: 621 for sect in ['DEFAULT', section]: 622 if config.has_option(sect, prop): 623 setattr(testgroup, prop, config.get(sect, prop)) 624 625 filenames = os.listdir(section) 626 # only files starting with "tst." are considered tests 627 filenames = [f for f in filenames if f.startswith("tst.")] 628 testgroup.tests = sorted(filenames) 629 630 if testgroup.verify(logger): 631 self.testgroups[section] = testgroup 632 633 else: 634 test = Test(section) 635 for prop in Test.props: 636 for sect in ['DEFAULT', section]: 637 if config.has_option(sect, prop): 638 setattr(test, prop, config.get(sect, prop)) 639 640 if test.verify(logger): 641 self.tests[section] = test 642 643 def write(self, options): 644 """ 645 Create a configuration file for editing and later use. The 646 'DEFAULT' section of the config file is created from the 647 properties that were specified on the command line. Tests are 648 simply added as sections that inherit everything from the 649 'DEFAULT' section. TestGroups are the same, except they get an 650 option including all the tests to run in that directory. 651 """ 652 653 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 654 self.defaults]) 655 config = configparser.RawConfigParser(defaults) 656 657 for test in sorted(self.tests.keys()): 658 config.add_section(test) 659 660 for testgroup in sorted(self.testgroups.keys()): 661 config.add_section(testgroup) 662 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 663 664 try: 665 with open(options.template, 'w') as f: 666 return config.write(f) 667 except IOError: 668 fail('Could not open \'%s\' for writing.' % options.template) 669 670 def complete_outputdirs(self): 671 """ 672 Collect all the pathnames for Tests, and TestGroups. Work 673 backwards one pathname component at a time, to create a unique 674 directory name in which to deposit test output. Tests will be able 675 to write output files directly in the newly modified outputdir. 676 TestGroups will be able to create one subdirectory per test in the 677 outputdir, and are guaranteed uniqueness because a group can only 678 contain files in one directory. Pre and post tests will create a 679 directory rooted at the outputdir of the Test or TestGroup in 680 question for their output. 681 """ 682 done = False 683 components = 0 684 tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) 685 total = len(tmp_dict) 686 base = self.outputdir 687 688 while not done: 689 l = [] 690 components -= 1 691 for testfile in list(tmp_dict.keys()): 692 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 693 if uniq not in l: 694 l.append(uniq) 695 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 696 else: 697 break 698 done = total == len(l) 699 700 def setup_logging(self, options): 701 """ 702 Two loggers are set up here. The first is for the logfile which 703 will contain one line summarizing the test, including the test 704 name, result, and running time. This logger will also capture the 705 timestamped combined stdout and stderr of each run. The second 706 logger is optional console output, which will contain only the one 707 line summary. The loggers are initialized at two different levels 708 to facilitate segregating the output. 709 """ 710 if options.dryrun is True: 711 return 712 713 testlogger = logging.getLogger(__name__) 714 testlogger.setLevel(logging.DEBUG) 715 716 if options.cmd != 'wrconfig': 717 try: 718 old = os.umask(0) 719 os.makedirs(self.outputdir, mode=0o777) 720 os.umask(old) 721 except OSError as e: 722 fail('%s' % e) 723 filename = os.path.join(self.outputdir, 'log') 724 725 logfile = WatchedFileHandlerClosed(filename) 726 logfile.setLevel(logging.DEBUG) 727 logfilefmt = logging.Formatter('%(message)s') 728 logfile.setFormatter(logfilefmt) 729 testlogger.addHandler(logfile) 730 731 cons = logging.StreamHandler() 732 cons.setLevel(logging.INFO) 733 consfmt = logging.Formatter('%(message)s') 734 cons.setFormatter(consfmt) 735 testlogger.addHandler(cons) 736 737 return testlogger 738 739 def run(self, options): 740 """ 741 Walk through all the Tests and TestGroups, calling run(). 742 """ 743 if not options.dryrun: 744 try: 745 os.chdir(self.outputdir) 746 except OSError: 747 fail('Could not change to directory %s' % self.outputdir) 748 for test in sorted(self.tests.keys()): 749 self.tests[test].run(self.logger, options) 750 for testgroup in sorted(self.testgroups.keys()): 751 self.testgroups[testgroup].run(self.logger, options) 752 753 def summary(self): 754 if Result.total == 0: 755 return 756 757 print('\nResults Summary') 758 for key in list(Result.runresults.keys()): 759 if Result.runresults[key] != 0: 760 print('%s\t% 4d' % (key, Result.runresults[key])) 761 762 m, s = divmod(time() - self.starttime, 60) 763 h, m = divmod(m, 60) 764 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 765 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 766 float(Result.total)) * 100)) 767 print('Log directory:\t%s' % self.outputdir) 768 769 770def verify_file(pathname): 771 """ 772 Verify that the supplied pathname is an executable regular file. 773 """ 774 if os.path.isdir(pathname) or os.path.islink(pathname): 775 return False 776 777 if os.path.isfile(pathname) and os.access(pathname, os.X_OK): 778 return True 779 780 return False 781 782 783def verify_user(user, logger): 784 """ 785 Verify that the specified user exists on this system, and can execute 786 sudo without being prompted for a password. 787 """ 788 testcmd = [SUDO, '-n', '-u', user, TRUE] 789 790 if user in Cmd.verified_users: 791 return True 792 793 try: 794 _ = getpwnam(user) 795 except KeyError: 796 logger.info("Warning: user '%s' does not exist.", user) 797 return False 798 799 p = Popen(testcmd) 800 p.wait() 801 if p.returncode != 0: 802 logger.info("Warning: user '%s' cannot use passwordless sudo.", user) 803 return False 804 else: 805 Cmd.verified_users.append(user) 806 807 return True 808 809 810def find_tests(testrun, options): 811 """ 812 For the given list of pathnames, add files as Tests. For directories, 813 if do_groups is True, add the directory as a TestGroup. If False, 814 recursively search for executable files. 815 """ 816 817 for p in sorted(options.pathnames): 818 if os.path.isdir(p): 819 for dirname, _, filenames in os.walk(p): 820 if options.do_groups: 821 testrun.addtestgroup(dirname, filenames, options) 822 else: 823 for f in sorted(filenames): 824 testrun.addtest(os.path.join(dirname, f), options) 825 else: 826 testrun.addtest(p, options) 827 828 829def fail(retstr, ret=1): 830 print('%s: %s' % (argv[0], retstr)) 831 exit(ret) 832 833 834def options_cb(option, opt_str, value, parser): 835 path_options = ['runfile', 'outputdir', 'template'] 836 837 if option.dest == 'runfile' and '-w' in parser.rargs or \ 838 option.dest == 'template' and '-c' in parser.rargs: 839 fail('-c and -w are mutually exclusive.') 840 841 if opt_str in parser.rargs: 842 fail('%s may only be specified once.' % opt_str) 843 844 if option.dest == 'runfile': 845 parser.values.cmd = 'rdconfig' 846 if option.dest == 'template': 847 parser.values.cmd = 'wrconfig' 848 849 setattr(parser.values, option.dest, value) 850 if option.dest in path_options: 851 setattr(parser.values, option.dest, os.path.abspath(value)) 852 853 854def parse_args(): 855 parser = OptionParser() 856 parser.add_option('-c', action='callback', callback=options_cb, 857 type='string', dest='runfile', metavar='runfile', 858 help='Specify tests to run via config file.') 859 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 860 help='Dry run. Print tests, but take no other action.') 861 parser.add_option('-g', action='store_true', default=False, 862 dest='do_groups', help='Make directories TestGroups.') 863 parser.add_option('-o', action='callback', callback=options_cb, 864 default=BASEDIR, dest='outputdir', type='string', 865 metavar='outputdir', help='Specify an output directory.') 866 parser.add_option('-p', action='callback', callback=options_cb, 867 default='', dest='pre', metavar='script', 868 type='string', help='Specify a pre script.') 869 parser.add_option('-P', action='callback', callback=options_cb, 870 default='', dest='post', metavar='script', 871 type='string', help='Specify a post script.') 872 parser.add_option('-q', action='store_true', default=False, dest='quiet', 873 help='Silence on the console during a test run.') 874 parser.add_option('-t', action='callback', callback=options_cb, default=60, 875 dest='timeout', metavar='seconds', type='int', 876 help='Timeout (in seconds) for an individual test.') 877 parser.add_option('-u', action='callback', callback=options_cb, 878 default='', dest='user', metavar='user', type='string', 879 help='Specify a different user name to run as.') 880 parser.add_option('-w', action='callback', callback=options_cb, 881 default=None, dest='template', metavar='template', 882 type='string', help='Create a new config file.') 883 parser.add_option('-x', action='callback', callback=options_cb, default='', 884 dest='pre_user', metavar='pre_user', type='string', 885 help='Specify a user to execute the pre script.') 886 parser.add_option('-X', action='callback', callback=options_cb, default='', 887 dest='post_user', metavar='post_user', type='string', 888 help='Specify a user to execute the post script.') 889 (options, pathnames) = parser.parse_args() 890 891 if not options.runfile and not options.template: 892 options.cmd = 'runtests' 893 894 if options.runfile and len(pathnames): 895 fail('Extraneous arguments.') 896 897 options.pathnames = [os.path.abspath(path) for path in pathnames] 898 899 return options 900 901 902def main(): 903 options = parse_args() 904 testrun = TestRun(options) 905 906 if options.cmd == 'runtests': 907 find_tests(testrun, options) 908 elif options.cmd == 'rdconfig': 909 testrun.read(testrun.logger, options) 910 elif options.cmd == 'wrconfig': 911 find_tests(testrun, options) 912 testrun.write(options) 913 exit(0) 914 else: 915 fail('Unknown command specified') 916 917 testrun.complete_outputdirs() 918 testrun.run(options) 919 testrun.summary() 920 exit(retcode) 921 922 923if __name__ == '__main__': 924 main() 925