1#!@PYTHON@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2012, 2016 by Delphix. All rights reserved. 16# Copyright (c) 2017, Chris Fraire <cfraire@me.com>. 17# Copyright 2019 Joyent, Inc. 18# 19 20from __future__ import print_function 21import sys 22PY3 = sys.version_info[0] == 3 23 24if PY3: 25 import configparser 26else: 27 import ConfigParser as configparser 28 29import io 30import os 31import logging 32import platform 33from logging.handlers import WatchedFileHandler 34from datetime import datetime 35from optparse import OptionParser 36from pwd import getpwnam 37from pwd import getpwuid 38from select import select 39from subprocess import PIPE 40from subprocess import Popen 41from sys import argv 42from sys import exit 43from sys import maxsize 44from threading import Timer 45from time import time 46 47BASEDIR = '/var/tmp/test_results' 48KILL = '/usr/bin/kill' 49TRUE = '/usr/bin/true' 50SUDO = '/usr/bin/sudo' 51 52retcode = 0 53 54# Custom class to reopen the log file in case it is forcibly closed by a test. 55class WatchedFileHandlerClosed(WatchedFileHandler): 56 """Watch files, including closed files. 57 Similar to (and inherits from) logging.handler.WatchedFileHandler, 58 except that IOErrors are handled by reopening the stream and retrying. 59 This will be retried up to a configurable number of times before 60 giving up, default 5. 61 """ 62 63 def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): 64 self.max_tries = max_tries 65 self.tries = 0 66 WatchedFileHandler.__init__(self, filename, mode, encoding, delay) 67 68 def emit(self, record): 69 while True: 70 try: 71 WatchedFileHandler.emit(self, record) 72 self.tries = 0 73 return 74 except IOError as err: 75 if self.tries == self.max_tries: 76 raise 77 self.stream.close() 78 self.stream = self._open() 79 self.tries += 1 80 81class Result(object): 82 total = 0 83 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} 84 85 def __init__(self): 86 self.starttime = None 87 self.returncode = None 88 self.runtime = '' 89 self.stdout = [] 90 self.stderr = [] 91 self.result = '' 92 93 def done(self, proc, killed): 94 """ 95 Finalize the results of this Cmd. 96 Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED) 97 as defined in ../stf/include/stf.shlib 98 """ 99 global retcode 100 101 Result.total += 1 102 m, s = divmod(time() - self.starttime, 60) 103 self.runtime = '%02d:%02d' % (m, s) 104 self.returncode = proc.returncode 105 if killed: 106 self.result = 'KILLED' 107 Result.runresults['KILLED'] += 1 108 retcode = 2; 109 elif self.returncode is 0: 110 self.result = 'PASS' 111 Result.runresults['PASS'] += 1 112 elif self.returncode is 3 or self.returncode is 4: 113 self.result = 'SKIP' 114 Result.runresults['SKIP'] += 1 115 elif self.returncode is not 0: 116 self.result = 'FAIL' 117 Result.runresults['FAIL'] += 1 118 retcode = 1; 119 120 121class Output(object): 122 """ 123 This class is a slightly modified version of the 'Stream' class found 124 here: http://goo.gl/aSGfv 125 """ 126 def __init__(self, stream): 127 self.stream = stream 128 self._buf = '' 129 self.lines = [] 130 131 def fileno(self): 132 return self.stream.fileno() 133 134 def read(self, drain=0): 135 """ 136 Read from the file descriptor. If 'drain' set, read until EOF. 137 """ 138 while self._read() is not None: 139 if not drain: 140 break 141 142 def _read(self): 143 """ 144 Read up to 4k of data from this output stream. Collect the output 145 up to the last newline, and append it to any leftover data from a 146 previous call. The lines are stored as a (timestamp, data) tuple 147 for easy sorting/merging later. 148 """ 149 fd = self.fileno() 150 buf = os.read(fd, 4096).decode('utf-8') 151 if not buf: 152 return None 153 if '\n' not in buf: 154 self._buf += buf 155 return [] 156 157 buf = self._buf + buf 158 tmp, rest = buf.rsplit('\n', 1) 159 self._buf = rest 160 now = datetime.now() 161 rows = tmp.split('\n') 162 self.lines += [(now, r) for r in rows] 163 164 165class Cmd(object): 166 verified_users = [] 167 168 def __init__(self, pathname, outputdir=None, timeout=None, user=None): 169 self.pathname = pathname 170 self.outputdir = outputdir or 'BASEDIR' 171 self.timeout = timeout 172 self.user = user or '' 173 self.killed = False 174 self.result = Result() 175 176 if self.timeout is None: 177 self.timeout = 60 178 179 def __str__(self): 180 return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nUser: %s\n" % \ 181 (self.pathname, self.outputdir, self.timeout, self.user) 182 183 def kill_cmd(self, proc): 184 """ 185 Kill a running command due to timeout, or ^C from the keyboard. If 186 sudo is required, this user was verified previously. 187 """ 188 self.killed = True 189 do_sudo = len(self.user) != 0 190 signal = '-TERM' 191 192 cmd = [SUDO, KILL, signal, str(proc.pid)] 193 if not do_sudo: 194 del cmd[0] 195 196 try: 197 kp = Popen(cmd) 198 kp.wait() 199 except: 200 pass 201 202 def update_cmd_privs(self, cmd, user): 203 """ 204 If a user has been specified to run this Cmd and we're not already 205 running as that user, prepend the appropriate sudo command to run 206 as that user. 207 """ 208 me = getpwuid(os.getuid()) 209 210 if not user or user is me: 211 return cmd 212 213 ret = '%s -E -u %s %s' % (SUDO, user, cmd) 214 return ret.split(' ') 215 216 def collect_output(self, proc): 217 """ 218 Read from stdout/stderr as data becomes available, until the 219 process is no longer running. Return the lines from the stdout and 220 stderr Output objects. 221 """ 222 out = Output(proc.stdout) 223 err = Output(proc.stderr) 224 res = [] 225 while proc.returncode is None: 226 proc.poll() 227 res = select([out, err], [], [], .1) 228 for fd in res[0]: 229 fd.read() 230 for fd in res[0]: 231 fd.read(drain=1) 232 233 return out.lines, err.lines 234 235 def run(self, options): 236 """ 237 This is the main function that runs each individual test. 238 Determine whether or not the command requires sudo, and modify it 239 if needed. Run the command, and update the result object. 240 """ 241 if options.dryrun is True: 242 print(self) 243 return 244 245 privcmd = self.update_cmd_privs(self.pathname, self.user) 246 try: 247 old = os.umask(0) 248 if not os.path.isdir(self.outputdir): 249 os.makedirs(self.outputdir, mode=0o777) 250 os.umask(old) 251 except OSError as e: 252 fail('%s' % e) 253 254 try: 255 self.result.starttime = time() 256 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, 257 universal_newlines=True) 258 proc.stdin.close() 259 260 # Allow a special timeout value of 0 to mean infinity 261 if int(self.timeout) == 0: 262 self.timeout = maxsize 263 t = Timer(int(self.timeout), self.kill_cmd, [proc]) 264 t.start() 265 self.result.stdout, self.result.stderr = self.collect_output(proc) 266 except KeyboardInterrupt: 267 self.kill_cmd(proc) 268 fail('\nRun terminated at user request.') 269 finally: 270 t.cancel() 271 272 self.result.done(proc, self.killed) 273 274 def skip(self): 275 """ 276 Initialize enough of the test result that we can log a skipped 277 command. 278 """ 279 Result.total += 1 280 Result.runresults['SKIP'] += 1 281 self.result.stdout = self.result.stderr = [] 282 self.result.starttime = time() 283 m, s = divmod(time() - self.result.starttime, 60) 284 self.result.runtime = '%02d:%02d' % (m, s) 285 self.result.result = 'SKIP' 286 287 def log(self, logger, options): 288 """ 289 This function is responsible for writing all output. This includes 290 the console output, the logfile of all results (with timestamped 291 merged stdout and stderr), and for each test, the unmodified 292 stdout/stderr/merged in it's own file. 293 """ 294 if logger is None: 295 return 296 297 logname = getpwuid(os.getuid()).pw_name 298 user = ' (run as %s)' % (self.user if len(self.user) else logname) 299 msga = 'Test: %s%s ' % (self.pathname, user) 300 msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) 301 pad = ' ' * (80 - (len(msga) + len(msgb))) 302 303 # If -q is specified, only print a line for tests that didn't pass. 304 # This means passing tests need to be logged as DEBUG, or the one 305 # line summary will only be printed in the logfile for failures. 306 if not options.quiet: 307 logger.info('%s%s%s' % (msga, pad, msgb)) 308 elif self.result.result is not 'PASS': 309 logger.info('%s%s%s' % (msga, pad, msgb)) 310 else: 311 logger.debug('%s%s%s' % (msga, pad, msgb)) 312 313 lines = sorted(self.result.stdout + self.result.stderr, 314 key=lambda x: x[0]) 315 316 for dt, line in lines: 317 logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) 318 319 if len(self.result.stdout): 320 with io.open(os.path.join(self.outputdir, 'stdout'), 321 encoding='utf-8', 322 errors='surrogateescape', 323 mode='w') as out: 324 for _, line in self.result.stdout: 325 out.write('%s\n' % line) 326 if len(self.result.stderr): 327 with io.open(os.path.join(self.outputdir, 'stderr'), 328 encoding='utf-8', 329 errors='surrogateescape', 330 mode='w') as err: 331 for _, line in self.result.stderr: 332 err.write('%s\n' % line) 333 if len(self.result.stdout) and len(self.result.stderr): 334 with io.open(os.path.join(self.outputdir, 'merged'), 335 encoding='utf-8', 336 errors='surrogateescape', 337 mode='w') as merged: 338 for _, line in lines: 339 merged.write('%s\n' % line) 340 341 342class Test(Cmd): 343 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 344 'post_user'] 345 346 def __init__(self, pathname, outputdir=None, timeout=None, user=None, 347 pre=None, pre_user=None, post=None, post_user=None): 348 super(Test, self).__init__(pathname, outputdir, timeout, user) 349 self.pre = pre or '' 350 self.pre_user = pre_user or '' 351 self.post = post or '' 352 self.post_user = post_user or '' 353 354 def __str__(self): 355 post_user = pre_user = '' 356 if len(self.pre_user): 357 pre_user = ' (as %s)' % (self.pre_user) 358 if len(self.post_user): 359 post_user = ' (as %s)' % (self.post_user) 360 return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \ 361 "%s%s\nUser: %s\n" % \ 362 (self.pathname, self.outputdir, self.timeout, self.pre, 363 pre_user, self.post, post_user, self.user) 364 365 def verify(self, logger): 366 """ 367 Check the pre/post scripts, user and Test. Omit the Test from this 368 run if there are any problems. 369 """ 370 files = [self.pre, self.pathname, self.post] 371 users = [self.pre_user, self.user, self.post_user] 372 373 for f in [f for f in files if len(f)]: 374 if not verify_file(f): 375 logger.info("Warning: Test '%s' not added to this run because" 376 " it failed verification." % f) 377 return False 378 379 for user in [user for user in users if len(user)]: 380 if not verify_user(user, logger): 381 logger.info("Not adding Test '%s' to this run." % 382 self.pathname) 383 return False 384 385 return True 386 387 def run(self, logger, options): 388 """ 389 Create Cmd instances for the pre/post scripts. If the pre script 390 doesn't pass, skip this Test. Run the post script regardless. 391 """ 392 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 393 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 394 user=self.pre_user) 395 test = Cmd(self.pathname, outputdir=self.outputdir, 396 timeout=self.timeout, user=self.user) 397 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 398 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 399 user=self.post_user) 400 401 cont = True 402 if len(pretest.pathname): 403 pretest.run(options) 404 cont = pretest.result.result is 'PASS' 405 pretest.log(logger, options) 406 407 if cont: 408 test.run(options) 409 else: 410 test.skip() 411 412 test.log(logger, options) 413 414 if len(posttest.pathname): 415 posttest.run(options) 416 posttest.log(logger, options) 417 418 419class TestGroup(Test): 420 props = Test.props + ['tests'] 421 422 def __init__(self, pathname, outputdir=None, timeout=None, user=None, 423 pre=None, pre_user=None, post=None, post_user=None, 424 tests=None): 425 super(TestGroup, self).__init__(pathname, outputdir, timeout, user, 426 pre, pre_user, post, post_user) 427 self.tests = tests or [] 428 429 def __str__(self): 430 post_user = pre_user = '' 431 if len(self.pre_user): 432 pre_user = ' (as %s)' % (self.pre_user) 433 if len(self.post_user): 434 post_user = ' (as %s)' % (self.post_user) 435 return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \ 436 "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \ 437 (self.pathname, self.outputdir, self.tests, self.timeout, 438 self.pre, pre_user, self.post, post_user, self.user) 439 440 def verify(self, logger): 441 """ 442 Check the pre/post scripts, user and tests in this TestGroup. Omit 443 the TestGroup entirely, or simply delete the relevant tests in the 444 group, if that's all that's required. 445 """ 446 # If the pre or post scripts are relative pathnames, convert to 447 # absolute, so they stand a chance of passing verification. 448 if len(self.pre) and not os.path.isabs(self.pre): 449 self.pre = os.path.join(self.pathname, self.pre) 450 if len(self.post) and not os.path.isabs(self.post): 451 self.post = os.path.join(self.pathname, self.post) 452 453 auxfiles = [self.pre, self.post] 454 users = [self.pre_user, self.user, self.post_user] 455 456 for f in [f for f in auxfiles if len(f)]: 457 if self.pathname != os.path.dirname(f): 458 logger.info("Warning: TestGroup '%s' not added to this run. " 459 "Auxiliary script '%s' exists in a different " 460 "directory." % (self.pathname, f)) 461 return False 462 463 if not verify_file(f): 464 logger.info("Warning: TestGroup '%s' not added to this run. " 465 "Auxiliary script '%s' failed verification." % 466 (self.pathname, f)) 467 return False 468 469 for user in [user for user in users if len(user)]: 470 if not verify_user(user, logger): 471 logger.info("Not adding TestGroup '%s' to this run." % 472 self.pathname) 473 return False 474 475 # If one of the tests is invalid, delete it, log it, and drive on. 476 self.tests[:] = [f for f in self.tests if 477 verify_file(os.path.join(self.pathname, f))] 478 479 return len(self.tests) is not 0 480 481 def run(self, logger, options): 482 """ 483 Create Cmd instances for the pre/post scripts. If the pre script 484 doesn't pass, skip all the tests in this TestGroup. Run the post 485 script regardless. 486 """ 487 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 488 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 489 user=self.pre_user) 490 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 491 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 492 user=self.post_user) 493 494 cont = True 495 if len(pretest.pathname): 496 pretest.run(options) 497 cont = pretest.result.result is 'PASS' 498 pretest.log(logger, options) 499 500 for fname in self.tests: 501 test = Cmd(os.path.join(self.pathname, fname), 502 outputdir=os.path.join(self.outputdir, fname), 503 timeout=self.timeout, user=self.user) 504 if cont: 505 test.run(options) 506 else: 507 test.skip() 508 509 test.log(logger, options) 510 511 if len(posttest.pathname): 512 posttest.run(options) 513 posttest.log(logger, options) 514 515 516class TestRun(object): 517 props = ['quiet', 'outputdir'] 518 519 def __init__(self, options): 520 self.tests = {} 521 self.testgroups = {} 522 self.starttime = time() 523 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 524 self.outputdir = os.path.join(options.outputdir, self.timestamp) 525 self.logger = self.setup_logging(options) 526 self.defaults = [ 527 ('outputdir', BASEDIR), 528 ('quiet', False), 529 ('timeout', 60), 530 ('user', ''), 531 ('pre', ''), 532 ('pre_user', ''), 533 ('post', ''), 534 ('post_user', '') 535 ] 536 537 def __str__(self): 538 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 539 s += 'TESTS:\n' 540 for key in sorted(self.tests.keys()): 541 s += '%s%s' % (self.tests[key].__str__(), '\n') 542 s += 'TESTGROUPS:\n' 543 for key in sorted(self.testgroups.keys()): 544 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 545 return s 546 547 def addtest(self, pathname, options): 548 """ 549 Create a new Test, and apply any properties that were passed in 550 from the command line. If it passes verification, add it to the 551 TestRun. 552 """ 553 test = Test(pathname) 554 for prop in Test.props: 555 setattr(test, prop, getattr(options, prop)) 556 557 if test.verify(self.logger): 558 self.tests[pathname] = test 559 560 def addtestgroup(self, dirname, filenames, options): 561 """ 562 Create a new TestGroup, and apply any properties that were passed 563 in from the command line. If it passes verification, add it to the 564 TestRun. 565 """ 566 if dirname not in self.testgroups: 567 testgroup = TestGroup(dirname) 568 for prop in Test.props: 569 setattr(testgroup, prop, getattr(options, prop)) 570 571 # Prevent pre/post scripts from running as regular tests 572 for f in [testgroup.pre, testgroup.post]: 573 if f in filenames: 574 del filenames[filenames.index(f)] 575 576 self.testgroups[dirname] = testgroup 577 self.testgroups[dirname].tests = sorted(filenames) 578 579 testgroup.verify(self.logger) 580 581 def read(self, logger, options): 582 """ 583 Read in the specified runfile, and apply the TestRun properties 584 listed in the 'DEFAULT' section to our TestRun. Then read each 585 section, and apply the appropriate properties to the Test or 586 TestGroup. Properties from individual sections override those set 587 in the 'DEFAULT' section. If the Test or TestGroup passes 588 verification, add it to the TestRun. 589 """ 590 config = configparser.RawConfigParser() 591 if not len(config.read(options.runfile)): 592 fail("Coulnd't read config file %s" % options.runfile) 593 594 for opt in TestRun.props: 595 if config.has_option('DEFAULT', opt): 596 setattr(self, opt, config.get('DEFAULT', opt)) 597 self.outputdir = os.path.join(self.outputdir, self.timestamp) 598 599 for section in config.sections(): 600 if ('arch' in config.options(section) and 601 platform.machine() != config.get(section, 'arch')): 602 continue 603 604 if 'tests' in config.options(section): 605 testgroup = TestGroup(section) 606 for prop in TestGroup.props: 607 for sect in ['DEFAULT', section]: 608 if config.has_option(sect, prop): 609 setattr(testgroup, prop, config.get(sect, prop)) 610 611 # Repopulate tests using eval to convert the string to a list 612 testgroup.tests = eval(config.get(section, 'tests')) 613 614 if testgroup.verify(logger): 615 self.testgroups[section] = testgroup 616 617 elif 'autotests' in config.options(section): 618 testgroup = TestGroup(section) 619 for prop in TestGroup.props: 620 for sect in ['DEFAULT', section]: 621 if config.has_option(sect, prop): 622 setattr(testgroup, prop, config.get(sect, prop)) 623 624 filenames = os.listdir(section) 625 # only files starting with "tst." are considered tests 626 filenames = [f for f in filenames if f.startswith("tst.")] 627 testgroup.tests = sorted(filenames) 628 629 if testgroup.verify(logger): 630 self.testgroups[section] = testgroup 631 632 else: 633 test = Test(section) 634 for prop in Test.props: 635 for sect in ['DEFAULT', section]: 636 if config.has_option(sect, prop): 637 setattr(test, prop, config.get(sect, prop)) 638 639 if test.verify(logger): 640 self.tests[section] = test 641 642 def write(self, options): 643 """ 644 Create a configuration file for editing and later use. The 645 'DEFAULT' section of the config file is created from the 646 properties that were specified on the command line. Tests are 647 simply added as sections that inherit everything from the 648 'DEFAULT' section. TestGroups are the same, except they get an 649 option including all the tests to run in that directory. 650 """ 651 652 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 653 self.defaults]) 654 config = configparser.RawConfigParser(defaults) 655 656 for test in sorted(self.tests.keys()): 657 config.add_section(test) 658 659 for testgroup in sorted(self.testgroups.keys()): 660 config.add_section(testgroup) 661 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 662 663 try: 664 with open(options.template, 'w') as f: 665 return config.write(f) 666 except IOError: 667 fail('Could not open \'%s\' for writing.' % options.template) 668 669 def complete_outputdirs(self): 670 """ 671 Collect all the pathnames for Tests, and TestGroups. Work 672 backwards one pathname component at a time, to create a unique 673 directory name in which to deposit test output. Tests will be able 674 to write output files directly in the newly modified outputdir. 675 TestGroups will be able to create one subdirectory per test in the 676 outputdir, and are guaranteed uniqueness because a group can only 677 contain files in one directory. Pre and post tests will create a 678 directory rooted at the outputdir of the Test or TestGroup in 679 question for their output. 680 """ 681 done = False 682 components = 0 683 tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) 684 total = len(tmp_dict) 685 base = self.outputdir 686 687 while not done: 688 l = [] 689 components -= 1 690 for testfile in list(tmp_dict.keys()): 691 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 692 if uniq not in l: 693 l.append(uniq) 694 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 695 else: 696 break 697 done = total == len(l) 698 699 def setup_logging(self, options): 700 """ 701 Two loggers are set up here. The first is for the logfile which 702 will contain one line summarizing the test, including the test 703 name, result, and running time. This logger will also capture the 704 timestamped combined stdout and stderr of each run. The second 705 logger is optional console output, which will contain only the one 706 line summary. The loggers are initialized at two different levels 707 to facilitate segregating the output. 708 """ 709 if options.dryrun is True: 710 return 711 712 testlogger = logging.getLogger(__name__) 713 testlogger.setLevel(logging.DEBUG) 714 715 if options.cmd is not 'wrconfig': 716 try: 717 old = os.umask(0) 718 os.makedirs(self.outputdir, mode=0o777) 719 os.umask(old) 720 except OSError as e: 721 fail('%s' % e) 722 filename = os.path.join(self.outputdir, 'log') 723 724 logfile = WatchedFileHandlerClosed(filename) 725 logfile.setLevel(logging.DEBUG) 726 logfilefmt = logging.Formatter('%(message)s') 727 logfile.setFormatter(logfilefmt) 728 testlogger.addHandler(logfile) 729 730 cons = logging.StreamHandler() 731 cons.setLevel(logging.INFO) 732 consfmt = logging.Formatter('%(message)s') 733 cons.setFormatter(consfmt) 734 testlogger.addHandler(cons) 735 736 return testlogger 737 738 def run(self, options): 739 """ 740 Walk through all the Tests and TestGroups, calling run(). 741 """ 742 if not options.dryrun: 743 try: 744 os.chdir(self.outputdir) 745 except OSError: 746 fail('Could not change to directory %s' % self.outputdir) 747 for test in sorted(self.tests.keys()): 748 self.tests[test].run(self.logger, options) 749 for testgroup in sorted(self.testgroups.keys()): 750 self.testgroups[testgroup].run(self.logger, options) 751 752 def summary(self): 753 if Result.total is 0: 754 return 755 756 print('\nResults Summary') 757 for key in list(Result.runresults.keys()): 758 if Result.runresults[key] is not 0: 759 print('%s\t% 4d' % (key, Result.runresults[key])) 760 761 m, s = divmod(time() - self.starttime, 60) 762 h, m = divmod(m, 60) 763 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 764 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 765 float(Result.total)) * 100)) 766 print('Log directory:\t%s' % self.outputdir) 767 768 769def verify_file(pathname): 770 """ 771 Verify that the supplied pathname is an executable regular file. 772 """ 773 if os.path.isdir(pathname) or os.path.islink(pathname): 774 return False 775 776 if os.path.isfile(pathname) and os.access(pathname, os.X_OK): 777 return True 778 779 return False 780 781 782def verify_user(user, logger): 783 """ 784 Verify that the specified user exists on this system, and can execute 785 sudo without being prompted for a password. 786 """ 787 testcmd = [SUDO, '-n', '-u', user, TRUE] 788 789 if user in Cmd.verified_users: 790 return True 791 792 try: 793 _ = getpwnam(user) 794 except KeyError: 795 logger.info("Warning: user '%s' does not exist.", user) 796 return False 797 798 p = Popen(testcmd) 799 p.wait() 800 if p.returncode is not 0: 801 logger.info("Warning: user '%s' cannot use passwordless sudo.", user) 802 return False 803 else: 804 Cmd.verified_users.append(user) 805 806 return True 807 808 809def find_tests(testrun, options): 810 """ 811 For the given list of pathnames, add files as Tests. For directories, 812 if do_groups is True, add the directory as a TestGroup. If False, 813 recursively search for executable files. 814 """ 815 816 for p in sorted(options.pathnames): 817 if os.path.isdir(p): 818 for dirname, _, filenames in os.walk(p): 819 if options.do_groups: 820 testrun.addtestgroup(dirname, filenames, options) 821 else: 822 for f in sorted(filenames): 823 testrun.addtest(os.path.join(dirname, f), options) 824 else: 825 testrun.addtest(p, options) 826 827 828def fail(retstr, ret=1): 829 print('%s: %s' % (argv[0], retstr)) 830 exit(ret) 831 832 833def options_cb(option, opt_str, value, parser): 834 path_options = ['runfile', 'outputdir', 'template'] 835 836 if option.dest is 'runfile' and '-w' in parser.rargs or \ 837 option.dest is 'template' and '-c' in parser.rargs: 838 fail('-c and -w are mutually exclusive.') 839 840 if opt_str in parser.rargs: 841 fail('%s may only be specified once.' % opt_str) 842 843 if option.dest is 'runfile': 844 parser.values.cmd = 'rdconfig' 845 if option.dest is 'template': 846 parser.values.cmd = 'wrconfig' 847 848 setattr(parser.values, option.dest, value) 849 if option.dest in path_options: 850 setattr(parser.values, option.dest, os.path.abspath(value)) 851 852 853def parse_args(): 854 parser = OptionParser() 855 parser.add_option('-c', action='callback', callback=options_cb, 856 type='string', dest='runfile', metavar='runfile', 857 help='Specify tests to run via config file.') 858 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 859 help='Dry run. Print tests, but take no other action.') 860 parser.add_option('-g', action='store_true', default=False, 861 dest='do_groups', help='Make directories TestGroups.') 862 parser.add_option('-o', action='callback', callback=options_cb, 863 default=BASEDIR, dest='outputdir', type='string', 864 metavar='outputdir', help='Specify an output directory.') 865 parser.add_option('-p', action='callback', callback=options_cb, 866 default='', dest='pre', metavar='script', 867 type='string', help='Specify a pre script.') 868 parser.add_option('-P', action='callback', callback=options_cb, 869 default='', dest='post', metavar='script', 870 type='string', help='Specify a post script.') 871 parser.add_option('-q', action='store_true', default=False, dest='quiet', 872 help='Silence on the console during a test run.') 873 parser.add_option('-t', action='callback', callback=options_cb, default=60, 874 dest='timeout', metavar='seconds', type='int', 875 help='Timeout (in seconds) for an individual test.') 876 parser.add_option('-u', action='callback', callback=options_cb, 877 default='', dest='user', metavar='user', type='string', 878 help='Specify a different user name to run as.') 879 parser.add_option('-w', action='callback', callback=options_cb, 880 default=None, dest='template', metavar='template', 881 type='string', help='Create a new config file.') 882 parser.add_option('-x', action='callback', callback=options_cb, default='', 883 dest='pre_user', metavar='pre_user', type='string', 884 help='Specify a user to execute the pre script.') 885 parser.add_option('-X', action='callback', callback=options_cb, default='', 886 dest='post_user', metavar='post_user', type='string', 887 help='Specify a user to execute the post script.') 888 (options, pathnames) = parser.parse_args() 889 890 if not options.runfile and not options.template: 891 options.cmd = 'runtests' 892 893 if options.runfile and len(pathnames): 894 fail('Extraneous arguments.') 895 896 options.pathnames = [os.path.abspath(path) for path in pathnames] 897 898 return options 899 900 901def main(): 902 options = parse_args() 903 testrun = TestRun(options) 904 905 if options.cmd is 'runtests': 906 find_tests(testrun, options) 907 elif options.cmd is 'rdconfig': 908 testrun.read(testrun.logger, options) 909 elif options.cmd is 'wrconfig': 910 find_tests(testrun, options) 911 testrun.write(options) 912 exit(0) 913 else: 914 fail('Unknown command specified') 915 916 testrun.complete_outputdirs() 917 testrun.run(options) 918 testrun.summary() 919 exit(retcode) 920 921 922if __name__ == '__main__': 923 main() 924