1#!@PYTHON@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2012, 2016 by Delphix. All rights reserved. 16# Copyright (c) 2017, Chris Fraire <cfraire@me.com>. 17# Copyright 2019 Joyent, Inc. 18# Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 19# 20 21from __future__ import print_function 22import sys 23PY3 = sys.version_info[0] == 3 24 25if PY3: 26 import configparser 27else: 28 import ConfigParser as configparser 29 30import io 31import os 32import logging 33import platform 34import re 35from logging.handlers import WatchedFileHandler 36from datetime import datetime 37from optparse import OptionParser 38from pwd import getpwnam 39from pwd import getpwuid 40from select import select 41from subprocess import PIPE 42from subprocess import Popen 43from sys import argv 44from sys import exit 45from sys import maxsize 46from threading import Timer 47from time import time 48 49BASEDIR = '/var/tmp/test_results' 50KILL = '/usr/bin/kill' 51TRUE = '/usr/bin/true' 52SUDO = '/usr/bin/sudo' 53 54retcode = 0 55 56# Custom class to reopen the log file in case it is forcibly closed by a test. 57class WatchedFileHandlerClosed(WatchedFileHandler): 58 """Watch files, including closed files. 59 Similar to (and inherits from) logging.handler.WatchedFileHandler, 60 except that IOErrors are handled by reopening the stream and retrying. 61 This will be retried up to a configurable number of times before 62 giving up, default 5. 63 """ 64 65 def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): 66 self.max_tries = max_tries 67 self.tries = 0 68 WatchedFileHandler.__init__(self, filename, mode, encoding, delay) 69 70 def emit(self, record): 71 while True: 72 try: 73 WatchedFileHandler.emit(self, record) 74 self.tries = 0 75 return 76 except IOError as err: 77 if self.tries == self.max_tries: 78 raise 79 self.stream.close() 80 self.stream = self._open() 81 self.tries += 1 82 83class Result(object): 84 total = 0 85 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} 86 87 def __init__(self): 88 self.starttime = None 89 self.returncode = None 90 self.runtime = '' 91 self.stdout = [] 92 self.stderr = [] 93 self.result = '' 94 95 def done(self, proc, killed): 96 """ 97 Finalize the results of this Cmd. 98 Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED) 99 as defined in ../stf/include/stf.shlib 100 """ 101 global retcode 102 103 Result.total += 1 104 m, s = divmod(time() - self.starttime, 60) 105 self.runtime = '%02d:%02d' % (m, s) 106 self.returncode = proc.returncode 107 if killed: 108 self.result = 'KILLED' 109 Result.runresults['KILLED'] += 1 110 retcode = 2; 111 elif self.returncode == 0: 112 self.result = 'PASS' 113 Result.runresults['PASS'] += 1 114 elif self.returncode == 3 or self.returncode == 4: 115 self.result = 'SKIP' 116 Result.runresults['SKIP'] += 1 117 elif self.returncode != 0: 118 self.result = 'FAIL' 119 Result.runresults['FAIL'] += 1 120 retcode = 1; 121 122 123class Output(object): 124 """ 125 This class is a slightly modified version of the 'Stream' class found 126 here: http://goo.gl/aSGfv 127 """ 128 def __init__(self, stream): 129 self.stream = stream 130 self._buf = '' 131 self.lines = [] 132 133 def fileno(self): 134 return self.stream.fileno() 135 136 def read(self, drain=0): 137 """ 138 Read from the file descriptor. If 'drain' set, read until EOF. 139 """ 140 while self._read() is not None: 141 if not drain: 142 break 143 144 def _read(self): 145 """ 146 Read up to 4k of data from this output stream. Collect the output 147 up to the last newline, and append it to any leftover data from a 148 previous call. The lines are stored as a (timestamp, data) tuple 149 for easy sorting/merging later. 150 """ 151 fd = self.fileno() 152 buf = os.read(fd, 4096).decode('utf-8', errors='ignore') 153 if not buf: 154 return None 155 if '\n' not in buf: 156 self._buf += buf 157 return [] 158 159 buf = self._buf + buf 160 tmp, rest = buf.rsplit('\n', 1) 161 self._buf = rest 162 now = datetime.now() 163 rows = tmp.split('\n') 164 self.lines += [(now, r) for r in rows] 165 166 167class Cmd(object): 168 verified_users = [] 169 170 def __init__(self, pathname, outputdir=None, timeout=None, user=None): 171 self.pathname = pathname 172 self.outputdir = outputdir or 'BASEDIR' 173 self.timeout = timeout 174 self.user = user or '' 175 self.killed = False 176 self.result = Result() 177 178 if self.timeout is None: 179 self.timeout = 60 180 181 def __str__(self): 182 return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nUser: %s\n" % \ 183 (self.pathname, self.outputdir, self.timeout, self.user) 184 185 def kill_cmd(self, proc): 186 """ 187 Kill a running command due to timeout, or ^C from the keyboard. If 188 sudo is required, this user was verified previously. 189 """ 190 self.killed = True 191 do_sudo = len(self.user) != 0 192 signal = '-TERM' 193 194 cmd = [SUDO, KILL, signal, str(proc.pid)] 195 if not do_sudo: 196 del cmd[0] 197 198 try: 199 kp = Popen(cmd) 200 kp.wait() 201 except: 202 pass 203 204 def update_cmd_privs(self, cmd, user): 205 """ 206 If a user has been specified to run this Cmd and we're not already 207 running as that user, prepend the appropriate sudo command to run 208 as that user. 209 """ 210 me = getpwuid(os.getuid()) 211 212 if not user or user == me: 213 return cmd 214 215 ret = '%s -E -u %s %s' % (SUDO, user, cmd) 216 return ret.split(' ') 217 218 def collect_output(self, proc): 219 """ 220 Read from stdout/stderr as data becomes available, until the 221 process is no longer running. Return the lines from the stdout and 222 stderr Output objects. 223 """ 224 out = Output(proc.stdout) 225 err = Output(proc.stderr) 226 res = [] 227 while proc.returncode is None: 228 proc.poll() 229 res = select([out, err], [], [], .1) 230 for fd in res[0]: 231 fd.read() 232 for fd in res[0]: 233 fd.read(drain=1) 234 235 return out.lines, err.lines 236 237 def run(self, options): 238 """ 239 This is the main function that runs each individual test. 240 Determine whether or not the command requires sudo, and modify it 241 if needed. Run the command, and update the result object. 242 """ 243 if options.dryrun is True: 244 print(self) 245 return 246 247 privcmd = self.update_cmd_privs(self.pathname, self.user) 248 try: 249 old = os.umask(0) 250 if not os.path.isdir(self.outputdir): 251 os.makedirs(self.outputdir, mode=0o777) 252 os.umask(old) 253 except OSError as e: 254 fail('%s' % e) 255 256 try: 257 self.result.starttime = time() 258 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, 259 universal_newlines=True) 260 proc.stdin.close() 261 262 # Allow a special timeout value of 0 to mean infinity 263 if int(self.timeout) == 0: 264 self.timeout = maxsize 265 t = Timer(int(self.timeout), self.kill_cmd, [proc]) 266 t.start() 267 self.result.stdout, self.result.stderr = self.collect_output(proc) 268 except KeyboardInterrupt: 269 self.kill_cmd(proc) 270 fail('\nRun terminated at user request.') 271 finally: 272 t.cancel() 273 274 self.result.done(proc, self.killed) 275 276 def skip(self): 277 """ 278 Initialize enough of the test result that we can log a skipped 279 command. 280 """ 281 Result.total += 1 282 Result.runresults['SKIP'] += 1 283 self.result.stdout = self.result.stderr = [] 284 self.result.starttime = time() 285 m, s = divmod(time() - self.result.starttime, 60) 286 self.result.runtime = '%02d:%02d' % (m, s) 287 self.result.result = 'SKIP' 288 289 def log(self, logger, options): 290 """ 291 This function is responsible for writing all output. This includes 292 the console output, the logfile of all results (with timestamped 293 merged stdout and stderr), and for each test, the unmodified 294 stdout/stderr/merged in it's own file. 295 """ 296 if logger is None: 297 return 298 299 logname = getpwuid(os.getuid()).pw_name 300 user = ' (run as %s)' % (self.user if len(self.user) else logname) 301 msga = 'Test: %s%s ' % (self.pathname, user) 302 msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) 303 pad = ' ' * (80 - (len(msga) + len(msgb))) 304 305 # If -q is specified, only print a line for tests that didn't pass. 306 # This means passing tests need to be logged as DEBUG, or the one 307 # line summary will only be printed in the logfile for failures. 308 if not options.quiet: 309 logger.info('%s%s%s' % (msga, pad, msgb)) 310 elif self.result.result != 'PASS': 311 logger.info('%s%s%s' % (msga, pad, msgb)) 312 else: 313 logger.debug('%s%s%s' % (msga, pad, msgb)) 314 315 lines = sorted(self.result.stdout + self.result.stderr, 316 key=lambda x: x[0]) 317 318 for dt, line in lines: 319 logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) 320 321 if len(self.result.stdout): 322 with io.open(os.path.join(self.outputdir, 'stdout'), 323 encoding='utf-8', 324 errors='surrogateescape', 325 mode='w') as out: 326 for _, line in self.result.stdout: 327 out.write('%s\n' % line) 328 if len(self.result.stderr): 329 with io.open(os.path.join(self.outputdir, 'stderr'), 330 encoding='utf-8', 331 errors='surrogateescape', 332 mode='w') as err: 333 for _, line in self.result.stderr: 334 err.write('%s\n' % line) 335 if len(self.result.stdout) and len(self.result.stderr): 336 with io.open(os.path.join(self.outputdir, 'merged'), 337 encoding='utf-8', 338 errors='surrogateescape', 339 mode='w') as merged: 340 for _, line in lines: 341 merged.write('%s\n' % line) 342 343 344class Test(Cmd): 345 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 346 'post_user'] 347 348 def __init__(self, pathname, outputdir=None, timeout=None, user=None, 349 pre=None, pre_user=None, post=None, post_user=None): 350 super(Test, self).__init__(pathname, outputdir, timeout, user) 351 self.pre = pre or '' 352 self.pre_user = pre_user or '' 353 self.post = post or '' 354 self.post_user = post_user or '' 355 356 def __str__(self): 357 post_user = pre_user = '' 358 if len(self.pre_user): 359 pre_user = ' (as %s)' % (self.pre_user) 360 if len(self.post_user): 361 post_user = ' (as %s)' % (self.post_user) 362 return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \ 363 "%s%s\nUser: %s\n" % \ 364 (self.pathname, self.outputdir, self.timeout, self.pre, 365 pre_user, self.post, post_user, self.user) 366 367 def verify(self, logger): 368 """ 369 Check the pre/post scripts, user and Test. Omit the Test from this 370 run if there are any problems. 371 """ 372 files = [self.pre, self.pathname, self.post] 373 users = [self.pre_user, self.user, self.post_user] 374 375 for f in [f for f in files if len(f)]: 376 if not verify_file(f): 377 logger.info("Warning: Test '%s' not added to this run because" 378 " it failed verification." % f) 379 return False 380 381 for user in [user for user in users if len(user)]: 382 if not verify_user(user, logger): 383 logger.info("Not adding Test '%s' to this run." % 384 self.pathname) 385 return False 386 387 return True 388 389 def run(self, logger, options): 390 """ 391 Create Cmd instances for the pre/post scripts. If the pre script 392 doesn't pass, skip this Test. Run the post script regardless. 393 """ 394 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 395 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 396 user=self.pre_user) 397 test = Cmd(self.pathname, outputdir=self.outputdir, 398 timeout=self.timeout, user=self.user) 399 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 400 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 401 user=self.post_user) 402 403 cont = True 404 if len(pretest.pathname): 405 pretest.run(options) 406 cont = pretest.result.result == 'PASS' 407 pretest.log(logger, options) 408 409 if cont: 410 test.run(options) 411 else: 412 test.skip() 413 414 test.log(logger, options) 415 416 if len(posttest.pathname): 417 posttest.run(options) 418 posttest.log(logger, options) 419 420 421class TestGroup(Test): 422 props = Test.props + ['tests'] 423 424 def __init__(self, pathname, outputdir=None, timeout=None, user=None, 425 pre=None, pre_user=None, post=None, post_user=None, 426 tests=None): 427 super(TestGroup, self).__init__(pathname, outputdir, timeout, user, 428 pre, pre_user, post, post_user) 429 self.tests = tests or [] 430 431 def __str__(self): 432 post_user = pre_user = '' 433 if len(self.pre_user): 434 pre_user = ' (as %s)' % (self.pre_user) 435 if len(self.post_user): 436 post_user = ' (as %s)' % (self.post_user) 437 return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \ 438 "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \ 439 (self.pathname, self.outputdir, self.tests, self.timeout, 440 self.pre, pre_user, self.post, post_user, self.user) 441 442 def filter(self, keeplist): 443 self.tests = [ x for x in self.tests if x in keeplist ] 444 445 def verify(self, logger): 446 """ 447 Check the pre/post scripts, user and tests in this TestGroup. Omit 448 the TestGroup entirely, or simply delete the relevant tests in the 449 group, if that's all that's required. 450 """ 451 # If the pre or post scripts are relative pathnames, convert to 452 # absolute, so they stand a chance of passing verification. 453 if len(self.pre) and not os.path.isabs(self.pre): 454 self.pre = os.path.join(self.pathname, self.pre) 455 if len(self.post) and not os.path.isabs(self.post): 456 self.post = os.path.join(self.pathname, self.post) 457 458 auxfiles = [self.pre, self.post] 459 users = [self.pre_user, self.user, self.post_user] 460 461 for f in [f for f in auxfiles if len(f)]: 462 if self.pathname != os.path.dirname(f): 463 logger.info("Warning: TestGroup '%s' not added to this run. " 464 "Auxiliary script '%s' exists in a different " 465 "directory." % (self.pathname, f)) 466 return False 467 468 if not verify_file(f): 469 logger.info("Warning: TestGroup '%s' not added to this run. " 470 "Auxiliary script '%s' failed verification." % 471 (self.pathname, f)) 472 return False 473 474 for user in [user for user in users if len(user)]: 475 if not verify_user(user, logger): 476 logger.info("Not adding TestGroup '%s' to this run." % 477 self.pathname) 478 return False 479 480 # If one of the tests is invalid, delete it, log it, and drive on. 481 self.tests[:] = [f for f in self.tests if 482 verify_file(os.path.join(self.pathname, f))] 483 484 return len(self.tests) != 0 485 486 def run(self, logger, options): 487 """ 488 Create Cmd instances for the pre/post scripts. If the pre script 489 doesn't pass, skip all the tests in this TestGroup. Run the post 490 script regardless. 491 """ 492 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 493 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 494 user=self.pre_user) 495 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 496 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 497 user=self.post_user) 498 499 cont = True 500 if len(pretest.pathname): 501 pretest.run(options) 502 cont = pretest.result.result == 'PASS' 503 pretest.log(logger, options) 504 505 for fname in self.tests: 506 test = Cmd(os.path.join(self.pathname, fname), 507 outputdir=os.path.join(self.outputdir, fname), 508 timeout=self.timeout, user=self.user) 509 if cont: 510 test.run(options) 511 else: 512 test.skip() 513 514 test.log(logger, options) 515 516 if len(posttest.pathname): 517 posttest.run(options) 518 posttest.log(logger, options) 519 520 521class TestRun(object): 522 props = ['quiet', 'outputdir'] 523 524 def __init__(self, options): 525 self.tests = {} 526 self.testgroups = {} 527 self.starttime = time() 528 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 529 self.outputdir = os.path.join(options.outputdir, self.timestamp) 530 self.logger = self.setup_logging(options) 531 self.defaults = [ 532 ('outputdir', BASEDIR), 533 ('quiet', False), 534 ('timeout', 60), 535 ('user', ''), 536 ('pre', ''), 537 ('pre_user', ''), 538 ('post', ''), 539 ('post_user', '') 540 ] 541 542 def __str__(self): 543 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 544 s += 'TESTS:\n' 545 for key in sorted(self.tests.keys()): 546 s += '%s%s' % (self.tests[key].__str__(), '\n') 547 s += 'TESTGROUPS:\n' 548 for key in sorted(self.testgroups.keys()): 549 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 550 return s 551 552 def addtest(self, pathname, options): 553 """ 554 Create a new Test, and apply any properties that were passed in 555 from the command line. If it passes verification, add it to the 556 TestRun. 557 """ 558 test = Test(pathname) 559 for prop in Test.props: 560 setattr(test, prop, getattr(options, prop)) 561 562 if test.verify(self.logger): 563 self.tests[pathname] = test 564 565 def addtestgroup(self, dirname, filenames, options): 566 """ 567 Create a new TestGroup, and apply any properties that were passed 568 in from the command line. If it passes verification, add it to the 569 TestRun. 570 """ 571 if dirname not in self.testgroups: 572 testgroup = TestGroup(dirname) 573 for prop in Test.props: 574 setattr(testgroup, prop, getattr(options, prop)) 575 576 # Prevent pre/post scripts from running as regular tests 577 for f in [testgroup.pre, testgroup.post]: 578 if f in filenames: 579 del filenames[filenames.index(f)] 580 581 self.testgroups[dirname] = testgroup 582 self.testgroups[dirname].tests = sorted(filenames) 583 584 testgroup.verify(self.logger) 585 586 def filter(self, keeplist): 587 for group in list(self.testgroups.keys()): 588 if group not in keeplist: 589 del self.testgroups[group] 590 continue 591 592 g = self.testgroups[group] 593 594 if g.pre and os.path.basename(g.pre) in keeplist[group]: 595 continue 596 597 g.filter(keeplist[group]) 598 599 for test in list(self.tests.keys()): 600 directory, base = os.path.split(test) 601 if directory not in keeplist or base not in keeplist[directory]: 602 del self.tests[test] 603 604 def read(self, logger, options): 605 """ 606 Read in the specified runfile, and apply the TestRun properties 607 listed in the 'DEFAULT' section to our TestRun. Then read each 608 section, and apply the appropriate properties to the Test or 609 TestGroup. Properties from individual sections override those set 610 in the 'DEFAULT' section. If the Test or TestGroup passes 611 verification, add it to the TestRun. 612 """ 613 config = configparser.RawConfigParser() 614 if not len(config.read(options.runfile)): 615 fail("Couldn't read config file %s" % options.runfile) 616 617 for opt in TestRun.props: 618 if config.has_option('DEFAULT', opt): 619 setattr(self, opt, config.get('DEFAULT', opt)) 620 self.outputdir = os.path.join(self.outputdir, self.timestamp) 621 622 for section in config.sections(): 623 if ('arch' in config.options(section) and 624 platform.machine() != config.get(section, 'arch')): 625 continue 626 627 if 'tests' in config.options(section): 628 testgroup = TestGroup(section) 629 for prop in TestGroup.props: 630 for sect in ['DEFAULT', section]: 631 if config.has_option(sect, prop): 632 setattr(testgroup, prop, config.get(sect, prop)) 633 634 # Repopulate tests using eval to convert the string to a list 635 testgroup.tests = eval(config.get(section, 'tests')) 636 637 if testgroup.verify(logger): 638 self.testgroups[section] = testgroup 639 640 elif 'autotests' in config.options(section): 641 testgroup = TestGroup(section) 642 for prop in TestGroup.props: 643 for sect in ['DEFAULT', section]: 644 if config.has_option(sect, prop): 645 setattr(testgroup, prop, config.get(sect, prop)) 646 647 filenames = os.listdir(section) 648 # only files starting with "tst." are considered tests 649 filenames = [f for f in filenames if f.startswith("tst.")] 650 testgroup.tests = sorted(filenames) 651 652 if testgroup.verify(logger): 653 self.testgroups[section] = testgroup 654 655 else: 656 test = Test(section) 657 for prop in Test.props: 658 for sect in ['DEFAULT', section]: 659 if config.has_option(sect, prop): 660 setattr(test, prop, config.get(sect, prop)) 661 662 if test.verify(logger): 663 self.tests[section] = test 664 665 def write(self, options): 666 """ 667 Create a configuration file for editing and later use. The 668 'DEFAULT' section of the config file is created from the 669 properties that were specified on the command line. Tests are 670 simply added as sections that inherit everything from the 671 'DEFAULT' section. TestGroups are the same, except they get an 672 option including all the tests to run in that directory. 673 """ 674 675 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 676 self.defaults]) 677 config = configparser.RawConfigParser(defaults) 678 679 for test in sorted(self.tests.keys()): 680 config.add_section(test) 681 for prop in Test.props: 682 if prop not in self.props: 683 config.set(testgroup, prop, 684 getattr(self.testgroups[testgroup], prop)) 685 686 for testgroup in sorted(self.testgroups.keys()): 687 config.add_section(testgroup) 688 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 689 for prop in TestGroup.props: 690 if prop not in self.props: 691 config.set(testgroup, prop, 692 getattr(self.testgroups[testgroup], prop)) 693 694 try: 695 with open(options.template, 'w') as f: 696 return config.write(f) 697 except IOError: 698 fail('Could not open \'%s\' for writing.' % options.template) 699 700 def complete_outputdirs(self): 701 """ 702 Collect all the pathnames for Tests, and TestGroups. Work 703 backwards one pathname component at a time, to create a unique 704 directory name in which to deposit test output. Tests will be able 705 to write output files directly in the newly modified outputdir. 706 TestGroups will be able to create one subdirectory per test in the 707 outputdir, and are guaranteed uniqueness because a group can only 708 contain files in one directory. Pre and post tests will create a 709 directory rooted at the outputdir of the Test or TestGroup in 710 question for their output. 711 """ 712 done = False 713 components = 0 714 tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) 715 total = len(tmp_dict) 716 base = self.outputdir 717 718 while not done: 719 l = [] 720 components -= 1 721 for testfile in list(tmp_dict.keys()): 722 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 723 if uniq not in l: 724 l.append(uniq) 725 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 726 else: 727 break 728 done = total == len(l) 729 730 def setup_logging(self, options): 731 """ 732 Two loggers are set up here. The first is for the logfile which 733 will contain one line summarizing the test, including the test 734 name, result, and running time. This logger will also capture the 735 timestamped combined stdout and stderr of each run. The second 736 logger is optional console output, which will contain only the one 737 line summary. The loggers are initialized at two different levels 738 to facilitate segregating the output. 739 """ 740 if options.dryrun is True: 741 return 742 743 testlogger = logging.getLogger(__name__) 744 testlogger.setLevel(logging.DEBUG) 745 746 if not options.template: 747 try: 748 old = os.umask(0) 749 os.makedirs(self.outputdir, mode=0o777) 750 os.umask(old) 751 except OSError as e: 752 fail('%s' % e) 753 filename = os.path.join(self.outputdir, 'log') 754 755 logfile = WatchedFileHandlerClosed(filename) 756 logfile.setLevel(logging.DEBUG) 757 logfilefmt = logging.Formatter('%(message)s') 758 logfile.setFormatter(logfilefmt) 759 testlogger.addHandler(logfile) 760 761 cons = logging.StreamHandler() 762 cons.setLevel(logging.INFO) 763 consfmt = logging.Formatter('%(message)s') 764 cons.setFormatter(consfmt) 765 testlogger.addHandler(cons) 766 767 return testlogger 768 769 def run(self, options): 770 """ 771 Walk through all the Tests and TestGroups, calling run(). 772 """ 773 if not options.dryrun: 774 try: 775 os.chdir(self.outputdir) 776 except OSError: 777 fail('Could not change to directory %s' % self.outputdir) 778 for test in sorted(self.tests.keys()): 779 self.tests[test].run(self.logger, options) 780 for testgroup in sorted(self.testgroups.keys()): 781 self.testgroups[testgroup].run(self.logger, options) 782 783 def summary(self): 784 if Result.total == 0: 785 return 786 787 print('\nResults Summary') 788 for key in list(Result.runresults.keys()): 789 if Result.runresults[key] != 0: 790 print('%s\t% 4d' % (key, Result.runresults[key])) 791 792 m, s = divmod(time() - self.starttime, 60) 793 h, m = divmod(m, 60) 794 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 795 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 796 float(Result.total)) * 100)) 797 print('Log directory:\t%s' % self.outputdir) 798 799 800def verify_file(pathname): 801 """ 802 Verify that the supplied pathname is an executable regular file. 803 """ 804 if os.path.isdir(pathname) or os.path.islink(pathname): 805 return False 806 807 if os.path.isfile(pathname) and os.access(pathname, os.X_OK): 808 return True 809 810 return False 811 812 813def verify_user(user, logger): 814 """ 815 Verify that the specified user exists on this system, and can execute 816 sudo without being prompted for a password. 817 """ 818 testcmd = [SUDO, '-n', '-u', user, TRUE] 819 820 if user in Cmd.verified_users: 821 return True 822 823 try: 824 _ = getpwnam(user) 825 except KeyError: 826 logger.info("Warning: user '%s' does not exist.", user) 827 return False 828 829 p = Popen(testcmd) 830 p.wait() 831 if p.returncode != 0: 832 logger.info("Warning: user '%s' cannot use passwordless sudo.", user) 833 return False 834 else: 835 Cmd.verified_users.append(user) 836 837 return True 838 839 840def find_tests(testrun, options): 841 """ 842 For the given list of pathnames, add files as Tests. For directories, 843 if do_groups is True, add the directory as a TestGroup. If False, 844 recursively search for executable files. 845 """ 846 847 for p in sorted(options.pathnames): 848 if os.path.isdir(p): 849 for dirname, _, filenames in os.walk(p): 850 if options.do_groups: 851 testrun.addtestgroup(dirname, filenames, options) 852 else: 853 for f in sorted(filenames): 854 testrun.addtest(os.path.join(dirname, f), options) 855 else: 856 testrun.addtest(p, options) 857 858 859def filter_tests(testrun, options): 860 try: 861 fh = open(options.logfile, "r", errors='replace') 862 except Exception as e: 863 fail('%s' % e) 864 865 failed = {} 866 while True: 867 line = fh.readline() 868 if not line: 869 break 870 m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line) 871 if not m: 872 continue 873 group, test = m.group(1, 2) 874 try: 875 failed[group].append(test) 876 except KeyError: 877 failed[group] = [ test ] 878 fh.close() 879 880 testrun.filter(failed) 881 882 883def fail(retstr, ret=1): 884 print('%s: %s' % (argv[0], retstr)) 885 exit(ret) 886 887 888def options_cb(option, opt_str, value, parser): 889 path_options = ['runfile', 'outputdir', 'template', 'logfile'] 890 891 if opt_str in parser.rargs: 892 fail('%s may only be specified once.' % opt_str) 893 894 if option.dest in path_options: 895 setattr(parser.values, option.dest, os.path.abspath(value)) 896 else: 897 setattr(parser.values, option.dest, value) 898 899 900def parse_args(): 901 parser = OptionParser() 902 parser.add_option('-c', action='callback', callback=options_cb, 903 type='string', dest='runfile', metavar='runfile', 904 help='Specify tests to run via config file.') 905 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 906 help='Dry run. Print tests, but take no other action.') 907 parser.add_option('-l', action='callback', callback=options_cb, 908 default=None, dest='logfile', metavar='logfile', 909 type='string', 910 help='Read logfile and re-run tests which failed.') 911 parser.add_option('-g', action='store_true', default=False, 912 dest='do_groups', help='Make directories TestGroups.') 913 parser.add_option('-o', action='callback', callback=options_cb, 914 default=BASEDIR, dest='outputdir', type='string', 915 metavar='outputdir', help='Specify an output directory.') 916 parser.add_option('-p', action='callback', callback=options_cb, 917 default='', dest='pre', metavar='script', 918 type='string', help='Specify a pre script.') 919 parser.add_option('-P', action='callback', callback=options_cb, 920 default='', dest='post', metavar='script', 921 type='string', help='Specify a post script.') 922 parser.add_option('-q', action='store_true', default=False, dest='quiet', 923 help='Silence on the console during a test run.') 924 parser.add_option('-t', action='callback', callback=options_cb, default=60, 925 dest='timeout', metavar='seconds', type='int', 926 help='Timeout (in seconds) for an individual test.') 927 parser.add_option('-u', action='callback', callback=options_cb, 928 default='', dest='user', metavar='user', type='string', 929 help='Specify a different user name to run as.') 930 parser.add_option('-w', action='callback', callback=options_cb, 931 default=None, dest='template', metavar='template', 932 type='string', help='Create a new config file.') 933 parser.add_option('-x', action='callback', callback=options_cb, default='', 934 dest='pre_user', metavar='pre_user', type='string', 935 help='Specify a user to execute the pre script.') 936 parser.add_option('-X', action='callback', callback=options_cb, default='', 937 dest='post_user', metavar='post_user', type='string', 938 help='Specify a user to execute the post script.') 939 (options, pathnames) = parser.parse_args() 940 941 if options.runfile and len(pathnames): 942 fail('Extraneous arguments.') 943 944 options.pathnames = [os.path.abspath(path) for path in pathnames] 945 946 return options 947 948 949def main(): 950 options = parse_args() 951 952 testrun = TestRun(options) 953 954 if options.runfile: 955 testrun.read(testrun.logger, options) 956 else: 957 find_tests(testrun, options) 958 959 if options.logfile: 960 filter_tests(testrun, options) 961 962 if options.template: 963 testrun.write(options) 964 exit(0) 965 966 testrun.complete_outputdirs() 967 testrun.run(options) 968 testrun.summary() 969 exit(retcode) 970 971 972if __name__ == '__main__': 973 main() 974