1#!@PYTHON@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2012, 2016 by Delphix. All rights reserved. 16# Copyright (c) 2017, Chris Fraire <cfraire@me.com>. 17# Copyright 2019 Joyent, Inc. 18# Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 19# 20 21from __future__ import print_function 22import sys 23PY3 = sys.version_info[0] == 3 24 25if PY3: 26 import configparser 27else: 28 import ConfigParser as configparser 29 30import io 31import os 32import logging 33import platform 34import re 35from logging.handlers import WatchedFileHandler 36from datetime import datetime 37from optparse import OptionParser 38from pwd import getpwnam 39from pwd import getpwuid 40from select import select 41from subprocess import PIPE 42from subprocess import Popen 43from sys import argv 44from sys import exit 45from sys import maxsize 46from threading import Timer 47from time import time 48 49BASEDIR = '/var/tmp/test_results' 50TESTDIR = '/opt/zfs-tests/' 51KILL = '/usr/bin/kill' 52TRUE = '/usr/bin/true' 53SUDO = '/usr/bin/sudo' 54 55retcode = 0 56 57# Custom class to reopen the log file in case it is forcibly closed by a test. 58class WatchedFileHandlerClosed(WatchedFileHandler): 59 """Watch files, including closed files. 60 Similar to (and inherits from) logging.handler.WatchedFileHandler, 61 except that IOErrors are handled by reopening the stream and retrying. 62 This will be retried up to a configurable number of times before 63 giving up, default 5. 64 """ 65 66 def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): 67 self.max_tries = max_tries 68 self.tries = 0 69 WatchedFileHandler.__init__(self, filename, mode, encoding, delay) 70 71 def emit(self, record): 72 while True: 73 try: 74 WatchedFileHandler.emit(self, record) 75 self.tries = 0 76 return 77 except IOError as err: 78 if self.tries == self.max_tries: 79 raise 80 self.stream.close() 81 self.stream = self._open() 82 self.tries += 1 83 84class Result(object): 85 total = 0 86 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} 87 88 def __init__(self): 89 self.starttime = None 90 self.returncode = None 91 self.runtime = '' 92 self.stdout = [] 93 self.stderr = [] 94 self.result = '' 95 96 def done(self, proc, killed): 97 """ 98 Finalize the results of this Cmd. 99 Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED) 100 as defined in ../stf/include/stf.shlib 101 """ 102 global retcode 103 104 Result.total += 1 105 m, s = divmod(time() - self.starttime, 60) 106 self.runtime = '%02d:%02d' % (m, s) 107 self.returncode = proc.returncode 108 if killed: 109 self.result = 'KILLED' 110 Result.runresults['KILLED'] += 1 111 retcode = 2; 112 elif self.returncode == 0: 113 self.result = 'PASS' 114 Result.runresults['PASS'] += 1 115 elif self.returncode == 3 or self.returncode == 4: 116 self.result = 'SKIP' 117 Result.runresults['SKIP'] += 1 118 elif self.returncode != 0: 119 self.result = 'FAIL' 120 Result.runresults['FAIL'] += 1 121 retcode = 1; 122 123 124class Output(object): 125 """ 126 This class is a slightly modified version of the 'Stream' class found 127 here: http://goo.gl/aSGfv 128 """ 129 def __init__(self, stream): 130 self.stream = stream 131 self._buf = '' 132 self.lines = [] 133 134 def fileno(self): 135 return self.stream.fileno() 136 137 def read(self, drain=0): 138 """ 139 Read from the file descriptor. If 'drain' set, read until EOF. 140 """ 141 while self._read() is not None: 142 if not drain: 143 break 144 145 def _read(self): 146 """ 147 Read up to 4k of data from this output stream. Collect the output 148 up to the last newline, and append it to any leftover data from a 149 previous call. The lines are stored as a (timestamp, data) tuple 150 for easy sorting/merging later. 151 """ 152 fd = self.fileno() 153 buf = os.read(fd, 4096).decode('utf-8', errors='ignore') 154 if not buf: 155 return None 156 if '\n' not in buf: 157 self._buf += buf 158 return [] 159 160 buf = self._buf + buf 161 tmp, rest = buf.rsplit('\n', 1) 162 self._buf = rest 163 now = datetime.now() 164 rows = tmp.split('\n') 165 self.lines += [(now, r) for r in rows] 166 167 168class Cmd(object): 169 verified_users = [] 170 171 def __init__(self, pathname, identifier=None, outputdir=None, 172 timeout=None, user=None, tags=None): 173 self.pathname = pathname 174 self.identifier = identifier 175 self.outputdir = outputdir or 'BASEDIR' 176 self.timeout = timeout 177 self.user = user or '' 178 self.killed = False 179 self.result = Result() 180 181 if self.timeout is None: 182 self.timeout = 60 183 184 def __str__(self): 185 return '''\ 186Pathname: %s 187Identifier: %s 188Outputdir: %s 189Timeout: %d 190User: %s 191''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 192 193 def kill_cmd(self, proc): 194 """ 195 Kill a running command due to timeout, or ^C from the keyboard. If 196 sudo is required, this user was verified previously. 197 """ 198 self.killed = True 199 do_sudo = len(self.user) != 0 200 signal = '-TERM' 201 202 cmd = [SUDO, KILL, signal, str(proc.pid)] 203 if not do_sudo: 204 del cmd[0] 205 206 try: 207 kp = Popen(cmd) 208 kp.wait() 209 except: 210 pass 211 212 def update_cmd_privs(self, cmd, user): 213 """ 214 If a user has been specified to run this Cmd and we're not already 215 running as that user, prepend the appropriate sudo command to run 216 as that user. 217 """ 218 me = getpwuid(os.getuid()) 219 220 if not user or user == me: 221 return cmd 222 223 ret = '%s -E -u %s %s' % (SUDO, user, cmd) 224 return ret.split(' ') 225 226 def collect_output(self, proc): 227 """ 228 Read from stdout/stderr as data becomes available, until the 229 process is no longer running. Return the lines from the stdout and 230 stderr Output objects. 231 """ 232 out = Output(proc.stdout) 233 err = Output(proc.stderr) 234 res = [] 235 while proc.returncode is None: 236 proc.poll() 237 res = select([out, err], [], [], .1) 238 for fd in res[0]: 239 fd.read() 240 for fd in res[0]: 241 fd.read(drain=1) 242 243 return out.lines, err.lines 244 245 def run(self, options): 246 """ 247 This is the main function that runs each individual test. 248 Determine whether or not the command requires sudo, and modify it 249 if needed. Run the command, and update the result object. 250 """ 251 if options.dryrun is True: 252 print(self) 253 return 254 255 privcmd = self.update_cmd_privs(self.pathname, self.user) 256 try: 257 old = os.umask(0) 258 if not os.path.isdir(self.outputdir): 259 os.makedirs(self.outputdir, mode=0o777) 260 os.umask(old) 261 except OSError as e: 262 fail('%s' % e) 263 264 try: 265 self.result.starttime = time() 266 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, 267 universal_newlines=True) 268 proc.stdin.close() 269 270 # Allow a special timeout value of 0 to mean infinity 271 if int(self.timeout) == 0: 272 self.timeout = maxsize 273 t = Timer(int(self.timeout), self.kill_cmd, [proc]) 274 t.start() 275 self.result.stdout, self.result.stderr = self.collect_output(proc) 276 except KeyboardInterrupt: 277 self.kill_cmd(proc) 278 fail('\nRun terminated at user request.') 279 finally: 280 t.cancel() 281 282 self.result.done(proc, self.killed) 283 284 def skip(self): 285 """ 286 Initialize enough of the test result that we can log a skipped 287 command. 288 """ 289 Result.total += 1 290 Result.runresults['SKIP'] += 1 291 self.result.stdout = self.result.stderr = [] 292 self.result.starttime = time() 293 m, s = divmod(time() - self.result.starttime, 60) 294 self.result.runtime = '%02d:%02d' % (m, s) 295 self.result.result = 'SKIP' 296 297 def log(self, logger, options): 298 """ 299 This function is responsible for writing all output. This includes 300 the console output, the logfile of all results (with timestamped 301 merged stdout and stderr), and for each test, the unmodified 302 stdout/stderr/merged in it's own file. 303 """ 304 if logger is None: 305 return 306 307 logname = getpwuid(os.getuid()).pw_name 308 user = ' (run as %s)' % (self.user if len(self.user) else logname) 309 if self.identifier: 310 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 311 else: 312 msga = 'Test: %s%s ' % (self.pathname, user) 313 msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) 314 pad = ' ' * (80 - (len(msga) + len(msgb))) 315 316 # If -q is specified, only print a line for tests that didn't pass. 317 # This means passing tests need to be logged as DEBUG, or the one 318 # line summary will only be printed in the logfile for failures. 319 if not options.quiet: 320 logger.info('%s%s%s' % (msga, pad, msgb)) 321 elif self.result.result != 'PASS': 322 logger.info('%s%s%s' % (msga, pad, msgb)) 323 else: 324 logger.debug('%s%s%s' % (msga, pad, msgb)) 325 326 lines = sorted(self.result.stdout + self.result.stderr, 327 key=lambda x: x[0]) 328 329 for dt, line in lines: 330 logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) 331 332 if len(self.result.stdout): 333 with io.open(os.path.join(self.outputdir, 'stdout'), 334 encoding='utf-8', 335 errors='surrogateescape', 336 mode='w') as out: 337 for _, line in self.result.stdout: 338 out.write('%s\n' % line) 339 if len(self.result.stderr): 340 with io.open(os.path.join(self.outputdir, 'stderr'), 341 encoding='utf-8', 342 errors='surrogateescape', 343 mode='w') as err: 344 for _, line in self.result.stderr: 345 err.write('%s\n' % line) 346 if len(self.result.stdout) and len(self.result.stderr): 347 with io.open(os.path.join(self.outputdir, 'merged'), 348 encoding='utf-8', 349 errors='surrogateescape', 350 mode='w') as merged: 351 for _, line in lines: 352 merged.write('%s\n' % line) 353 354 355class Test(Cmd): 356 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 357 'post_user', 'tags'] 358 359 def __init__(self, pathname, 360 pre=None, pre_user=None, post=None, post_user=None, 361 tags=None, **kwargs): 362 super(Test, self).__init__(pathname, **kwargs) 363 self.pre = pre or '' 364 self.pre_user = pre_user or '' 365 self.post = post or '' 366 self.post_user = post_user or '' 367 self.tags = tags or [] 368 369 def __str__(self): 370 post_user = pre_user = '' 371 if len(self.pre_user): 372 pre_user = ' (as %s)' % (self.pre_user) 373 if len(self.post_user): 374 post_user = ' (as %s)' % (self.post_user) 375 return '''\ 376Pathname: %s 377Identifier: %s 378Outputdir: %s 379Timeout: %d 380User: %s 381Pre: %s%s 382Post: %s%s 383Tags: %s 384''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 385 self.pre, pre_user, self.post, post_user, self.tags) 386 387 def verify(self, logger): 388 """ 389 Check the pre/post scripts, user and Test. Omit the Test from this 390 run if there are any problems. 391 """ 392 files = [self.pre, self.pathname, self.post] 393 users = [self.pre_user, self.user, self.post_user] 394 395 for f in [f for f in files if len(f)]: 396 if not verify_file(f): 397 logger.info("Warning: Test '%s' not added to this run because" 398 " it failed verification." % f) 399 return False 400 401 for user in [user for user in users if len(user)]: 402 if not verify_user(user, logger): 403 logger.info("Not adding Test '%s' to this run." % 404 self.pathname) 405 return False 406 407 return True 408 409 def run(self, logger, options): 410 """ 411 Create Cmd instances for the pre/post scripts. If the pre script 412 doesn't pass, skip this Test. Run the post script regardless. 413 """ 414 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 415 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 416 timeout=self.timeout, user=self.pre_user) 417 test = Cmd(self.pathname, identifier=self.identifier, 418 outputdir=self.outputdir, timeout=self.timeout, 419 user=self.user) 420 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 421 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 422 timeout=self.timeout, user=self.post_user) 423 424 cont = True 425 if len(pretest.pathname): 426 pretest.run(options) 427 cont = pretest.result.result == 'PASS' 428 pretest.log(logger, options) 429 430 if cont: 431 test.run(options) 432 else: 433 test.skip() 434 435 test.log(logger, options) 436 437 if len(posttest.pathname): 438 posttest.run(options) 439 posttest.log(logger, options) 440 441 442class TestGroup(Test): 443 props = Test.props + ['tests'] 444 445 def __init__(self, pathname, tests=None, **kwargs): 446 super(TestGroup, self).__init__(pathname, **kwargs) 447 self.tests = tests or [] 448 449 def __str__(self): 450 post_user = pre_user = '' 451 if len(self.pre_user): 452 pre_user = ' (as %s)' % (self.pre_user) 453 if len(self.post_user): 454 post_user = ' (as %s)' % (self.post_user) 455 return '''\ 456Pathname: %s 457Identifier: %s 458Outputdir: %s 459Tests: %s 460Timeout: %d 461User: %s 462Pre: %s%s 463Post: %s%s 464Tags: %s 465''' % (self.pathname, self.identifier, self.outputdir, self.tests, 466 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 467 self.tags) 468 469 def filter(self, keeplist): 470 self.tests = [ x for x in self.tests if x in keeplist ] 471 472 def verify(self, logger): 473 """ 474 Check the pre/post scripts, user and tests in this TestGroup. Omit 475 the TestGroup entirely, or simply delete the relevant tests in the 476 group, if that's all that's required. 477 """ 478 # If the pre or post scripts are relative pathnames, convert to 479 # absolute, so they stand a chance of passing verification. 480 if len(self.pre) and not os.path.isabs(self.pre): 481 self.pre = os.path.join(self.pathname, self.pre) 482 if len(self.post) and not os.path.isabs(self.post): 483 self.post = os.path.join(self.pathname, self.post) 484 485 auxfiles = [self.pre, self.post] 486 users = [self.pre_user, self.user, self.post_user] 487 488 for f in [f for f in auxfiles if len(f)]: 489 if self.pathname != os.path.dirname(f): 490 logger.info("Warning: TestGroup '%s' not added to this run. " 491 "Auxiliary script '%s' exists in a different " 492 "directory." % (self.pathname, f)) 493 return False 494 495 if not verify_file(f): 496 logger.info("Warning: TestGroup '%s' not added to this run. " 497 "Auxiliary script '%s' failed verification." % 498 (self.pathname, f)) 499 return False 500 501 for user in [user for user in users if len(user)]: 502 if not verify_user(user, logger): 503 logger.info("Not adding TestGroup '%s' to this run." % 504 self.pathname) 505 return False 506 507 # If one of the tests is invalid, delete it, log it, and drive on. 508 self.tests[:] = [f for f in self.tests if 509 verify_file(os.path.join(self.pathname, f))] 510 511 return len(self.tests) != 0 512 513 def run(self, logger, options): 514 """ 515 Create Cmd instances for the pre/post scripts. If the pre script 516 doesn't pass, skip all the tests in this TestGroup. Run the post 517 script regardless. 518 """ 519 # tags assigned to this test group also include the test names 520 if options.tags and not set(self.tags).intersection(set(options.tags)): 521 return 522 523 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 524 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 525 user=self.pre_user, identifier=self.identifier) 526 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 527 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 528 user=self.post_user, identifier=self.identifier) 529 530 cont = True 531 if len(pretest.pathname): 532 pretest.run(options) 533 cont = pretest.result.result == 'PASS' 534 pretest.log(logger, options) 535 536 for fname in self.tests: 537 test = Cmd(os.path.join(self.pathname, fname), 538 outputdir=os.path.join(self.outputdir, fname), 539 timeout=self.timeout, user=self.user, 540 identifier=self.identifier) 541 if cont: 542 test.run(options) 543 else: 544 test.skip() 545 546 test.log(logger, options) 547 548 if len(posttest.pathname): 549 posttest.run(options) 550 posttest.log(logger, options) 551 552 553class TestRun(object): 554 props = ['quiet', 'outputdir'] 555 556 def __init__(self, options): 557 self.tests = {} 558 self.testgroups = {} 559 self.starttime = time() 560 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 561 self.outputdir = os.path.join(options.outputdir, self.timestamp) 562 self.logger = self.setup_logging(options) 563 self.defaults = [ 564 ('outputdir', BASEDIR), 565 ('quiet', False), 566 ('timeout', 60), 567 ('user', ''), 568 ('pre', ''), 569 ('pre_user', ''), 570 ('post', ''), 571 ('post_user', ''), 572 ('tags', []) 573 ] 574 575 def __str__(self): 576 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 577 s += 'TESTS:\n' 578 for key in sorted(self.tests.keys()): 579 s += '%s%s' % (self.tests[key].__str__(), '\n') 580 s += 'TESTGROUPS:\n' 581 for key in sorted(self.testgroups.keys()): 582 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 583 return s 584 585 def addtest(self, pathname, options): 586 """ 587 Create a new Test, and apply any properties that were passed in 588 from the command line. If it passes verification, add it to the 589 TestRun. 590 """ 591 test = Test(pathname) 592 for prop in Test.props: 593 setattr(test, prop, getattr(options, prop)) 594 595 if test.verify(self.logger): 596 self.tests[pathname] = test 597 598 def addtestgroup(self, dirname, filenames, options): 599 """ 600 Create a new TestGroup, and apply any properties that were passed 601 in from the command line. If it passes verification, add it to the 602 TestRun. 603 """ 604 if dirname not in self.testgroups: 605 testgroup = TestGroup(dirname) 606 for prop in Test.props: 607 setattr(testgroup, prop, getattr(options, prop)) 608 609 # Prevent pre/post scripts from running as regular tests 610 for f in [testgroup.pre, testgroup.post]: 611 if f in filenames: 612 del filenames[filenames.index(f)] 613 614 self.testgroups[dirname] = testgroup 615 self.testgroups[dirname].tests = sorted(filenames) 616 617 testgroup.verify(self.logger) 618 619 def filter(self, keeplist): 620 for group in list(self.testgroups.keys()): 621 if group not in keeplist: 622 del self.testgroups[group] 623 continue 624 625 g = self.testgroups[group] 626 627 if g.pre and os.path.basename(g.pre) in keeplist[group]: 628 continue 629 630 g.filter(keeplist[group]) 631 632 for test in list(self.tests.keys()): 633 directory, base = os.path.split(test) 634 if directory not in keeplist or base not in keeplist[directory]: 635 del self.tests[test] 636 637 def read(self, logger, options): 638 """ 639 Read in the specified runfile, and apply the TestRun properties 640 listed in the 'DEFAULT' section to our TestRun. Then read each 641 section, and apply the appropriate properties to the Test or 642 TestGroup. Properties from individual sections override those set 643 in the 'DEFAULT' section. If the Test or TestGroup passes 644 verification, add it to the TestRun. 645 """ 646 config = configparser.RawConfigParser() 647 parsed = config.read(options.runfiles) 648 failed = options.runfiles - set(parsed) 649 if len(failed): 650 files = ' '.join(sorted(failed)) 651 fail("Couldn't read config files: %s" % files) 652 653 for opt in TestRun.props: 654 if config.has_option('DEFAULT', opt): 655 setattr(self, opt, config.get('DEFAULT', opt)) 656 self.outputdir = os.path.join(self.outputdir, self.timestamp) 657 658 testdir = options.testdir 659 660 for section in config.sections(): 661 if ('arch' in config.options(section) and 662 platform.machine() != config.get(section, 'arch')): 663 continue 664 665 parts = section.split(':', 1) 666 sectiondir = parts[0] 667 identifier = parts[1] if len(parts) == 2 else None 668 if os.path.isdir(sectiondir): 669 pathname = sectiondir 670 elif os.path.isdir(os.path.join(testdir, sectiondir)): 671 pathname = os.path.join(testdir, sectiondir) 672 else: 673 pathname = sectiondir 674 675 testgroup = TestGroup(os.path.abspath(pathname), 676 identifier=identifier) 677 if 'tests' in config.options(section): 678 for prop in TestGroup.props: 679 for sect in ['DEFAULT', section]: 680 if config.has_option(sect, prop): 681 if prop == 'tags': 682 setattr(testgroup, prop, 683 eval(config.get(sect, prop))) 684 else: 685 setattr(testgroup, prop, 686 config.get(sect, prop)) 687 688 # Repopulate tests using eval to convert the string to a list 689 testgroup.tests = eval(config.get(section, 'tests')) 690 691 if testgroup.verify(logger): 692 self.testgroups[section] = testgroup 693 694 elif 'autotests' in config.options(section): 695 for prop in TestGroup.props: 696 for sect in ['DEFAULT', section]: 697 if config.has_option(sect, prop): 698 setattr(testgroup, prop, config.get(sect, prop)) 699 700 filenames = os.listdir(pathname) 701 # only files starting with "tst." are considered tests 702 filenames = [f for f in filenames if f.startswith("tst.")] 703 testgroup.tests = sorted(filenames) 704 705 if testgroup.verify(logger): 706 self.testgroups[section] = testgroup 707 else: 708 test = Test(section) 709 for prop in Test.props: 710 for sect in ['DEFAULT', section]: 711 if config.has_option(sect, prop): 712 setattr(test, prop, config.get(sect, prop)) 713 714 if test.verify(logger): 715 self.tests[section] = test 716 717 def write(self, options): 718 """ 719 Create a configuration file for editing and later use. The 720 'DEFAULT' section of the config file is created from the 721 properties that were specified on the command line. Tests are 722 simply added as sections that inherit everything from the 723 'DEFAULT' section. TestGroups are the same, except they get an 724 option including all the tests to run in that directory. 725 """ 726 727 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 728 self.defaults]) 729 config = configparser.RawConfigParser(defaults) 730 731 for test in sorted(self.tests.keys()): 732 config.add_section(test) 733 for prop in Test.props: 734 if prop not in self.props: 735 config.set(testgroup, prop, 736 getattr(self.testgroups[testgroup], prop)) 737 738 for testgroup in sorted(self.testgroups.keys()): 739 config.add_section(testgroup) 740 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 741 for prop in TestGroup.props: 742 if prop not in self.props: 743 config.set(testgroup, prop, 744 getattr(self.testgroups[testgroup], prop)) 745 746 try: 747 with open(options.template, 'w') as f: 748 return config.write(f) 749 except IOError: 750 fail('Could not open \'%s\' for writing.' % options.template) 751 752 def complete_outputdirs(self): 753 """ 754 Collect all the pathnames for Tests, and TestGroups. Work 755 backwards one pathname component at a time, to create a unique 756 directory name in which to deposit test output. Tests will be able 757 to write output files directly in the newly modified outputdir. 758 TestGroups will be able to create one subdirectory per test in the 759 outputdir, and are guaranteed uniqueness because a group can only 760 contain files in one directory. Pre and post tests will create a 761 directory rooted at the outputdir of the Test or TestGroup in 762 question for their output. 763 """ 764 done = False 765 components = 0 766 tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) 767 total = len(tmp_dict) 768 base = self.outputdir 769 770 while not done: 771 l = [] 772 components -= 1 773 for testfile in list(tmp_dict.keys()): 774 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 775 if uniq not in l: 776 l.append(uniq) 777 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 778 else: 779 break 780 done = total == len(l) 781 782 def setup_logging(self, options): 783 """ 784 Two loggers are set up here. The first is for the logfile which 785 will contain one line summarizing the test, including the test 786 name, result, and running time. This logger will also capture the 787 timestamped combined stdout and stderr of each run. The second 788 logger is optional console output, which will contain only the one 789 line summary. The loggers are initialized at two different levels 790 to facilitate segregating the output. 791 """ 792 if options.dryrun is True: 793 return 794 795 testlogger = logging.getLogger(__name__) 796 testlogger.setLevel(logging.DEBUG) 797 798 if not options.template: 799 try: 800 old = os.umask(0) 801 os.makedirs(self.outputdir, mode=0o777) 802 os.umask(old) 803 except OSError as e: 804 fail('%s' % e) 805 filename = os.path.join(self.outputdir, 'log') 806 807 logfile = WatchedFileHandlerClosed(filename) 808 logfile.setLevel(logging.DEBUG) 809 logfilefmt = logging.Formatter('%(message)s') 810 logfile.setFormatter(logfilefmt) 811 testlogger.addHandler(logfile) 812 813 cons = logging.StreamHandler() 814 cons.setLevel(logging.INFO) 815 consfmt = logging.Formatter('%(message)s') 816 cons.setFormatter(consfmt) 817 testlogger.addHandler(cons) 818 819 return testlogger 820 821 def run(self, options): 822 """ 823 Walk through all the Tests and TestGroups, calling run(). 824 """ 825 if not options.dryrun: 826 try: 827 os.chdir(self.outputdir) 828 except OSError: 829 fail('Could not change to directory %s' % self.outputdir) 830 for test in sorted(self.tests.keys()): 831 self.tests[test].run(self.logger, options) 832 for testgroup in sorted(self.testgroups.keys()): 833 self.testgroups[testgroup].run(self.logger, options) 834 835 def summary(self): 836 if Result.total == 0: 837 print('No tests to run') 838 return 839 840 print('\nResults Summary') 841 for key in list(Result.runresults.keys()): 842 if Result.runresults[key] != 0: 843 print('%s\t% 4d' % (key, Result.runresults[key])) 844 845 m, s = divmod(time() - self.starttime, 60) 846 h, m = divmod(m, 60) 847 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 848 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 849 float(Result.total)) * 100)) 850 print('Log directory:\t%s' % self.outputdir) 851 852 853def verify_file(pathname): 854 """ 855 Verify that the supplied pathname is an executable regular file. 856 """ 857 if os.path.isdir(pathname) or os.path.islink(pathname): 858 return False 859 860 if os.path.isfile(pathname) and os.access(pathname, os.X_OK): 861 return True 862 863 return False 864 865 866def verify_user(user, logger): 867 """ 868 Verify that the specified user exists on this system, and can execute 869 sudo without being prompted for a password. 870 """ 871 testcmd = [SUDO, '-n', '-u', user, TRUE] 872 873 if user in Cmd.verified_users: 874 return True 875 876 try: 877 _ = getpwnam(user) 878 except KeyError: 879 logger.info("Warning: user '%s' does not exist.", user) 880 return False 881 882 p = Popen(testcmd) 883 p.wait() 884 if p.returncode != 0: 885 logger.info("Warning: user '%s' cannot use passwordless sudo.", user) 886 return False 887 else: 888 Cmd.verified_users.append(user) 889 890 return True 891 892 893def find_tests(testrun, options): 894 """ 895 For the given list of pathnames, add files as Tests. For directories, 896 if do_groups is True, add the directory as a TestGroup. If False, 897 recursively search for executable files. 898 """ 899 900 for p in sorted(options.pathnames): 901 if os.path.isdir(p): 902 for dirname, _, filenames in os.walk(p): 903 if options.do_groups: 904 testrun.addtestgroup(dirname, filenames, options) 905 else: 906 for f in sorted(filenames): 907 testrun.addtest(os.path.join(dirname, f), options) 908 else: 909 testrun.addtest(p, options) 910 911 912def filter_tests(testrun, options): 913 try: 914 fh = open(options.logfile, "r", errors='replace') 915 except Exception as e: 916 fail('%s' % e) 917 918 failed = {} 919 while True: 920 line = fh.readline() 921 if not line: 922 break 923 m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line) 924 if not m: 925 continue 926 group, test = m.group(1, 2) 927 m = re.match(re.escape(options.testdir) + r'(.*)', group) 928 if m: 929 group = m.group(1) 930 try: 931 failed[group].append(test) 932 except KeyError: 933 failed[group] = [ test ] 934 fh.close() 935 936 testrun.filter(failed) 937 938 939def fail(retstr, ret=1): 940 print('%s: %s' % (argv[0], retstr)) 941 exit(ret) 942 943 944def options_cb(option, opt_str, value, parser): 945 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 946 947 if opt_str in parser.rargs: 948 fail('%s may only be specified once.' % opt_str) 949 950 if option.dest == 'runfiles': 951 parser.values.cmd = 'rdconfig' 952 value = set(os.path.abspath(p) for p in value.split(',')) 953 if option.dest == 'tags': 954 value = [x.strip() for x in value.split(',')] 955 956 if option.dest in path_options: 957 setattr(parser.values, option.dest, os.path.abspath(value)) 958 else: 959 setattr(parser.values, option.dest, value) 960 961 962def parse_args(): 963 parser = OptionParser() 964 parser.add_option('-c', action='callback', callback=options_cb, 965 type='string', dest='runfiles', metavar='runfiles', 966 help='Specify tests to run via config files.') 967 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 968 help='Dry run. Print tests, but take no other action.') 969 parser.add_option('-l', action='callback', callback=options_cb, 970 default=None, dest='logfile', metavar='logfile', 971 type='string', 972 help='Read logfile and re-run tests which failed.') 973 parser.add_option('-g', action='store_true', default=False, 974 dest='do_groups', help='Make directories TestGroups.') 975 parser.add_option('-o', action='callback', callback=options_cb, 976 default=BASEDIR, dest='outputdir', type='string', 977 metavar='outputdir', help='Specify an output directory.') 978 parser.add_option('-i', action='callback', callback=options_cb, 979 default=TESTDIR, dest='testdir', type='string', 980 metavar='testdir', help='Specify a test directory.') 981 parser.add_option('-p', action='callback', callback=options_cb, 982 default='', dest='pre', metavar='script', 983 type='string', help='Specify a pre script.') 984 parser.add_option('-P', action='callback', callback=options_cb, 985 default='', dest='post', metavar='script', 986 type='string', help='Specify a post script.') 987 parser.add_option('-q', action='store_true', default=False, dest='quiet', 988 help='Silence on the console during a test run.') 989 parser.add_option('-t', action='callback', callback=options_cb, default=60, 990 dest='timeout', metavar='seconds', type='int', 991 help='Timeout (in seconds) for an individual test.') 992 parser.add_option('-u', action='callback', callback=options_cb, 993 default='', dest='user', metavar='user', type='string', 994 help='Specify a different user name to run as.') 995 parser.add_option('-w', action='callback', callback=options_cb, 996 default=None, dest='template', metavar='template', 997 type='string', help='Create a new config file.') 998 parser.add_option('-x', action='callback', callback=options_cb, default='', 999 dest='pre_user', metavar='pre_user', type='string', 1000 help='Specify a user to execute the pre script.') 1001 parser.add_option('-X', action='callback', callback=options_cb, default='', 1002 dest='post_user', metavar='post_user', type='string', 1003 help='Specify a user to execute the post script.') 1004 parser.add_option('-T', action='callback', callback=options_cb, default='', 1005 dest='tags', metavar='tags', type='string', 1006 help='Specify tags to execute specific test groups.') 1007 (options, pathnames) = parser.parse_args() 1008 1009 if options.runfiles and len(pathnames): 1010 fail('Extraneous arguments.') 1011 1012 options.pathnames = [os.path.abspath(path) for path in pathnames] 1013 1014 return options 1015 1016 1017def main(): 1018 options = parse_args() 1019 1020 testrun = TestRun(options) 1021 1022 if options.runfiles: 1023 testrun.read(testrun.logger, options) 1024 else: 1025 find_tests(testrun, options) 1026 1027 if options.logfile: 1028 filter_tests(testrun, options) 1029 1030 if options.template: 1031 testrun.write(options) 1032 exit(0) 1033 1034 testrun.complete_outputdirs() 1035 testrun.run(options) 1036 testrun.summary() 1037 exit(retcode) 1038 1039 1040if __name__ == '__main__': 1041 main() 1042