1#!@PYTHON@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2012, 2016 by Delphix. All rights reserved. 16# Copyright (c) 2017, Chris Fraire <cfraire@me.com>. 17# Copyright 2019 Joyent, Inc. 18# Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 19# Copyright 2026 Gordon W. Ross 20# 21 22from __future__ import print_function 23import sys 24PY3 = sys.version_info[0] == 3 25 26if PY3: 27 import configparser 28else: 29 import ConfigParser as configparser 30 31import io 32import os 33import logging 34import platform 35import re 36from logging.handlers import WatchedFileHandler 37from datetime import datetime 38from optparse import OptionParser 39from pwd import getpwnam 40from pwd import getpwuid 41from select import select 42from subprocess import PIPE 43from subprocess import Popen 44from sys import argv 45from sys import exit 46from sys import maxsize 47from threading import Timer 48from time import time 49 50BASEDIR = '/var/tmp/test_results' 51TESTDIR = '/opt/zfs-tests/' 52KILL = '/usr/bin/kill' 53TRUE = '/usr/bin/true' 54SUDO = '/usr/bin/sudo' 55 56# Exit codes as defined in stf/include/stf.shlib 57STF_PASS = 0 58STF_FAIL = 1 59STF_UNRESOLVED = 2 60STF_NOTINUSE = 3 61STF_UNSUPPORTED = 4 62STF_UNTESTED = 5 63STF_UNINITIATED = 6 64STF_NORESULT = 7 65STF_WARNING = 8 66STF_TIMED_OUT = 9 67STF_ABORTED = 10 68STF_OTHER = 11 69 70retcode = 0 71 72# Custom class to reopen the log file in case it is forcibly closed by a test. 73class WatchedFileHandlerClosed(WatchedFileHandler): 74 """Watch files, including closed files. 75 Similar to (and inherits from) logging.handler.WatchedFileHandler, 76 except that IOErrors are handled by reopening the stream and retrying. 77 This will be retried up to a configurable number of times before 78 giving up, default 5. 79 """ 80 81 def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): 82 self.max_tries = max_tries 83 self.tries = 0 84 WatchedFileHandler.__init__(self, filename, mode, encoding, delay) 85 86 def emit(self, record): 87 while True: 88 try: 89 WatchedFileHandler.emit(self, record) 90 self.tries = 0 91 return 92 except IOError as err: 93 if self.tries == self.max_tries: 94 raise 95 self.stream.close() 96 self.stream = self._open() 97 self.tries += 1 98 99 100class RunAborted(Exception): 101 pass 102 103 104class Result(object): 105 total = 0 106 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} 107 108 def __init__(self): 109 self.starttime = None 110 self.returncode = None 111 self.runtime = '' 112 self.stdout = [] 113 self.stderr = [] 114 self.result = '' 115 116 def done(self, proc, killed): 117 """ 118 Finalize the results of this Cmd, mapping exit codes to a 119 result code, one of: (PASS, FAIL, SKIP, KILLED, ABORTED) 120 and increment one of the four runresults[] counters. 121 """ 122 global retcode 123 124 Result.total += 1 125 m, s = divmod(time() - self.starttime, 60) 126 self.runtime = '%02d:%02d' % (m, s) 127 self.returncode = proc.returncode 128 if killed: 129 self.result = 'KILLED' 130 Result.runresults['KILLED'] += 1 131 retcode = 2; 132 elif self.returncode == 0: 133 self.result = 'PASS' 134 Result.runresults['PASS'] += 1 135 elif self.returncode in (STF_NOTINUSE, STF_UNSUPPORTED): 136 self.result = 'SKIP' 137 Result.runresults['SKIP'] += 1 138 elif self.returncode == STF_ABORTED: 139 self.result = 'ABORTED' 140 Result.runresults['FAIL'] += 1 141 retcode = 1 142 elif self.returncode != 0: 143 self.result = 'FAIL' 144 Result.runresults['FAIL'] += 1 145 retcode = 1; 146 147 148class Output(object): 149 """ 150 This class is a slightly modified version of the 'Stream' class found 151 here: http://goo.gl/aSGfv 152 """ 153 def __init__(self, stream): 154 self.stream = stream 155 self._buf = '' 156 self.lines = [] 157 158 def fileno(self): 159 return self.stream.fileno() 160 161 def read(self, drain=0): 162 """ 163 Read from the file descriptor. If 'drain' set, read until EOF. 164 """ 165 while self._read() is not None: 166 if not drain: 167 break 168 169 def _read(self): 170 """ 171 Read up to 4k of data from this output stream. Collect the output 172 up to the last newline, and append it to any leftover data from a 173 previous call. The lines are stored as a (timestamp, data) tuple 174 for easy sorting/merging later. 175 """ 176 fd = self.fileno() 177 buf = os.read(fd, 4096).decode('utf-8', errors='ignore') 178 if not buf: 179 return None 180 if '\n' not in buf: 181 self._buf += buf 182 return [] 183 184 buf = self._buf + buf 185 tmp, rest = buf.rsplit('\n', 1) 186 self._buf = rest 187 now = datetime.now() 188 rows = tmp.split('\n') 189 self.lines += [(now, r) for r in rows] 190 191 192class Cmd(object): 193 verified_users = [] 194 195 def __init__(self, pathname, identifier=None, outputdir=None, 196 timeout=None, user=None, tags=None): 197 self.pathname = pathname 198 self.identifier = identifier 199 self.outputdir = outputdir or 'BASEDIR' 200 self.timeout = timeout 201 self.user = user or '' 202 self.killed = False 203 self.result = Result() 204 205 if self.timeout is None: 206 self.timeout = 60 207 208 def __str__(self): 209 return '''\ 210Pathname: %s 211Identifier: %s 212Outputdir: %s 213Timeout: %d 214User: %s 215''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 216 217 def ensure_outputdir(self): 218 try: 219 old = os.umask(0) 220 if not os.path.isdir(self.outputdir): 221 os.makedirs(self.outputdir, mode=0o777) 222 os.umask(old) 223 except OSError as e: 224 fail('%s' % e) 225 226 def kill_cmd(self, proc): 227 """ 228 Kill a running command due to timeout, or ^C from the keyboard. If 229 sudo is required, this user was verified previously. We kill the 230 entire process group so that SIGTERM reaches the test script and 231 its children (e.g. when launched via sudo), allowing EXIT traps 232 and cleanup functions to run. 233 """ 234 self.killed = True 235 do_sudo = len(self.user) != 0 236 signal = '-TERM' 237 238 # Negative PID sends signal to the entire process group. 239 cmd = [SUDO, KILL, signal, '-%s' % str(proc.pid)] 240 if not do_sudo: 241 del cmd[0] 242 243 try: 244 kp = Popen(cmd) 245 kp.wait() 246 except: 247 pass 248 249 def update_cmd_privs(self, cmd, user): 250 """ 251 If a user has been specified to run this Cmd and we're not already 252 running as that user, prepend the appropriate sudo command to run 253 as that user. 254 """ 255 me = getpwuid(os.getuid()) 256 257 if not user or user == me.pw_name: 258 return cmd 259 260 ret = '%s -E -u %s %s' % (SUDO, user, cmd) 261 return ret.split(' ') 262 263 def collect_output(self, proc): 264 """ 265 Read from stdout/stderr as data becomes available, until the 266 process is no longer running. Return the lines from the stdout and 267 stderr Output objects. 268 """ 269 out = Output(proc.stdout) 270 err = Output(proc.stderr) 271 res = [] 272 while proc.returncode is None: 273 proc.poll() 274 res = select([out, err], [], [], .1) 275 for fd in res[0]: 276 fd.read() 277 for fd in res[0]: 278 fd.read(drain=1) 279 280 return out.lines, err.lines 281 282 def run(self, options): 283 """ 284 This is the main function that runs each individual test. 285 Determine whether or not the command requires sudo, and modify it 286 if needed. Run the command, and update the result object. 287 """ 288 if options.dryrun is True: 289 print(self) 290 return 291 292 privcmd = self.update_cmd_privs(self.pathname, self.user) 293 self.ensure_outputdir() 294 295 try: 296 self.result.starttime = time() 297 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, 298 universal_newlines=True, start_new_session=True) 299 proc.stdin.close() 300 301 # Allow a special timeout value of 0 to mean infinity 302 if int(self.timeout) == 0: 303 self.timeout = maxsize 304 t = Timer(int(self.timeout), self.kill_cmd, [proc]) 305 t.start() 306 self.result.stdout, self.result.stderr = self.collect_output(proc) 307 except KeyboardInterrupt: 308 self.kill_cmd(proc) 309 raise 310 finally: 311 t.cancel() 312 313 self.result.done(proc, self.killed) 314 315 def skip(self): 316 """ 317 Initialize enough of the test result that we can log a skipped 318 command. 319 """ 320 Result.total += 1 321 Result.runresults['SKIP'] += 1 322 self.result.stdout = self.result.stderr = [] 323 self.result.starttime = time() 324 m, s = divmod(time() - self.result.starttime, 60) 325 self.result.runtime = '%02d:%02d' % (m, s) 326 self.result.result = 'SKIP' 327 328 def not_found(self, kind='test'): 329 """ 330 Initialize enough of the test result that we can log a test whose 331 file was not found on disk. Reported as a failure. 332 """ 333 global retcode 334 self.ensure_outputdir() 335 Result.total += 1 336 Result.runresults['FAIL'] += 1 337 retcode = 1 338 self.result.stdout = [] 339 self.result.stderr = [(datetime.now(), 340 '%s file not found: %s' % 341 (kind, self.pathname))] 342 self.result.starttime = time() 343 m, s = divmod(time() - self.result.starttime, 60) 344 self.result.runtime = '%02d:%02d' % (m, s) 345 self.result.result = 'FAIL' 346 347 def log(self, logger, options): 348 """ 349 This function is responsible for writing all output. This includes 350 the console output, the logfile of all results (with timestamped 351 merged stdout and stderr), and for each test, the unmodified 352 stdout/stderr/merged in it's own file. 353 """ 354 if logger is None: 355 return 356 357 logname = getpwuid(os.getuid()).pw_name 358 user = ' (run as %s)' % (self.user if len(self.user) else logname) 359 if self.identifier: 360 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 361 else: 362 msga = 'Test: %s%s ' % (self.pathname, user) 363 msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) 364 pad = ' ' * (80 - (len(msga) + len(msgb))) 365 366 # If -q is specified, only print a line for tests that didn't pass. 367 # This means passing tests need to be logged as DEBUG, or the one 368 # line summary will only be printed in the logfile for failures. 369 if not options.quiet: 370 logger.info('%s%s%s' % (msga, pad, msgb)) 371 elif self.result.result != 'PASS': 372 logger.info('%s%s%s' % (msga, pad, msgb)) 373 else: 374 logger.debug('%s%s%s' % (msga, pad, msgb)) 375 376 lines = sorted(self.result.stdout + self.result.stderr, 377 key=lambda x: x[0]) 378 379 for dt, line in lines: 380 logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) 381 382 if len(self.result.stdout): 383 with io.open(os.path.join(self.outputdir, 'stdout'), 384 encoding='utf-8', 385 errors='surrogateescape', 386 mode='w') as out: 387 for _, line in self.result.stdout: 388 out.write('%s\n' % line) 389 if len(self.result.stderr): 390 with io.open(os.path.join(self.outputdir, 'stderr'), 391 encoding='utf-8', 392 errors='surrogateescape', 393 mode='w') as err: 394 for _, line in self.result.stderr: 395 err.write('%s\n' % line) 396 if len(self.result.stdout) and len(self.result.stderr): 397 with io.open(os.path.join(self.outputdir, 'merged'), 398 encoding='utf-8', 399 errors='surrogateescape', 400 mode='w') as merged: 401 for _, line in lines: 402 merged.write('%s\n' % line) 403 404 405class Test(Cmd): 406 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 407 'post_user', 'tags'] 408 409 def __init__(self, pathname, 410 pre=None, pre_user=None, post=None, post_user=None, 411 tags=None, **kwargs): 412 super(Test, self).__init__(pathname, **kwargs) 413 self.pre = pre or '' 414 self.pre_user = pre_user or '' 415 self.post = post or '' 416 self.post_user = post_user or '' 417 self.tags = tags or [] 418 self.missing_pre = False 419 self.missing_post = False 420 421 def __str__(self): 422 post_user = pre_user = '' 423 if len(self.pre_user): 424 pre_user = ' (as %s)' % (self.pre_user) 425 if len(self.post_user): 426 post_user = ' (as %s)' % (self.post_user) 427 return '''\ 428Pathname: %s 429Identifier: %s 430Outputdir: %s 431Timeout: %d 432User: %s 433Pre: %s%s 434Post: %s%s 435Tags: %s 436''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 437 self.pre, pre_user, self.post, post_user, self.tags) 438 439 def verify(self, logger): 440 """ 441 Check the pre/post scripts, user and Test. Omit the Test from this 442 run if there are any problems. 443 """ 444 users = [self.pre_user, self.user, self.post_user] 445 446 if len(self.pre) and not verify_file(self.pre): 447 logger.warning("pre script '%s' for test '%s' failed " 448 "verification; tests will be skipped." % 449 (self.pre, self.pathname)) 450 self.missing_pre = True 451 452 if not verify_file(self.pathname): 453 logger.warning("Test '%s' not added to this run because" 454 " it failed verification." % self.pathname) 455 return False 456 457 if len(self.post) and not verify_file(self.post): 458 logger.warning("post script '%s' for test '%s' failed " 459 "verification; tests will be skipped.." % 460 (self.post, self.pathname)) 461 self.missing_post = True 462 463 for user in [user for user in users if len(user)]: 464 if not verify_user(user, logger): 465 logger.info("Not adding Test '%s' to this run." % 466 self.pathname) 467 return False 468 469 return True 470 471 def run(self, logger, options): 472 """ 473 Create Cmd instances for the pre/post scripts. If the post script 474 is missing, refuse to run and report a FAIL for post. If the pre 475 script is missing, report a FAIL for pre and skip this test and 476 post. If the pre script exists but fails, skip this test. Run the 477 post script if pre passed or failed (but not if pre was missing). 478 """ 479 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 480 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 481 timeout=self.timeout, user=self.pre_user) 482 test = Cmd(self.pathname, identifier=self.identifier, 483 outputdir=self.outputdir, timeout=self.timeout, 484 user=self.user) 485 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 486 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 487 timeout=self.timeout, user=self.post_user) 488 489 cont = True 490 if self.missing_post: 491 # post script is missing: refuse to run, report FAIL for post. 492 posttest.not_found('post') 493 posttest.log(logger, options) 494 cont = False 495 elif self.missing_pre: 496 # pre script is missing: report FAIL for pre, skip the test. 497 pretest.not_found('pre') 498 pretest.log(logger, options) 499 cont = False 500 elif len(pretest.pathname): 501 pretest.run(options) 502 cont = pretest.result.result == 'PASS' 503 pretest.log(logger, options) 504 505 if cont: 506 test.run(options) 507 else: 508 test.skip() 509 510 test.log(logger, options) 511 if test.result.result == 'ABORTED': 512 raise RunAborted('test %s' % test.pathname) 513 514 if not self.missing_post and len(posttest.pathname): 515 if self.missing_pre: 516 posttest.skip() 517 posttest.log(logger, options) 518 else: 519 logger.debug('Running post cleanup: %s' % posttest.pathname) 520 posttest.run(options) 521 posttest.log(logger, options) 522 if posttest.result.result == 'ABORTED': 523 raise RunAborted( 524 'post cleanup for %s' % self.pathname) 525 526 527class TestGroup(Test): 528 props = Test.props + ['tests'] 529 530 def __init__(self, pathname, tests=None, **kwargs): 531 super(TestGroup, self).__init__(pathname, **kwargs) 532 self.tests = tests or [] 533 self.missing_tests = set() 534 535 def __str__(self): 536 post_user = pre_user = '' 537 if len(self.pre_user): 538 pre_user = ' (as %s)' % (self.pre_user) 539 if len(self.post_user): 540 post_user = ' (as %s)' % (self.post_user) 541 return '''\ 542Pathname: %s 543Identifier: %s 544Outputdir: %s 545Tests: %s 546Timeout: %d 547User: %s 548Pre: %s%s 549Post: %s%s 550Tags: %s 551''' % (self.pathname, self.identifier, self.outputdir, self.tests, 552 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 553 self.tags) 554 555 def filter(self, keeplist): 556 self.tests = [ x for x in self.tests if x in keeplist ] 557 558 def verify(self, logger): 559 """ 560 Check the pre/post scripts, user and tests in this TestGroup. Omit 561 the TestGroup entirely, or simply delete the relevant tests in the 562 group, if that's all that's required. 563 """ 564 # If the pre or post scripts are relative pathnames, convert to 565 # absolute, so they stand a chance of passing verification. 566 if len(self.pre) and not os.path.isabs(self.pre): 567 self.pre = os.path.join(self.pathname, self.pre) 568 if len(self.post) and not os.path.isabs(self.post): 569 self.post = os.path.join(self.pathname, self.post) 570 571 users = [self.pre_user, self.user, self.post_user] 572 573 if len(self.pre) and not verify_file(self.pre): 574 logger.warning("pre script '%s' for TestGroup '%s' failed " 575 "verification; tests will be skipped." % 576 (self.pre, self.pathname)) 577 self.missing_pre = True 578 579 if len(self.post) and not verify_file(self.post): 580 logger.warning("post script '%s' for TestGroup '%s' failed " 581 "verification." % (self.post, self.pathname)) 582 self.missing_post = True 583 584 for user in [user for user in users if len(user)]: 585 if not verify_user(user, logger): 586 logger.info("Not adding TestGroup '%s' to this run." % 587 self.pathname) 588 return False 589 590 # If one of the tests is invalid, record it as missing so it can be 591 # reported as a failure, rather than silently dropping it. 592 self.missing_tests = set(f for f in self.tests 593 if not verify_file(os.path.join(self.pathname, 594 f))) 595 596 return len(self.tests) != 0 597 598 def run(self, logger, options): 599 """ 600 Create Cmd instances for the pre/post scripts. If the post script 601 is missing, refuse to run and report a FAIL for post. If the pre 602 script is missing, report a FAIL for pre and skip all tests and 603 post. If the pre script exists but fails, skip all tests. Run the 604 post script if pre passed or failed (but not if pre was missing). 605 """ 606 # tags assigned to this test group also include the test names 607 if options.tags and not set(self.tags).intersection(set(options.tags)): 608 return 609 610 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 611 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 612 user=self.pre_user, identifier=self.identifier) 613 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 614 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 615 user=self.post_user, identifier=self.identifier) 616 617 cont = True 618 if self.missing_post: 619 # post script is missing: refuse to run, report FAIL for post. 620 posttest.not_found('post') 621 posttest.log(logger, options) 622 cont = False 623 elif self.missing_pre: 624 # pre script is missing: report FAIL for pre, skip all tests. 625 pretest.not_found('pre') 626 pretest.log(logger, options) 627 cont = False 628 elif len(pretest.pathname): 629 pretest.run(options) 630 cont = pretest.result.result == 'PASS' 631 pretest.log(logger, options) 632 633 try: 634 for fname in self.tests: 635 test = Cmd(os.path.join(self.pathname, fname), 636 outputdir=os.path.join(self.outputdir, fname), 637 timeout=self.timeout, user=self.user, 638 identifier=self.identifier) 639 if fname in self.missing_tests: 640 test.not_found() 641 elif cont: 642 test.run(options) 643 else: 644 test.skip() 645 646 test.log(logger, options) 647 if test.result.result == 'ABORTED': 648 raise RunAborted('test %s' % test.pathname) 649 except KeyboardInterrupt: 650 # Run the post (cleanup) script even when interrupted, in case 651 # the test's own EXIT trap did not clean up (e.g. pools, zinject). 652 if not self.missing_post and len(posttest.pathname): 653 print('\nInterrupted, running cleanup...', 654 flush=True) 655 posttest.run(options) 656 posttest.log(logger, options) 657 raise 658 659 if not self.missing_post and len(posttest.pathname): 660 if self.missing_pre: 661 posttest.skip() 662 posttest.log(logger, options) 663 else: 664 logger.debug('Running post cleanup: %s' % posttest.pathname) 665 posttest.run(options) 666 posttest.log(logger, options) 667 if posttest.result.result == 'ABORTED': 668 raise RunAborted( 669 'post cleanup for %s' % self.pathname) 670 671 672class TestRun(object): 673 props = ['quiet', 'outputdir'] 674 675 def __init__(self, options): 676 self.tests = {} 677 self.testgroups = {} 678 self.starttime = time() 679 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 680 self.outputdir = os.path.join(options.outputdir, self.timestamp) 681 self.logger = self.setup_logging(options) 682 self.defaults = [ 683 ('outputdir', BASEDIR), 684 ('quiet', False), 685 ('timeout', 60), 686 ('user', ''), 687 ('pre', ''), 688 ('pre_user', ''), 689 ('post', ''), 690 ('post_user', ''), 691 ('tags', []) 692 ] 693 694 def __str__(self): 695 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 696 s += 'TESTS:\n' 697 for key in sorted(self.tests.keys()): 698 s += '%s%s' % (self.tests[key].__str__(), '\n') 699 s += 'TESTGROUPS:\n' 700 for key in sorted(self.testgroups.keys()): 701 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 702 return s 703 704 def addtest(self, pathname, options): 705 """ 706 Create a new Test, and apply any properties that were passed in 707 from the command line. If it passes verification, add it to the 708 TestRun. 709 """ 710 test = Test(pathname) 711 for prop in Test.props: 712 setattr(test, prop, getattr(options, prop)) 713 714 if test.verify(self.logger): 715 self.tests[pathname] = test 716 717 def addtestgroup(self, dirname, filenames, options): 718 """ 719 Create a new TestGroup, and apply any properties that were passed 720 in from the command line. If it passes verification, add it to the 721 TestRun. 722 """ 723 if dirname not in self.testgroups: 724 testgroup = TestGroup(dirname) 725 for prop in Test.props: 726 setattr(testgroup, prop, getattr(options, prop)) 727 728 # Prevent pre/post scripts from running as regular tests 729 for f in [testgroup.pre, testgroup.post]: 730 if f in filenames: 731 del filenames[filenames.index(f)] 732 733 self.testgroups[dirname] = testgroup 734 self.testgroups[dirname].tests = sorted(filenames) 735 736 testgroup.verify(self.logger) 737 738 def filter(self, keeplist): 739 for group in list(self.testgroups.keys()): 740 if group not in keeplist: 741 del self.testgroups[group] 742 continue 743 744 g = self.testgroups[group] 745 746 if g.pre and os.path.basename(g.pre) in keeplist[group]: 747 continue 748 749 g.filter(keeplist[group]) 750 751 for test in list(self.tests.keys()): 752 directory, base = os.path.split(test) 753 if directory not in keeplist or base not in keeplist[directory]: 754 del self.tests[test] 755 756 def read(self, logger, options): 757 """ 758 Read in the specified runfile, and apply the TestRun properties 759 listed in the 'DEFAULT' section to our TestRun. Then read each 760 section, and apply the appropriate properties to the Test or 761 TestGroup. Properties from individual sections override those set 762 in the 'DEFAULT' section. If the Test or TestGroup passes 763 verification, add it to the TestRun. 764 """ 765 config = configparser.RawConfigParser() 766 parsed = config.read(options.runfiles) 767 failed = options.runfiles - set(parsed) 768 if len(failed): 769 files = ' '.join(sorted(failed)) 770 fail("Couldn't read config files: %s" % files) 771 772 for opt in TestRun.props: 773 if config.has_option('DEFAULT', opt): 774 setattr(self, opt, config.get('DEFAULT', opt)) 775 if opt == 'outputdir': 776 self.outputdir = os.path.join(self.outputdir, self.timestamp) 777 778 testdir = options.testdir 779 780 for section in config.sections(): 781 if ('arch' in config.options(section) and 782 platform.machine() != config.get(section, 'arch')): 783 continue 784 785 parts = section.split(':', 1) 786 sectiondir = parts[0] 787 identifier = parts[1] if len(parts) == 2 else None 788 if os.path.isdir(sectiondir): 789 pathname = sectiondir 790 elif os.path.isdir(os.path.join(testdir, sectiondir)): 791 pathname = os.path.join(testdir, sectiondir) 792 else: 793 pathname = sectiondir 794 795 testgroup = TestGroup(os.path.abspath(pathname), 796 identifier=identifier) 797 if 'tests' in config.options(section): 798 for prop in TestGroup.props: 799 for sect in ['DEFAULT', section]: 800 if config.has_option(sect, prop): 801 if prop == 'tags': 802 setattr(testgroup, prop, 803 eval(config.get(sect, prop))) 804 else: 805 setattr(testgroup, prop, 806 config.get(sect, prop)) 807 808 # Repopulate tests using eval to convert the string to a list 809 testgroup.tests = eval(config.get(section, 'tests')) 810 811 if testgroup.verify(logger): 812 self.testgroups[section] = testgroup 813 814 elif 'autotests' in config.options(section): 815 for prop in TestGroup.props: 816 for sect in ['DEFAULT', section]: 817 if config.has_option(sect, prop): 818 if prop == 'tags': 819 setattr(testgroup, prop, 820 eval(config.get(sect, prop))) 821 else: 822 setattr(testgroup, prop, 823 config.get(sect, prop)) 824 825 filenames = os.listdir(pathname) 826 # Only executable files starting with "tst." are tests. 827 # data files might have that prefix but are not tests. 828 filenames = [f for f in filenames if f.startswith("tst.") and 829 verify_file(os.path.join(pathname, f))] 830 testgroup.tests = sorted(filenames) 831 832 if testgroup.verify(logger): 833 self.testgroups[section] = testgroup 834 else: 835 test = Test(section) 836 for prop in Test.props: 837 for sect in ['DEFAULT', section]: 838 if config.has_option(sect, prop): 839 setattr(test, prop, config.get(sect, prop)) 840 841 if test.verify(logger): 842 self.tests[section] = test 843 844 def write(self, options): 845 """ 846 Create a configuration file for editing and later use. The 847 'DEFAULT' section of the config file is created from the 848 properties that were specified on the command line. Tests are 849 simply added as sections that inherit everything from the 850 'DEFAULT' section. TestGroups are the same, except they get an 851 option including all the tests to run in that directory. 852 """ 853 854 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 855 self.defaults]) 856 config = configparser.RawConfigParser(defaults) 857 858 for test in sorted(self.tests.keys()): 859 config.add_section(test) 860 for prop in Test.props: 861 if prop not in self.props: 862 config.set(testgroup, prop, 863 getattr(self.testgroups[testgroup], prop)) 864 865 for testgroup in sorted(self.testgroups.keys()): 866 config.add_section(testgroup) 867 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 868 for prop in TestGroup.props: 869 if prop not in self.props: 870 config.set(testgroup, prop, 871 getattr(self.testgroups[testgroup], prop)) 872 873 try: 874 with open(options.template, 'w') as f: 875 return config.write(f) 876 except IOError: 877 fail('Could not open \'%s\' for writing.' % options.template) 878 879 def complete_outputdirs(self): 880 """ 881 Collect all the pathnames for Tests, and TestGroups. Work 882 backwards one pathname component at a time, to create a unique 883 directory name in which to deposit test output. Tests will be able 884 to write output files directly in the newly modified outputdir. 885 TestGroups will be able to create one subdirectory per test in the 886 outputdir, and are guaranteed uniqueness because a group can only 887 contain files in one directory. Pre and post tests will create a 888 directory rooted at the outputdir of the Test or TestGroup in 889 question for their output. 890 """ 891 done = False 892 components = 0 893 tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) 894 total = len(tmp_dict) 895 base = self.outputdir 896 897 while not done: 898 l = [] 899 components -= 1 900 for testfile in list(tmp_dict.keys()): 901 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 902 if uniq not in l: 903 l.append(uniq) 904 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 905 else: 906 break 907 done = total == len(l) 908 909 def setup_logging(self, options): 910 """ 911 Two loggers are set up here. The first is for the logfile which 912 will contain one line summarizing the test, including the test 913 name, result, and running time. This logger will also capture the 914 timestamped combined stdout and stderr of each run. The second 915 logger is optional console output, which will contain only the one 916 line summary. The loggers are initialized at two different levels 917 to facilitate segregating the output. 918 """ 919 if options.dryrun is True: 920 return 921 922 testlogger = logging.getLogger(__name__) 923 testlogger.setLevel(logging.DEBUG) 924 925 if not options.template: 926 try: 927 old = os.umask(0) 928 os.makedirs(self.outputdir, mode=0o777) 929 os.umask(old) 930 except OSError as e: 931 fail('%s' % e) 932 filename = os.path.join(self.outputdir, 'log') 933 934 logfile = WatchedFileHandlerClosed(filename) 935 logfile.setLevel(logging.DEBUG) 936 logfilefmt = logging.Formatter('%(message)s') 937 logfile.setFormatter(logfilefmt) 938 testlogger.addHandler(logfile) 939 940 cons = logging.StreamHandler() 941 cons.setLevel(logging.INFO) 942 consfmt = logging.Formatter('%(message)s') 943 cons.setFormatter(consfmt) 944 testlogger.addHandler(cons) 945 946 return testlogger 947 948 def run(self, options): 949 """ 950 Walk through all the Tests and TestGroups, calling run(). 951 """ 952 if not options.dryrun: 953 try: 954 old = os.umask(0) 955 os.makedirs(self.outputdir, mode=0o777, exist_ok=True) 956 os.umask(old) 957 os.chdir(self.outputdir) 958 except OSError: 959 fail('Could not change to directory %s' % self.outputdir) 960 961 uname = Popen(['uname', '-a'], stdout=PIPE, stderr=PIPE, 962 universal_newlines=True) 963 out, _ = uname.communicate() 964 self.logger.debug('Test system: %s' % out.strip()) 965 966 for test in sorted(self.tests.keys()): 967 self.tests[test].run(self.logger, options) 968 try: 969 for testgroup in sorted(self.testgroups.keys()): 970 self.testgroups[testgroup].run(self.logger, options) 971 except KeyboardInterrupt: 972 fail('\nRun terminated at user request.') 973 except RunAborted as e: 974 fail('\nRun aborted: %s' % e) 975 976 def summary(self): 977 if Result.total == 0: 978 print('No tests to run') 979 return 980 981 print('\nResults Summary') 982 for key in list(Result.runresults.keys()): 983 if Result.runresults[key] != 0: 984 print('%s\t% 4d' % (key, Result.runresults[key])) 985 986 m, s = divmod(time() - self.starttime, 60) 987 h, m = divmod(m, 60) 988 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 989 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 990 float(Result.total)) * 100)) 991 print('Log directory:\t%s' % self.outputdir) 992 993 994def verify_file(pathname): 995 """ 996 Verify that the supplied pathname is an executable regular file. 997 """ 998 if os.path.isdir(pathname) or os.path.islink(pathname): 999 return False 1000 1001 if os.path.isfile(pathname) and os.access(pathname, os.X_OK): 1002 return True 1003 1004 return False 1005 1006 1007def verify_user(user, logger): 1008 """ 1009 Verify that the specified user exists on this system, and can execute 1010 sudo without being prompted for a password. 1011 """ 1012 testcmd = [SUDO, '-n', '-u', user, TRUE] 1013 1014 if user in Cmd.verified_users: 1015 return True 1016 1017 try: 1018 _ = getpwnam(user) 1019 except KeyError: 1020 logger.warning("user '%s' does not exist.", user) 1021 return False 1022 1023 p = Popen(testcmd) 1024 p.wait() 1025 if p.returncode != 0: 1026 logger.warning("user '%s' cannot use passwordless sudo.", user) 1027 return False 1028 else: 1029 Cmd.verified_users.append(user) 1030 1031 return True 1032 1033 1034def find_tests(testrun, options): 1035 """ 1036 For the given list of pathnames, add files as Tests. For directories, 1037 if do_groups is True, add the directory as a TestGroup. If False, 1038 recursively search for executable files. 1039 """ 1040 1041 for p in sorted(options.pathnames): 1042 if os.path.isdir(p): 1043 for dirname, _, filenames in os.walk(p): 1044 if options.do_groups: 1045 testrun.addtestgroup(dirname, filenames, options) 1046 else: 1047 for f in sorted(filenames): 1048 testrun.addtest(os.path.join(dirname, f), options) 1049 else: 1050 testrun.addtest(p, options) 1051 1052 1053def filter_tests(testrun, options): 1054 try: 1055 fh = open(options.logfile, "r", errors='replace') 1056 except Exception as e: 1057 fail('%s' % e) 1058 1059 failed = {} 1060 while True: 1061 line = fh.readline() 1062 if not line: 1063 break 1064 m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line) 1065 if not m: 1066 continue 1067 group, test = m.group(1, 2) 1068 m = re.match(re.escape(options.testdir) + r'(.*)', group) 1069 if m: 1070 group = m.group(1) 1071 try: 1072 failed[group].append(test) 1073 except KeyError: 1074 failed[group] = [ test ] 1075 fh.close() 1076 1077 testrun.filter(failed) 1078 1079 1080def fail(retstr, ret=1): 1081 print('%s: %s' % (argv[0], retstr)) 1082 exit(ret) 1083 1084 1085def options_cb(option, opt_str, value, parser): 1086 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 1087 1088 if opt_str in parser.rargs: 1089 fail('%s may only be specified once.' % opt_str) 1090 1091 if option.dest == 'runfiles': 1092 parser.values.cmd = 'rdconfig' 1093 value = set(os.path.abspath(p) for p in value.split(',')) 1094 if option.dest == 'tags': 1095 value = [x.strip() for x in value.split(',')] 1096 1097 if option.dest in path_options: 1098 setattr(parser.values, option.dest, os.path.abspath(value)) 1099 else: 1100 setattr(parser.values, option.dest, value) 1101 1102 1103def parse_args(): 1104 parser = OptionParser() 1105 parser.add_option('-c', action='callback', callback=options_cb, 1106 type='string', dest='runfiles', metavar='runfiles', 1107 help='Specify tests to run via config files.') 1108 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 1109 help='Dry run. Print tests, but take no other action.') 1110 parser.add_option('-l', action='callback', callback=options_cb, 1111 default=None, dest='logfile', metavar='logfile', 1112 type='string', 1113 help='Read logfile and re-run tests which failed.') 1114 parser.add_option('-g', action='store_true', default=False, 1115 dest='do_groups', help='Make directories TestGroups.') 1116 parser.add_option('-o', action='callback', callback=options_cb, 1117 default=BASEDIR, dest='outputdir', type='string', 1118 metavar='outputdir', help='Specify an output directory.') 1119 parser.add_option('-i', action='callback', callback=options_cb, 1120 default=TESTDIR, dest='testdir', type='string', 1121 metavar='testdir', help='Specify a test directory.') 1122 parser.add_option('-p', action='callback', callback=options_cb, 1123 default='', dest='pre', metavar='script', 1124 type='string', help='Specify a pre script.') 1125 parser.add_option('-P', action='callback', callback=options_cb, 1126 default='', dest='post', metavar='script', 1127 type='string', help='Specify a post script.') 1128 parser.add_option('-q', action='store_true', default=False, dest='quiet', 1129 help='Silence on the console during a test run.') 1130 parser.add_option('-t', action='callback', callback=options_cb, default=60, 1131 dest='timeout', metavar='seconds', type='int', 1132 help='Timeout (in seconds) for an individual test.') 1133 parser.add_option('-u', action='callback', callback=options_cb, 1134 default='', dest='user', metavar='user', type='string', 1135 help='Specify a different user name to run as.') 1136 parser.add_option('-w', action='callback', callback=options_cb, 1137 default=None, dest='template', metavar='template', 1138 type='string', help='Create a new config file.') 1139 parser.add_option('-x', action='callback', callback=options_cb, default='', 1140 dest='pre_user', metavar='pre_user', type='string', 1141 help='Specify a user to execute the pre script.') 1142 parser.add_option('-X', action='callback', callback=options_cb, default='', 1143 dest='post_user', metavar='post_user', type='string', 1144 help='Specify a user to execute the post script.') 1145 parser.add_option('-T', action='callback', callback=options_cb, default='', 1146 dest='tags', metavar='tags', type='string', 1147 help='Specify tags to execute specific test groups.') 1148 (options, pathnames) = parser.parse_args() 1149 1150 if options.runfiles and len(pathnames): 1151 fail('Extraneous arguments.') 1152 1153 options.pathnames = [os.path.abspath(path) for path in pathnames] 1154 1155 return options 1156 1157 1158def main(): 1159 options = parse_args() 1160 1161 testrun = TestRun(options) 1162 1163 if options.runfiles: 1164 testrun.read(testrun.logger, options) 1165 else: 1166 find_tests(testrun, options) 1167 1168 if options.logfile: 1169 filter_tests(testrun, options) 1170 1171 if options.template: 1172 testrun.write(options) 1173 exit(0) 1174 1175 testrun.complete_outputdirs() 1176 testrun.run(options) 1177 testrun.summary() 1178 exit(retcode) 1179 1180 1181if __name__ == '__main__': 1182 main() 1183