1#!@PYTHON@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2012, 2016 by Delphix. All rights reserved. 16# Copyright (c) 2017, Chris Fraire <cfraire@me.com>. 17# Copyright 2019 Joyent, Inc. 18# Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 19# Copyright 2026 Gordon W. Ross 20# 21 22from __future__ import print_function 23import sys 24PY3 = sys.version_info[0] == 3 25 26if PY3: 27 import configparser 28else: 29 import ConfigParser as configparser 30 31import io 32import os 33import logging 34import platform 35import re 36from logging.handlers import WatchedFileHandler 37from datetime import datetime 38from optparse import OptionParser 39from pwd import getpwnam 40from pwd import getpwuid 41from select import select 42from subprocess import PIPE 43from subprocess import Popen 44from sys import argv 45from sys import exit 46from sys import maxsize 47from threading import Timer 48from time import time 49 50BASEDIR = '/var/tmp/test_results' 51TESTDIR = '/opt/zfs-tests/' 52KILL = '/usr/bin/kill' 53TRUE = '/usr/bin/true' 54SUDO = '/usr/bin/sudo' 55 56# Exit codes as defined in stf/include/stf.shlib 57STF_PASS = 0 58STF_FAIL = 1 59STF_UNRESOLVED = 2 60STF_NOTINUSE = 3 61STF_UNSUPPORTED = 4 62STF_UNTESTED = 5 63STF_UNINITIATED = 6 64STF_NORESULT = 7 65STF_WARNING = 8 66STF_TIMED_OUT = 9 67STF_ABORTED = 10 68STF_OTHER = 11 69 70retcode = 0 71 72# Custom class to reopen the log file in case it is forcibly closed by a test. 73class WatchedFileHandlerClosed(WatchedFileHandler): 74 """Watch files, including closed files. 75 Similar to (and inherits from) logging.handler.WatchedFileHandler, 76 except that IOErrors are handled by reopening the stream and retrying. 77 This will be retried up to a configurable number of times before 78 giving up, default 5. 79 """ 80 81 def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5): 82 self.max_tries = max_tries 83 self.tries = 0 84 WatchedFileHandler.__init__(self, filename, mode, encoding, delay) 85 86 def emit(self, record): 87 while True: 88 try: 89 WatchedFileHandler.emit(self, record) 90 self.tries = 0 91 return 92 except IOError as err: 93 if self.tries == self.max_tries: 94 raise 95 self.stream.close() 96 self.stream = self._open() 97 self.tries += 1 98 99 100class RunAborted(Exception): 101 pass 102 103 104class Result(object): 105 total = 0 106 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0} 107 108 def __init__(self): 109 self.starttime = None 110 self.returncode = None 111 self.runtime = '' 112 self.stdout = [] 113 self.stderr = [] 114 self.result = '' 115 116 def done(self, proc, killed): 117 """ 118 Finalize the results of this Cmd, mapping exit codes to a 119 result code, one of: (PASS, FAIL, SKIP, KILLED, ABORTED) 120 and increment one of the four runresults[] counters. 121 """ 122 global retcode 123 124 Result.total += 1 125 m, s = divmod(time() - self.starttime, 60) 126 self.runtime = '%02d:%02d' % (m, s) 127 self.returncode = proc.returncode 128 if killed: 129 self.result = 'KILLED' 130 Result.runresults['KILLED'] += 1 131 retcode = 2; 132 elif self.returncode == 0: 133 self.result = 'PASS' 134 Result.runresults['PASS'] += 1 135 elif self.returncode in (STF_NOTINUSE, STF_UNSUPPORTED): 136 self.result = 'SKIP' 137 Result.runresults['SKIP'] += 1 138 elif self.returncode == STF_ABORTED: 139 self.result = 'ABORTED' 140 Result.runresults['FAIL'] += 1 141 retcode = 1 142 elif self.returncode != 0: 143 self.result = 'FAIL' 144 Result.runresults['FAIL'] += 1 145 retcode = 1; 146 147 148class Output(object): 149 """ 150 This class is a slightly modified version of the 'Stream' class found 151 here: http://goo.gl/aSGfv 152 """ 153 def __init__(self, stream): 154 self.stream = stream 155 self._buf = '' 156 self.lines = [] 157 158 def fileno(self): 159 return self.stream.fileno() 160 161 def read(self, drain=0): 162 """ 163 Read from the file descriptor. If 'drain' set, read until EOF. 164 """ 165 while self._read() is not None: 166 if not drain: 167 break 168 169 def _read(self): 170 """ 171 Read up to 4k of data from this output stream. Collect the output 172 up to the last newline, and append it to any leftover data from a 173 previous call. The lines are stored as a (timestamp, data) tuple 174 for easy sorting/merging later. 175 """ 176 fd = self.fileno() 177 buf = os.read(fd, 4096).decode('utf-8', errors='ignore') 178 if not buf: 179 return None 180 if '\n' not in buf: 181 self._buf += buf 182 return [] 183 184 buf = self._buf + buf 185 tmp, rest = buf.rsplit('\n', 1) 186 self._buf = rest 187 now = datetime.now() 188 rows = tmp.split('\n') 189 self.lines += [(now, r) for r in rows] 190 191 192class Cmd(object): 193 verified_users = [] 194 195 def __init__(self, pathname, identifier=None, outputdir=None, 196 timeout=None, user=None, tags=None): 197 self.pathname = pathname 198 self.identifier = identifier 199 self.outputdir = outputdir or 'BASEDIR' 200 self.timeout = timeout 201 self.user = user or '' 202 self.killed = False 203 self.result = Result() 204 205 if self.timeout is None: 206 self.timeout = 60 207 208 def __str__(self): 209 return '''\ 210Pathname: %s 211Identifier: %s 212Outputdir: %s 213Timeout: %d 214User: %s 215''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 216 217 def ensure_outputdir(self): 218 try: 219 old = os.umask(0) 220 if not os.path.isdir(self.outputdir): 221 os.makedirs(self.outputdir, mode=0o777) 222 os.umask(old) 223 except OSError as e: 224 fail('%s' % e) 225 226 def kill_cmd(self, proc): 227 """ 228 Kill a running command due to timeout, or ^C from the keyboard. If 229 sudo is required, this user was verified previously. We kill the 230 entire process group so that SIGTERM reaches the test script and 231 its children (e.g. when launched via sudo), allowing EXIT traps 232 and cleanup functions to run. 233 """ 234 self.killed = True 235 do_sudo = len(self.user) != 0 236 signal = '-TERM' 237 238 # Negative PID sends signal to the entire process group. 239 cmd = [SUDO, KILL, signal, '-%s' % str(proc.pid)] 240 if not do_sudo: 241 del cmd[0] 242 243 try: 244 kp = Popen(cmd) 245 kp.wait() 246 except: 247 pass 248 249 def update_cmd_privs(self, cmd, user): 250 """ 251 If a user has been specified to run this Cmd and we're not already 252 running as that user, prepend the appropriate sudo command to run 253 as that user. 254 """ 255 me = getpwuid(os.getuid()) 256 257 if not user or user == me.pw_name: 258 return cmd 259 260 ret = '%s -E -u %s %s' % (SUDO, user, cmd) 261 return ret.split(' ') 262 263 def collect_output(self, proc): 264 """ 265 Read from stdout/stderr as data becomes available, until the 266 process is no longer running. Return the lines from the stdout and 267 stderr Output objects. 268 """ 269 out = Output(proc.stdout) 270 err = Output(proc.stderr) 271 res = [] 272 while proc.returncode is None: 273 proc.poll() 274 res = select([out, err], [], [], .1) 275 for fd in res[0]: 276 fd.read() 277 for fd in res[0]: 278 fd.read(drain=1) 279 280 return out.lines, err.lines 281 282 def run(self, options): 283 """ 284 This is the main function that runs each individual test. 285 Determine whether or not the command requires sudo, and modify it 286 if needed. Run the command, and update the result object. 287 """ 288 if options.dryrun is True: 289 print(self) 290 return 291 292 privcmd = self.update_cmd_privs(self.pathname, self.user) 293 self.ensure_outputdir() 294 295 try: 296 self.result.starttime = time() 297 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, 298 universal_newlines=True, start_new_session=True) 299 proc.stdin.close() 300 301 # Allow a special timeout value of 0 to mean infinity 302 if int(self.timeout) == 0: 303 self.timeout = maxsize 304 t = Timer(int(self.timeout), self.kill_cmd, [proc]) 305 t.start() 306 self.result.stdout, self.result.stderr = self.collect_output(proc) 307 except KeyboardInterrupt: 308 self.kill_cmd(proc) 309 raise 310 finally: 311 t.cancel() 312 313 self.result.done(proc, self.killed) 314 315 def skip(self): 316 """ 317 Initialize enough of the test result that we can log a skipped 318 command. 319 """ 320 Result.total += 1 321 Result.runresults['SKIP'] += 1 322 self.result.stdout = self.result.stderr = [] 323 self.result.starttime = time() 324 m, s = divmod(time() - self.result.starttime, 60) 325 self.result.runtime = '%02d:%02d' % (m, s) 326 self.result.result = 'SKIP' 327 328 def not_found(self, kind='test'): 329 """ 330 Initialize enough of the test result that we can log a test whose 331 file was not found on disk. Reported as a failure. 332 """ 333 global retcode 334 self.ensure_outputdir() 335 Result.total += 1 336 Result.runresults['FAIL'] += 1 337 retcode = 1 338 self.result.stdout = [] 339 self.result.stderr = [(datetime.now(), 340 '%s file not found: %s' % 341 (kind, self.pathname))] 342 self.result.starttime = time() 343 m, s = divmod(time() - self.result.starttime, 60) 344 self.result.runtime = '%02d:%02d' % (m, s) 345 self.result.result = 'FAIL' 346 347 def log(self, logger, options): 348 """ 349 This function is responsible for writing all output. This includes 350 the console output, the logfile of all results (with timestamped 351 merged stdout and stderr), and for each test, the unmodified 352 stdout/stderr/merged in it's own file. 353 """ 354 if logger is None: 355 return 356 357 logname = getpwuid(os.getuid()).pw_name 358 user = ' (run as %s)' % (self.user if len(self.user) else logname) 359 if self.identifier: 360 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 361 else: 362 msga = 'Test: %s%s ' % (self.pathname, user) 363 msgb = '[%s] [%s]' % (self.result.runtime, self.result.result) 364 pad = ' ' * (80 - (len(msga) + len(msgb))) 365 366 # If -q is specified, only print a line for tests that didn't pass. 367 # This means passing tests need to be logged as DEBUG, or the one 368 # line summary will only be printed in the logfile for failures. 369 if not options.quiet: 370 logger.info('%s%s%s' % (msga, pad, msgb)) 371 elif self.result.result != 'PASS': 372 logger.info('%s%s%s' % (msga, pad, msgb)) 373 else: 374 logger.debug('%s%s%s' % (msga, pad, msgb)) 375 376 lines = sorted(self.result.stdout + self.result.stderr, 377 key=lambda x: x[0]) 378 379 for dt, line in lines: 380 logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line)) 381 382 if len(self.result.stdout): 383 with io.open(os.path.join(self.outputdir, 'stdout'), 384 encoding='utf-8', 385 errors='surrogateescape', 386 mode='w') as out: 387 for _, line in self.result.stdout: 388 out.write('%s\n' % line) 389 if len(self.result.stderr): 390 with io.open(os.path.join(self.outputdir, 'stderr'), 391 encoding='utf-8', 392 errors='surrogateescape', 393 mode='w') as err: 394 for _, line in self.result.stderr: 395 err.write('%s\n' % line) 396 if len(self.result.stdout) and len(self.result.stderr): 397 with io.open(os.path.join(self.outputdir, 'merged'), 398 encoding='utf-8', 399 errors='surrogateescape', 400 mode='w') as merged: 401 for _, line in lines: 402 merged.write('%s\n' % line) 403 404 405class Test(Cmd): 406 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 407 'post_user', 'tags'] 408 409 def __init__(self, pathname, 410 pre=None, pre_user=None, post=None, post_user=None, 411 tags=None, **kwargs): 412 super(Test, self).__init__(pathname, **kwargs) 413 self.pre = pre or '' 414 self.pre_user = pre_user or '' 415 self.post = post or '' 416 self.post_user = post_user or '' 417 self.tags = tags or [] 418 self.missing_pre = False 419 self.missing_post = False 420 421 def __str__(self): 422 post_user = pre_user = '' 423 if len(self.pre_user): 424 pre_user = ' (as %s)' % (self.pre_user) 425 if len(self.post_user): 426 post_user = ' (as %s)' % (self.post_user) 427 return '''\ 428Pathname: %s 429Identifier: %s 430Outputdir: %s 431Timeout: %d 432User: %s 433Pre: %s%s 434Post: %s%s 435Tags: %s 436''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 437 self.pre, pre_user, self.post, post_user, self.tags) 438 439 def verify(self, logger): 440 """ 441 Check the pre/post scripts, user and Test. Omit the Test from this 442 run if there are any problems. 443 """ 444 users = [self.pre_user, self.user, self.post_user] 445 446 if len(self.pre) and not verify_file(self.pre): 447 logger.warning("pre script '%s' for test '%s' failed " 448 "verification; tests will be skipped." % 449 (self.pre, self.pathname)) 450 self.missing_pre = True 451 452 if not verify_file(self.pathname): 453 logger.warning("Test '%s' not added to this run because" 454 " it failed verification." % self.pathname) 455 return False 456 457 if len(self.post) and not verify_file(self.post): 458 logger.warning("post script '%s' for test '%s' failed " 459 "verification; tests will be skipped.." % 460 (self.post, self.pathname)) 461 self.missing_post = True 462 463 for user in [user for user in users if len(user)]: 464 if not verify_user(user, logger): 465 logger.info("Not adding Test '%s' to this run." % 466 self.pathname) 467 return False 468 469 return True 470 471 def run(self, logger, options): 472 """ 473 Create Cmd instances for the pre/post scripts. If the post script 474 is missing, refuse to run and report a FAIL for post. If the pre 475 script is missing, report a FAIL for pre and skip this test and 476 post. If the pre script exists but fails, skip this test. Run the 477 post script if pre passed or failed (but not if pre was missing). 478 """ 479 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 480 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 481 timeout=self.timeout, user=self.pre_user) 482 test = Cmd(self.pathname, identifier=self.identifier, 483 outputdir=self.outputdir, timeout=self.timeout, 484 user=self.user) 485 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 486 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 487 timeout=self.timeout, user=self.post_user) 488 489 cont = True 490 if self.missing_post: 491 # post script is missing: refuse to run, report FAIL for post. 492 posttest.not_found('post') 493 posttest.log(logger, options) 494 cont = False 495 elif self.missing_pre: 496 # pre script is missing: report FAIL for pre, skip the test. 497 pretest.not_found('pre') 498 pretest.log(logger, options) 499 cont = False 500 elif len(pretest.pathname): 501 pretest.run(options) 502 cont = pretest.result.result == 'PASS' 503 pretest.log(logger, options) 504 505 if cont: 506 test.run(options) 507 else: 508 test.skip() 509 510 test.log(logger, options) 511 if test.result.result == 'ABORTED': 512 raise RunAborted('test %s' % test.pathname) 513 514 if not self.missing_post and len(posttest.pathname): 515 if self.missing_pre: 516 posttest.skip() 517 posttest.log(logger, options) 518 else: 519 logger.debug('Running post cleanup: %s' % posttest.pathname) 520 posttest.run(options) 521 posttest.log(logger, options) 522 if posttest.result.result == 'ABORTED': 523 raise RunAborted( 524 'post cleanup for %s' % self.pathname) 525 526 527class TestGroup(Test): 528 props = Test.props + ['tests'] 529 530 def __init__(self, pathname, tests=None, **kwargs): 531 super(TestGroup, self).__init__(pathname, **kwargs) 532 self.tests = tests or [] 533 self.missing_tests = set() 534 535 def __str__(self): 536 post_user = pre_user = '' 537 if len(self.pre_user): 538 pre_user = ' (as %s)' % (self.pre_user) 539 if len(self.post_user): 540 post_user = ' (as %s)' % (self.post_user) 541 return '''\ 542Pathname: %s 543Identifier: %s 544Outputdir: %s 545Tests: %s 546Timeout: %d 547User: %s 548Pre: %s%s 549Post: %s%s 550Tags: %s 551''' % (self.pathname, self.identifier, self.outputdir, self.tests, 552 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 553 self.tags) 554 555 def filter(self, keeplist): 556 self.tests = [ x for x in self.tests if x in keeplist ] 557 558 def verify(self, logger): 559 """ 560 Check the pre/post scripts, user and tests in this TestGroup. Omit 561 the TestGroup entirely, or simply delete the relevant tests in the 562 group, if that's all that's required. 563 """ 564 # If the pre or post scripts are relative pathnames, convert to 565 # absolute, so they stand a chance of passing verification. 566 if len(self.pre) and not os.path.isabs(self.pre): 567 self.pre = os.path.join(self.pathname, self.pre) 568 if len(self.post) and not os.path.isabs(self.post): 569 self.post = os.path.join(self.pathname, self.post) 570 571 users = [self.pre_user, self.user, self.post_user] 572 573 if len(self.pre) and not verify_file(self.pre): 574 logger.warning("pre script '%s' for TestGroup '%s' failed " 575 "verification; tests will be skipped." % 576 (self.pre, self.pathname)) 577 self.missing_pre = True 578 579 if len(self.post) and not verify_file(self.post): 580 logger.warning("post script '%s' for TestGroup '%s' failed " 581 "verification." % (self.post, self.pathname)) 582 self.missing_post = True 583 584 for user in [user for user in users if len(user)]: 585 if not verify_user(user, logger): 586 logger.info("Not adding TestGroup '%s' to this run." % 587 self.pathname) 588 return False 589 590 # If one of the tests is invalid, record it as missing so it can be 591 # reported as a failure, rather than silently dropping it. 592 self.missing_tests = set(f for f in self.tests 593 if not verify_file(os.path.join(self.pathname, 594 f))) 595 596 return len(self.tests) != 0 597 598 def run(self, logger, options): 599 """ 600 Create Cmd instances for the pre/post scripts. If the post script 601 is missing, refuse to run and report a FAIL for post. If the pre 602 script is missing, report a FAIL for pre and skip all tests and 603 post. If the pre script exists but fails, skip all tests. Run the 604 post script if pre passed or failed (but not if pre was missing). 605 """ 606 # tags assigned to this test group also include the test names 607 if options.tags and not set(self.tags).intersection(set(options.tags)): 608 return 609 610 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 611 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 612 user=self.pre_user, identifier=self.identifier) 613 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 614 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 615 user=self.post_user, identifier=self.identifier) 616 617 cont = True 618 if self.missing_post: 619 # post script is missing: refuse to run, report FAIL for post. 620 posttest.not_found('post') 621 posttest.log(logger, options) 622 cont = False 623 elif self.missing_pre: 624 # pre script is missing: report FAIL for pre, skip all tests. 625 pretest.not_found('pre') 626 pretest.log(logger, options) 627 cont = False 628 elif len(pretest.pathname): 629 pretest.run(options) 630 cont = pretest.result.result == 'PASS' 631 pretest.log(logger, options) 632 633 try: 634 for fname in self.tests: 635 test = Cmd(os.path.join(self.pathname, fname), 636 outputdir=os.path.join(self.outputdir, fname), 637 timeout=self.timeout, user=self.user, 638 identifier=self.identifier) 639 if fname in self.missing_tests: 640 test.not_found() 641 elif cont: 642 test.run(options) 643 else: 644 test.skip() 645 646 test.log(logger, options) 647 if test.result.result == 'ABORTED': 648 raise RunAborted('test %s' % test.pathname) 649 except KeyboardInterrupt: 650 # Run the post (cleanup) script even when interrupted, in case 651 # the test's own EXIT trap did not clean up (e.g. pools, zinject). 652 if not self.missing_post and len(posttest.pathname): 653 print('\nInterrupted, running cleanup...', 654 flush=True) 655 posttest.run(options) 656 posttest.log(logger, options) 657 raise 658 659 if not self.missing_post and len(posttest.pathname): 660 if self.missing_pre: 661 posttest.skip() 662 posttest.log(logger, options) 663 else: 664 logger.debug('Running post cleanup: %s' % posttest.pathname) 665 posttest.run(options) 666 posttest.log(logger, options) 667 if posttest.result.result == 'ABORTED': 668 raise RunAborted( 669 'post cleanup for %s' % self.pathname) 670 671 672class TestRun(object): 673 props = ['quiet', 'outputdir'] 674 675 def __init__(self, options): 676 self.tests = {} 677 self.testgroups = {} 678 self.starttime = time() 679 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 680 self.outputdir = os.path.join(options.outputdir, self.timestamp) 681 self.logger = self.setup_logging(options) 682 self.defaults = [ 683 ('outputdir', BASEDIR), 684 ('quiet', False), 685 ('timeout', 60), 686 ('user', ''), 687 ('pre', ''), 688 ('pre_user', ''), 689 ('post', ''), 690 ('post_user', ''), 691 ('tags', []) 692 ] 693 694 def __str__(self): 695 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 696 s += 'TESTS:\n' 697 for key in sorted(self.tests.keys()): 698 s += '%s%s' % (self.tests[key].__str__(), '\n') 699 s += 'TESTGROUPS:\n' 700 for key in sorted(self.testgroups.keys()): 701 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 702 return s 703 704 def addtest(self, pathname, options): 705 """ 706 Create a new Test, and apply any properties that were passed in 707 from the command line. If it passes verification, add it to the 708 TestRun. 709 """ 710 test = Test(pathname) 711 for prop in Test.props: 712 setattr(test, prop, getattr(options, prop)) 713 714 if test.verify(self.logger): 715 self.tests[pathname] = test 716 717 def addtestgroup(self, dirname, filenames, options): 718 """ 719 Create a new TestGroup, and apply any properties that were passed 720 in from the command line. If it passes verification, add it to the 721 TestRun. 722 """ 723 if dirname not in self.testgroups: 724 testgroup = TestGroup(dirname) 725 for prop in Test.props: 726 setattr(testgroup, prop, getattr(options, prop)) 727 728 # Prevent pre/post scripts from running as regular tests 729 for f in [testgroup.pre, testgroup.post]: 730 if f in filenames: 731 del filenames[filenames.index(f)] 732 733 self.testgroups[dirname] = testgroup 734 self.testgroups[dirname].tests = sorted(filenames) 735 736 testgroup.verify(self.logger) 737 738 def filter(self, keeplist): 739 for group in list(self.testgroups.keys()): 740 if group not in keeplist: 741 del self.testgroups[group] 742 continue 743 744 g = self.testgroups[group] 745 746 if g.pre and os.path.basename(g.pre) in keeplist[group]: 747 continue 748 749 g.filter(keeplist[group]) 750 751 for test in list(self.tests.keys()): 752 directory, base = os.path.split(test) 753 if directory not in keeplist or base not in keeplist[directory]: 754 del self.tests[test] 755 756 def read(self, logger, options): 757 """ 758 Read in the specified runfile, and apply the TestRun properties 759 listed in the 'DEFAULT' section to our TestRun. Then read each 760 section, and apply the appropriate properties to the Test or 761 TestGroup. Properties from individual sections override those set 762 in the 'DEFAULT' section. If the Test or TestGroup passes 763 verification, add it to the TestRun. 764 """ 765 config = configparser.RawConfigParser() 766 parsed = config.read(options.runfiles) 767 failed = options.runfiles - set(parsed) 768 if len(failed): 769 files = ' '.join(sorted(failed)) 770 fail("Couldn't read config files: %s" % files) 771 772 for opt in TestRun.props: 773 if config.has_option('DEFAULT', opt): 774 setattr(self, opt, config.get('DEFAULT', opt)) 775 self.outputdir = os.path.join(self.outputdir, self.timestamp) 776 777 testdir = options.testdir 778 779 for section in config.sections(): 780 if ('arch' in config.options(section) and 781 platform.machine() != config.get(section, 'arch')): 782 continue 783 784 parts = section.split(':', 1) 785 sectiondir = parts[0] 786 identifier = parts[1] if len(parts) == 2 else None 787 if os.path.isdir(sectiondir): 788 pathname = sectiondir 789 elif os.path.isdir(os.path.join(testdir, sectiondir)): 790 pathname = os.path.join(testdir, sectiondir) 791 else: 792 pathname = sectiondir 793 794 testgroup = TestGroup(os.path.abspath(pathname), 795 identifier=identifier) 796 if 'tests' in config.options(section): 797 for prop in TestGroup.props: 798 for sect in ['DEFAULT', section]: 799 if config.has_option(sect, prop): 800 if prop == 'tags': 801 setattr(testgroup, prop, 802 eval(config.get(sect, prop))) 803 else: 804 setattr(testgroup, prop, 805 config.get(sect, prop)) 806 807 # Repopulate tests using eval to convert the string to a list 808 testgroup.tests = eval(config.get(section, 'tests')) 809 810 if testgroup.verify(logger): 811 self.testgroups[section] = testgroup 812 813 elif 'autotests' in config.options(section): 814 for prop in TestGroup.props: 815 for sect in ['DEFAULT', section]: 816 if config.has_option(sect, prop): 817 if prop == 'tags': 818 setattr(testgroup, prop, 819 eval(config.get(sect, prop))) 820 else: 821 setattr(testgroup, prop, 822 config.get(sect, prop)) 823 824 filenames = os.listdir(pathname) 825 # Only executable files starting with "tst." are tests. 826 # data files might have that prefix but are not tests. 827 filenames = [f for f in filenames if f.startswith("tst.") and 828 verify_file(os.path.join(pathname, f))] 829 testgroup.tests = sorted(filenames) 830 831 if testgroup.verify(logger): 832 self.testgroups[section] = testgroup 833 else: 834 test = Test(section) 835 for prop in Test.props: 836 for sect in ['DEFAULT', section]: 837 if config.has_option(sect, prop): 838 setattr(test, prop, config.get(sect, prop)) 839 840 if test.verify(logger): 841 self.tests[section] = test 842 843 def write(self, options): 844 """ 845 Create a configuration file for editing and later use. The 846 'DEFAULT' section of the config file is created from the 847 properties that were specified on the command line. Tests are 848 simply added as sections that inherit everything from the 849 'DEFAULT' section. TestGroups are the same, except they get an 850 option including all the tests to run in that directory. 851 """ 852 853 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 854 self.defaults]) 855 config = configparser.RawConfigParser(defaults) 856 857 for test in sorted(self.tests.keys()): 858 config.add_section(test) 859 for prop in Test.props: 860 if prop not in self.props: 861 config.set(testgroup, prop, 862 getattr(self.testgroups[testgroup], prop)) 863 864 for testgroup in sorted(self.testgroups.keys()): 865 config.add_section(testgroup) 866 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 867 for prop in TestGroup.props: 868 if prop not in self.props: 869 config.set(testgroup, prop, 870 getattr(self.testgroups[testgroup], prop)) 871 872 try: 873 with open(options.template, 'w') as f: 874 return config.write(f) 875 except IOError: 876 fail('Could not open \'%s\' for writing.' % options.template) 877 878 def complete_outputdirs(self): 879 """ 880 Collect all the pathnames for Tests, and TestGroups. Work 881 backwards one pathname component at a time, to create a unique 882 directory name in which to deposit test output. Tests will be able 883 to write output files directly in the newly modified outputdir. 884 TestGroups will be able to create one subdirectory per test in the 885 outputdir, and are guaranteed uniqueness because a group can only 886 contain files in one directory. Pre and post tests will create a 887 directory rooted at the outputdir of the Test or TestGroup in 888 question for their output. 889 """ 890 done = False 891 components = 0 892 tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items())) 893 total = len(tmp_dict) 894 base = self.outputdir 895 896 while not done: 897 l = [] 898 components -= 1 899 for testfile in list(tmp_dict.keys()): 900 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 901 if uniq not in l: 902 l.append(uniq) 903 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 904 else: 905 break 906 done = total == len(l) 907 908 def setup_logging(self, options): 909 """ 910 Two loggers are set up here. The first is for the logfile which 911 will contain one line summarizing the test, including the test 912 name, result, and running time. This logger will also capture the 913 timestamped combined stdout and stderr of each run. The second 914 logger is optional console output, which will contain only the one 915 line summary. The loggers are initialized at two different levels 916 to facilitate segregating the output. 917 """ 918 if options.dryrun is True: 919 return 920 921 testlogger = logging.getLogger(__name__) 922 testlogger.setLevel(logging.DEBUG) 923 924 if not options.template: 925 try: 926 old = os.umask(0) 927 os.makedirs(self.outputdir, mode=0o777) 928 os.umask(old) 929 except OSError as e: 930 fail('%s' % e) 931 filename = os.path.join(self.outputdir, 'log') 932 933 logfile = WatchedFileHandlerClosed(filename) 934 logfile.setLevel(logging.DEBUG) 935 logfilefmt = logging.Formatter('%(message)s') 936 logfile.setFormatter(logfilefmt) 937 testlogger.addHandler(logfile) 938 939 cons = logging.StreamHandler() 940 cons.setLevel(logging.INFO) 941 consfmt = logging.Formatter('%(message)s') 942 cons.setFormatter(consfmt) 943 testlogger.addHandler(cons) 944 945 return testlogger 946 947 def run(self, options): 948 """ 949 Walk through all the Tests and TestGroups, calling run(). 950 """ 951 if not options.dryrun: 952 try: 953 os.chdir(self.outputdir) 954 except OSError: 955 fail('Could not change to directory %s' % self.outputdir) 956 957 uname = Popen(['uname', '-a'], stdout=PIPE, stderr=PIPE, 958 universal_newlines=True) 959 out, _ = uname.communicate() 960 self.logger.debug('Test system: %s' % out.strip()) 961 962 for test in sorted(self.tests.keys()): 963 self.tests[test].run(self.logger, options) 964 try: 965 for testgroup in sorted(self.testgroups.keys()): 966 self.testgroups[testgroup].run(self.logger, options) 967 except KeyboardInterrupt: 968 fail('\nRun terminated at user request.') 969 except RunAborted as e: 970 fail('\nRun aborted: %s' % e) 971 972 def summary(self): 973 if Result.total == 0: 974 print('No tests to run') 975 return 976 977 print('\nResults Summary') 978 for key in list(Result.runresults.keys()): 979 if Result.runresults[key] != 0: 980 print('%s\t% 4d' % (key, Result.runresults[key])) 981 982 m, s = divmod(time() - self.starttime, 60) 983 h, m = divmod(m, 60) 984 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 985 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 986 float(Result.total)) * 100)) 987 print('Log directory:\t%s' % self.outputdir) 988 989 990def verify_file(pathname): 991 """ 992 Verify that the supplied pathname is an executable regular file. 993 """ 994 if os.path.isdir(pathname) or os.path.islink(pathname): 995 return False 996 997 if os.path.isfile(pathname) and os.access(pathname, os.X_OK): 998 return True 999 1000 return False 1001 1002 1003def verify_user(user, logger): 1004 """ 1005 Verify that the specified user exists on this system, and can execute 1006 sudo without being prompted for a password. 1007 """ 1008 testcmd = [SUDO, '-n', '-u', user, TRUE] 1009 1010 if user in Cmd.verified_users: 1011 return True 1012 1013 try: 1014 _ = getpwnam(user) 1015 except KeyError: 1016 logger.warning("user '%s' does not exist.", user) 1017 return False 1018 1019 p = Popen(testcmd) 1020 p.wait() 1021 if p.returncode != 0: 1022 logger.warning("user '%s' cannot use passwordless sudo.", user) 1023 return False 1024 else: 1025 Cmd.verified_users.append(user) 1026 1027 return True 1028 1029 1030def find_tests(testrun, options): 1031 """ 1032 For the given list of pathnames, add files as Tests. For directories, 1033 if do_groups is True, add the directory as a TestGroup. If False, 1034 recursively search for executable files. 1035 """ 1036 1037 for p in sorted(options.pathnames): 1038 if os.path.isdir(p): 1039 for dirname, _, filenames in os.walk(p): 1040 if options.do_groups: 1041 testrun.addtestgroup(dirname, filenames, options) 1042 else: 1043 for f in sorted(filenames): 1044 testrun.addtest(os.path.join(dirname, f), options) 1045 else: 1046 testrun.addtest(p, options) 1047 1048 1049def filter_tests(testrun, options): 1050 try: 1051 fh = open(options.logfile, "r", errors='replace') 1052 except Exception as e: 1053 fail('%s' % e) 1054 1055 failed = {} 1056 while True: 1057 line = fh.readline() 1058 if not line: 1059 break 1060 m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line) 1061 if not m: 1062 continue 1063 group, test = m.group(1, 2) 1064 m = re.match(re.escape(options.testdir) + r'(.*)', group) 1065 if m: 1066 group = m.group(1) 1067 try: 1068 failed[group].append(test) 1069 except KeyError: 1070 failed[group] = [ test ] 1071 fh.close() 1072 1073 testrun.filter(failed) 1074 1075 1076def fail(retstr, ret=1): 1077 print('%s: %s' % (argv[0], retstr)) 1078 exit(ret) 1079 1080 1081def options_cb(option, opt_str, value, parser): 1082 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 1083 1084 if opt_str in parser.rargs: 1085 fail('%s may only be specified once.' % opt_str) 1086 1087 if option.dest == 'runfiles': 1088 parser.values.cmd = 'rdconfig' 1089 value = set(os.path.abspath(p) for p in value.split(',')) 1090 if option.dest == 'tags': 1091 value = [x.strip() for x in value.split(',')] 1092 1093 if option.dest in path_options: 1094 setattr(parser.values, option.dest, os.path.abspath(value)) 1095 else: 1096 setattr(parser.values, option.dest, value) 1097 1098 1099def parse_args(): 1100 parser = OptionParser() 1101 parser.add_option('-c', action='callback', callback=options_cb, 1102 type='string', dest='runfiles', metavar='runfiles', 1103 help='Specify tests to run via config files.') 1104 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 1105 help='Dry run. Print tests, but take no other action.') 1106 parser.add_option('-l', action='callback', callback=options_cb, 1107 default=None, dest='logfile', metavar='logfile', 1108 type='string', 1109 help='Read logfile and re-run tests which failed.') 1110 parser.add_option('-g', action='store_true', default=False, 1111 dest='do_groups', help='Make directories TestGroups.') 1112 parser.add_option('-o', action='callback', callback=options_cb, 1113 default=BASEDIR, dest='outputdir', type='string', 1114 metavar='outputdir', help='Specify an output directory.') 1115 parser.add_option('-i', action='callback', callback=options_cb, 1116 default=TESTDIR, dest='testdir', type='string', 1117 metavar='testdir', help='Specify a test directory.') 1118 parser.add_option('-p', action='callback', callback=options_cb, 1119 default='', dest='pre', metavar='script', 1120 type='string', help='Specify a pre script.') 1121 parser.add_option('-P', action='callback', callback=options_cb, 1122 default='', dest='post', metavar='script', 1123 type='string', help='Specify a post script.') 1124 parser.add_option('-q', action='store_true', default=False, dest='quiet', 1125 help='Silence on the console during a test run.') 1126 parser.add_option('-t', action='callback', callback=options_cb, default=60, 1127 dest='timeout', metavar='seconds', type='int', 1128 help='Timeout (in seconds) for an individual test.') 1129 parser.add_option('-u', action='callback', callback=options_cb, 1130 default='', dest='user', metavar='user', type='string', 1131 help='Specify a different user name to run as.') 1132 parser.add_option('-w', action='callback', callback=options_cb, 1133 default=None, dest='template', metavar='template', 1134 type='string', help='Create a new config file.') 1135 parser.add_option('-x', action='callback', callback=options_cb, default='', 1136 dest='pre_user', metavar='pre_user', type='string', 1137 help='Specify a user to execute the pre script.') 1138 parser.add_option('-X', action='callback', callback=options_cb, default='', 1139 dest='post_user', metavar='post_user', type='string', 1140 help='Specify a user to execute the post script.') 1141 parser.add_option('-T', action='callback', callback=options_cb, default='', 1142 dest='tags', metavar='tags', type='string', 1143 help='Specify tags to execute specific test groups.') 1144 (options, pathnames) = parser.parse_args() 1145 1146 if options.runfiles and len(pathnames): 1147 fail('Extraneous arguments.') 1148 1149 options.pathnames = [os.path.abspath(path) for path in pathnames] 1150 1151 return options 1152 1153 1154def main(): 1155 options = parse_args() 1156 1157 testrun = TestRun(options) 1158 1159 if options.runfiles: 1160 testrun.read(testrun.logger, options) 1161 else: 1162 find_tests(testrun, options) 1163 1164 if options.logfile: 1165 filter_tests(testrun, options) 1166 1167 if options.template: 1168 testrun.write(options) 1169 exit(0) 1170 1171 testrun.complete_outputdirs() 1172 testrun.run(options) 1173 testrun.summary() 1174 exit(retcode) 1175 1176 1177if __name__ == '__main__': 1178 main() 1179