1#!/usr/bin/env @PYTHON_SHEBANG@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2012, 2018 by Delphix. All rights reserved. 16# Copyright (c) 2019 Datto Inc. 17# 18# This script must remain compatible with Python 3.6+. 19# 20 21import os 22import sys 23import ctypes 24import re 25import configparser 26 27from datetime import datetime 28from optparse import OptionParser 29from pwd import getpwnam 30from pwd import getpwuid 31from select import select 32from subprocess import PIPE 33from subprocess import Popen 34from subprocess import check_output 35from threading import Timer 36from time import time, CLOCK_MONOTONIC 37from os.path import exists 38 39BASEDIR = '/var/tmp/test_results' 40TESTDIR = '/usr/share/zfs/' 41KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak' 42KILL = 'kill' 43TRUE = 'true' 44SUDO = 'sudo' 45LOG_FILE = 'LOG_FILE' 46LOG_OUT = 'LOG_OUT' 47LOG_ERR = 'LOG_ERR' 48LOG_FILE_OBJ = None 49 50try: 51 from time import monotonic as monotonic_time 52except ImportError: 53 class timespec(ctypes.Structure): 54 _fields_ = [ 55 ('tv_sec', ctypes.c_long), 56 ('tv_nsec', ctypes.c_long) 57 ] 58 59 librt = ctypes.CDLL('librt.so.1', use_errno=True) 60 clock_gettime = librt.clock_gettime 61 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] 62 63 def monotonic_time(): 64 t = timespec() 65 if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0: 66 errno_ = ctypes.get_errno() 67 raise OSError(errno_, os.strerror(errno_)) 68 return t.tv_sec + t.tv_nsec * 1e-9 69 70 71class Result(object): 72 total = 0 73 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0} 74 75 def __init__(self): 76 self.starttime = None 77 self.returncode = None 78 self.runtime = '' 79 self.stdout = [] 80 self.stderr = [] 81 self.kmemleak = '' 82 self.result = '' 83 84 def done(self, proc, killed, reran): 85 """ 86 Finalize the results of this Cmd. 87 """ 88 Result.total += 1 89 m, s = divmod(monotonic_time() - self.starttime, 60) 90 self.runtime = '%02d:%02d' % (m, s) 91 self.returncode = proc.returncode 92 if reran is True: 93 Result.runresults['RERAN'] += 1 94 if killed: 95 self.result = 'KILLED' 96 Result.runresults['KILLED'] += 1 97 elif len(self.kmemleak) > 0: 98 self.result = 'FAIL' 99 Result.runresults['FAIL'] += 1 100 elif self.returncode == 0: 101 self.result = 'PASS' 102 Result.runresults['PASS'] += 1 103 elif self.returncode == 4: 104 self.result = 'SKIP' 105 Result.runresults['SKIP'] += 1 106 elif self.returncode != 0: 107 self.result = 'FAIL' 108 Result.runresults['FAIL'] += 1 109 110 111class Output(object): 112 """ 113 This class is a slightly modified version of the 'Stream' class found 114 here: https://stackoverflow.com/q/4984549/ 115 """ 116 def __init__(self, stream, debug=False): 117 self.stream = stream 118 self.debug = debug 119 self._buf = b'' 120 self.lines = [] 121 122 def fileno(self): 123 return self.stream.fileno() 124 125 def read(self, drain=0): 126 """ 127 Read from the file descriptor. If 'drain' set, read until EOF. 128 """ 129 while self._read() is not None: 130 if not drain: 131 break 132 133 def _read(self): 134 """ 135 Read up to 4k of data from this output stream. Collect the output 136 up to the last newline, and append it to any leftover data from a 137 previous call. The lines are stored as a (timestamp, data) tuple 138 for easy sorting/merging later. 139 """ 140 fd = self.fileno() 141 buf = os.read(fd, 4096) 142 if not buf: 143 return None 144 if self.debug: 145 os.write(sys.stderr.fileno(), buf) 146 if b'\n' not in buf: 147 self._buf += buf 148 return [] 149 150 buf = self._buf + buf 151 tmp, rest = buf.rsplit(b'\n', 1) 152 self._buf = rest 153 now = datetime.now() 154 rows = tmp.split(b'\n') 155 self.lines += [(now, r) for r in rows] 156 157 158class Cmd(object): 159 verified_users = [] 160 161 def __init__(self, pathname, identifier=None, outputdir=None, 162 timeout=None, user=None, tags=None): 163 self.pathname = pathname 164 self.identifier = identifier 165 self.outputdir = outputdir or 'BASEDIR' 166 """ 167 The timeout for tests is measured in wall-clock time 168 """ 169 self.timeout = timeout 170 self.user = user or '' 171 self.killed = False 172 self.reran = None 173 self.result = Result() 174 175 if self.timeout is None: 176 self.timeout = 60 177 178 def __str__(self): 179 return '''\ 180Pathname: %s 181Identifier: %s 182Outputdir: %s 183Timeout: %d 184User: %s 185''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 186 187 def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False): 188 """ 189 Kill a running command due to timeout, or ^C from the keyboard. If 190 sudo is required, this user was verified previously. 191 """ 192 self.killed = True 193 do_sudo = len(self.user) != 0 194 signal = '-TERM' 195 196 cmd = [SUDO, KILL, signal, str(proc.pid)] 197 if not do_sudo: 198 del cmd[0] 199 200 try: 201 kp = Popen(cmd) 202 kp.wait() 203 except Exception: 204 pass 205 206 """ 207 If this is not a user-initiated kill and the test has not been 208 reran before we consider if the test needs to be reran: 209 If the test has spent some time hibernating and didn't run the whole 210 length of time before being timed out we will rerun the test. 211 """ 212 if keyboard_interrupt is False and self.reran is None: 213 runtime = monotonic_time() - self.result.starttime 214 if int(self.timeout) > runtime: 215 self.killed = False 216 self.reran = False 217 self.run(options, dryrun=False, kmemleak=kmemleak) 218 self.reran = True 219 220 def update_cmd_privs(self, cmd, user): 221 """ 222 If a user has been specified to run this Cmd and we're not already 223 running as that user, prepend the appropriate sudo command to run 224 as that user. 225 """ 226 me = getpwuid(os.getuid()) 227 228 if not user or user is me: 229 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 230 cmd += '.ksh' 231 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 232 cmd += '.sh' 233 return cmd 234 235 if not os.path.isfile(cmd): 236 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 237 cmd += '.ksh' 238 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 239 cmd += '.sh' 240 241 # glibc (at least) will not pass TMPDIR through to setuid programs. 242 # if set, arrange for it to be reset before running the target cmd 243 tmpdir = os.getenv('TMPDIR') 244 if tmpdir: 245 tmpdirarg = 'env TMPDIR=%s' % tmpdir 246 else: 247 tmpdirarg = '' 248 249 ret = '%s -E -u %s %s %s' % (SUDO, user, tmpdirarg, cmd) 250 return ret.split(' ') 251 252 def collect_output(self, proc, debug=False): 253 """ 254 Read from stdout/stderr as data becomes available, until the 255 process is no longer running. Return the lines from the stdout and 256 stderr Output objects. 257 """ 258 out = Output(proc.stdout, debug) 259 err = Output(proc.stderr, debug) 260 res = [] 261 while proc.returncode is None: 262 proc.poll() 263 res = select([out, err], [], [], .1) 264 for fd in res[0]: 265 fd.read() 266 for fd in res[0]: 267 fd.read(drain=1) 268 269 return out.lines, err.lines 270 271 def run(self, options, dryrun=None, kmemleak=None): 272 """ 273 This is the main function that runs each individual test. 274 Determine whether or not the command requires sudo, and modify it 275 if needed. Run the command, and update the result object. 276 """ 277 if dryrun is None: 278 dryrun = options.dryrun 279 if dryrun is True: 280 print(self) 281 return 282 if kmemleak is None: 283 kmemleak = options.kmemleak 284 285 privcmd = self.update_cmd_privs(self.pathname, self.user) 286 try: 287 old = os.umask(0) 288 if not os.path.isdir(self.outputdir): 289 os.makedirs(self.outputdir, mode=0o777) 290 os.umask(old) 291 except OSError as e: 292 fail('%s' % e) 293 294 """ 295 Log each test we run to /dev/kmsg (on Linux), so if there's a kernel 296 warning we'll be able to match it up to a particular test. 297 """ 298 if options.kmsg is True and exists("/dev/kmsg"): 299 try: 300 kp = Popen([SUDO, "sh", "-c", 301 f"echo ZTS run {self.pathname} > /dev/kmsg"]) 302 kp.wait() 303 except Exception: 304 pass 305 306 """ 307 Log each test we run to /dev/ttyu0 (on FreeBSD), so if there's a kernel 308 warning we'll be able to match it up to a particular test. 309 """ 310 if options.kmsg is True and exists("/dev/ttyu0"): 311 try: 312 kp = Popen([SUDO, "sh", "-c", 313 f"echo ZTS run {self.pathname} > /dev/ttyu0"]) 314 kp.wait() 315 except Exception: 316 pass 317 318 self.result.starttime = monotonic_time() 319 320 if kmemleak: 321 cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"' 322 check_output(cmd, shell=True) 323 324 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) 325 # Allow a special timeout value of 0 to mean infinity 326 if int(self.timeout) == 0: 327 self.timeout = sys.maxsize / (10 ** 9) 328 t = Timer( 329 int(self.timeout), self.kill_cmd, [proc, options, kmemleak] 330 ) 331 332 try: 333 t.start() 334 335 out, err = self.collect_output(proc, options.debug) 336 self.result.stdout = out 337 self.result.stderr = err 338 339 if kmemleak: 340 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"' 341 check_output(cmd, shell=True) 342 cmd = f'{SUDO} cat {KMEMLEAK_FILE}' 343 self.result.kmemleak = check_output(cmd, shell=True) 344 except KeyboardInterrupt: 345 self.kill_cmd(proc, options, kmemleak, True) 346 fail('\nRun terminated at user request.') 347 finally: 348 t.cancel() 349 350 if self.reran is not False: 351 self.result.done(proc, self.killed, self.reran) 352 353 def skip(self): 354 """ 355 Initialize enough of the test result that we can log a skipped 356 command. 357 """ 358 Result.total += 1 359 Result.runresults['SKIP'] += 1 360 self.result.stdout = self.result.stderr = [] 361 self.result.starttime = monotonic_time() 362 m, s = divmod(monotonic_time() - self.result.starttime, 60) 363 self.result.runtime = '%02d:%02d' % (m, s) 364 self.result.result = 'SKIP' 365 366 def log(self, options, suppress_console=False): 367 """ 368 This function is responsible for writing all output. This includes 369 the console output, the logfile of all results (with timestamped 370 merged stdout and stderr), and for each test, the unmodified 371 stdout/stderr/merged in its own file. 372 """ 373 374 logname = getpwuid(os.getuid()).pw_name 375 rer = '' 376 if self.reran is True: 377 rer = ' (RERAN)' 378 user = ' (run as %s)' % (self.user if len(self.user) else logname) 379 if self.identifier: 380 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 381 else: 382 msga = 'Test: %s%s ' % (self.pathname, user) 383 msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer) 384 pad = ' ' * (80 - (len(msga) + len(msgb))) 385 result_line = msga + pad + msgb 386 387 # The result line is always written to the log file. If -q was 388 # specified only failures are written to the console, otherwise 389 # the result line is written to the console. The console output 390 # may be suppressed by calling log() with suppress_console=True. 391 write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE) 392 if not suppress_console: 393 if not options.quiet: 394 write_log(result_line, LOG_OUT) 395 elif options.quiet and self.result.result != 'PASS': 396 write_log(result_line, LOG_OUT) 397 398 lines = sorted(self.result.stdout + self.result.stderr, 399 key=lambda x: x[0]) 400 401 # Write timestamped output (stdout and stderr) to the logfile 402 for dt, line in lines: 403 timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11], 404 encoding='utf-8') 405 write_log(b'%s %s\n' % (timestamp, line), LOG_FILE) 406 407 # Write the separate stdout/stderr/merged files, if the data exists 408 if len(self.result.stdout): 409 with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out: 410 for _, line in self.result.stdout: 411 os.write(out.fileno(), b'%s\n' % line) 412 if len(self.result.stderr): 413 with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err: 414 for _, line in self.result.stderr: 415 os.write(err.fileno(), b'%s\n' % line) 416 if len(self.result.stdout) and len(self.result.stderr): 417 with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged: 418 for _, line in lines: 419 os.write(merged.fileno(), b'%s\n' % line) 420 if len(self.result.kmemleak): 421 with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem: 422 kmem.write(self.result.kmemleak) 423 424 425class Test(Cmd): 426 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 427 'post_user', 'failsafe', 'failsafe_user', 'tags'] 428 429 def __init__(self, pathname, 430 pre=None, pre_user=None, post=None, post_user=None, 431 failsafe=None, failsafe_user=None, tags=None, **kwargs): 432 super(Test, self).__init__(pathname, **kwargs) 433 self.pre = pre or '' 434 self.pre_user = pre_user or '' 435 self.post = post or '' 436 self.post_user = post_user or '' 437 self.failsafe = failsafe or '' 438 self.failsafe_user = failsafe_user or '' 439 self.tags = tags or [] 440 441 def __str__(self): 442 post_user = pre_user = failsafe_user = '' 443 if len(self.pre_user): 444 pre_user = ' (as %s)' % (self.pre_user) 445 if len(self.post_user): 446 post_user = ' (as %s)' % (self.post_user) 447 if len(self.failsafe_user): 448 failsafe_user = ' (as %s)' % (self.failsafe_user) 449 return '''\ 450Pathname: %s 451Identifier: %s 452Outputdir: %s 453Timeout: %d 454User: %s 455Pre: %s%s 456Post: %s%s 457Failsafe: %s%s 458Tags: %s 459''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 460 self.pre, pre_user, self.post, post_user, self.failsafe, 461 failsafe_user, self.tags) 462 463 def verify(self): 464 """ 465 Check the pre/post/failsafe scripts, user and Test. Omit the Test from 466 this run if there are any problems. 467 """ 468 files = [self.pre, self.pathname, self.post, self.failsafe] 469 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 470 471 for f in [f for f in files if len(f)]: 472 if not verify_file(f): 473 write_log("Warning: Test '%s' not added to this run because" 474 " it failed verification.\n" % f, LOG_ERR) 475 return False 476 477 for user in [user for user in users if len(user)]: 478 if not verify_user(user): 479 write_log("Not adding Test '%s' to this run.\n" % 480 self.pathname, LOG_ERR) 481 return False 482 483 return True 484 485 def run(self, options, dryrun=None, kmemleak=None): 486 """ 487 Create Cmd instances for the pre/post/failsafe scripts. If the pre 488 script doesn't pass, skip this Test. Run the post script regardless. 489 If the Test is killed, also run the failsafe script. 490 """ 491 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 492 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 493 timeout=self.timeout, user=self.pre_user) 494 test = Cmd(self.pathname, identifier=self.identifier, 495 outputdir=self.outputdir, timeout=self.timeout, 496 user=self.user) 497 odir = os.path.join(self.outputdir, os.path.basename(self.failsafe)) 498 failsafe = Cmd(self.failsafe, identifier=self.identifier, 499 outputdir=odir, timeout=self.timeout, 500 user=self.failsafe_user) 501 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 502 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 503 timeout=self.timeout, user=self.post_user) 504 505 cont = True 506 if len(pretest.pathname): 507 pretest.run(options, kmemleak=False) 508 cont = pretest.result.result == 'PASS' 509 pretest.log(options) 510 511 if cont: 512 test.run(options, kmemleak=kmemleak) 513 if test.result.result == 'KILLED' and len(failsafe.pathname): 514 failsafe.run(options, kmemleak=False) 515 failsafe.log(options, suppress_console=True) 516 else: 517 test.skip() 518 519 test.log(options) 520 521 if len(posttest.pathname): 522 posttest.run(options, kmemleak=False) 523 posttest.log(options) 524 525 526class TestGroup(Test): 527 props = Test.props + ['tests'] 528 529 def __init__(self, pathname, tests=None, **kwargs): 530 super(TestGroup, self).__init__(pathname, **kwargs) 531 self.tests = tests or [] 532 533 def __str__(self): 534 post_user = pre_user = failsafe_user = '' 535 if len(self.pre_user): 536 pre_user = ' (as %s)' % (self.pre_user) 537 if len(self.post_user): 538 post_user = ' (as %s)' % (self.post_user) 539 if len(self.failsafe_user): 540 failsafe_user = ' (as %s)' % (self.failsafe_user) 541 return '''\ 542Pathname: %s 543Identifier: %s 544Outputdir: %s 545Tests: %s 546Timeout: %s 547User: %s 548Pre: %s%s 549Post: %s%s 550Failsafe: %s%s 551Tags: %s 552''' % (self.pathname, self.identifier, self.outputdir, self.tests, 553 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 554 self.failsafe, failsafe_user, self.tags) 555 556 def filter(self, keeplist): 557 self.tests = [x for x in self.tests if x in keeplist] 558 559 def verify(self): 560 """ 561 Check the pre/post/failsafe scripts, user and tests in this TestGroup. 562 Omit the TestGroup entirely, or simply delete the relevant tests in the 563 group, if that's all that's required. 564 """ 565 # If the pre/post/failsafe scripts are relative pathnames, convert to 566 # absolute, so they stand a chance of passing verification. 567 if len(self.pre) and not os.path.isabs(self.pre): 568 self.pre = os.path.join(self.pathname, self.pre) 569 if len(self.post) and not os.path.isabs(self.post): 570 self.post = os.path.join(self.pathname, self.post) 571 if len(self.failsafe) and not os.path.isabs(self.failsafe): 572 self.post = os.path.join(self.pathname, self.post) 573 574 auxfiles = [self.pre, self.post, self.failsafe] 575 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 576 577 for f in [f for f in auxfiles if len(f)]: 578 if f != self.failsafe and self.pathname != os.path.dirname(f): 579 write_log("Warning: TestGroup '%s' not added to this run. " 580 "Auxiliary script '%s' exists in a different " 581 "directory.\n" % (self.pathname, f), LOG_ERR) 582 return False 583 584 if not verify_file(f): 585 write_log("Warning: TestGroup '%s' not added to this run. " 586 "Auxiliary script '%s' failed verification.\n" % 587 (self.pathname, f), LOG_ERR) 588 return False 589 590 for user in [user for user in users if len(user)]: 591 if not verify_user(user): 592 write_log("Not adding TestGroup '%s' to this run.\n" % 593 self.pathname, LOG_ERR) 594 return False 595 596 # If one of the tests is invalid, delete it, log it, and drive on. 597 for test in self.tests: 598 if not verify_file(os.path.join(self.pathname, test)): 599 del self.tests[self.tests.index(test)] 600 write_log("Warning: Test '%s' removed from TestGroup '%s' " 601 "because it failed verification.\n" % 602 (test, self.pathname), LOG_ERR) 603 604 return len(self.tests) != 0 605 606 def run(self, options, dryrun=None, kmemleak=None): 607 """ 608 Create Cmd instances for the pre/post/failsafe scripts. If the pre 609 script doesn't pass, skip all the tests in this TestGroup. Run the 610 post script regardless. Run the failsafe script when a test is killed. 611 """ 612 # tags assigned to this test group also include the test names 613 if options.tags and not set(self.tags).intersection(set(options.tags)): 614 return 615 616 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 617 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 618 user=self.pre_user, identifier=self.identifier) 619 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 620 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 621 user=self.post_user, identifier=self.identifier) 622 623 cont = True 624 if len(pretest.pathname): 625 pretest.run(options, dryrun=dryrun, kmemleak=False) 626 cont = pretest.result.result == 'PASS' 627 pretest.log(options) 628 629 for fname in self.tests: 630 odir = os.path.join(self.outputdir, fname) 631 test = Cmd(os.path.join(self.pathname, fname), outputdir=odir, 632 timeout=self.timeout, user=self.user, 633 identifier=self.identifier) 634 odir = os.path.join(odir, os.path.basename(self.failsafe)) 635 failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout, 636 user=self.failsafe_user, identifier=self.identifier) 637 if cont: 638 test.run(options, dryrun=dryrun, kmemleak=kmemleak) 639 if test.result.result == 'KILLED' and len(failsafe.pathname): 640 failsafe.run(options, dryrun=dryrun, kmemleak=False) 641 failsafe.log(options, suppress_console=True) 642 else: 643 test.skip() 644 645 test.log(options) 646 647 if len(posttest.pathname): 648 posttest.run(options, dryrun=dryrun, kmemleak=False) 649 posttest.log(options) 650 651 652class TestRun(object): 653 props = ['quiet', 'outputdir', 'debug'] 654 655 def __init__(self, options): 656 self.tests = {} 657 self.testgroups = {} 658 self.starttime = time() 659 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 660 self.outputdir = os.path.join(options.outputdir, self.timestamp) 661 self.setup_logging(options) 662 self.defaults = [ 663 ('outputdir', BASEDIR), 664 ('quiet', False), 665 ('timeout', 60), 666 ('user', ''), 667 ('pre', ''), 668 ('pre_user', ''), 669 ('post', ''), 670 ('post_user', ''), 671 ('failsafe', ''), 672 ('failsafe_user', ''), 673 ('tags', []), 674 ('debug', False) 675 ] 676 677 def __str__(self): 678 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 679 s += 'TESTS:\n' 680 for key in sorted(self.tests.keys()): 681 s += '%s%s' % (self.tests[key].__str__(), '\n') 682 s += 'TESTGROUPS:\n' 683 for key in sorted(self.testgroups.keys()): 684 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 685 return s 686 687 def addtest(self, pathname, options): 688 """ 689 Create a new Test, and apply any properties that were passed in 690 from the command line. If it passes verification, add it to the 691 TestRun. 692 """ 693 test = Test(pathname) 694 for prop in Test.props: 695 setattr(test, prop, getattr(options, prop)) 696 697 if test.verify(): 698 self.tests[pathname] = test 699 700 def addtestgroup(self, dirname, filenames, options): 701 """ 702 Create a new TestGroup, and apply any properties that were passed 703 in from the command line. If it passes verification, add it to the 704 TestRun. 705 """ 706 if dirname not in self.testgroups: 707 testgroup = TestGroup(dirname) 708 for prop in Test.props: 709 setattr(testgroup, prop, getattr(options, prop)) 710 711 # Prevent pre/post/failsafe scripts from running as regular tests 712 for f in [testgroup.pre, testgroup.post, testgroup.failsafe]: 713 if f in filenames: 714 del filenames[filenames.index(f)] 715 716 self.testgroups[dirname] = testgroup 717 self.testgroups[dirname].tests = sorted(filenames) 718 719 testgroup.verify() 720 721 def filter(self, keeplist): 722 for group in list(self.testgroups.keys()): 723 if group not in keeplist: 724 del self.testgroups[group] 725 continue 726 727 g = self.testgroups[group] 728 729 if g.pre and os.path.basename(g.pre) in keeplist[group]: 730 continue 731 732 g.filter(keeplist[group]) 733 734 for test in list(self.tests.keys()): 735 directory, base = os.path.split(test) 736 if directory not in keeplist or base not in keeplist[directory]: 737 del self.tests[test] 738 739 def read(self, options): 740 """ 741 Read in the specified runfiles, and apply the TestRun properties 742 listed in the 'DEFAULT' section to our TestRun. Then read each 743 section, and apply the appropriate properties to the Test or 744 TestGroup. Properties from individual sections override those set 745 in the 'DEFAULT' section. If the Test or TestGroup passes 746 verification, add it to the TestRun. 747 """ 748 config = configparser.RawConfigParser() 749 parsed = config.read(options.runfiles) 750 failed = options.runfiles - set(parsed) 751 if len(failed): 752 files = ' '.join(sorted(failed)) 753 fail("Couldn't read config files: %s" % files) 754 755 for opt in TestRun.props: 756 if config.has_option('DEFAULT', opt): 757 if opt == 'outputdir': 758 outputdir = config.get('DEFAULT', opt) 759 setattr(self, opt, os.path.join(outputdir, self.timestamp)) 760 else: 761 setattr(self, opt, config.get('DEFAULT', opt)) 762 763 testdir = options.testdir 764 765 for section in config.sections(): 766 if 'tests' in config.options(section): 767 parts = section.split(':', 1) 768 sectiondir = parts[0] 769 identifier = parts[1] if len(parts) == 2 else None 770 if os.path.isdir(sectiondir): 771 pathname = sectiondir 772 elif os.path.isdir(os.path.join(testdir, sectiondir)): 773 pathname = os.path.join(testdir, sectiondir) 774 else: 775 pathname = sectiondir 776 777 testgroup = TestGroup(os.path.abspath(pathname), 778 identifier=identifier) 779 for prop in TestGroup.props: 780 for sect in ['DEFAULT', section]: 781 if config.has_option(sect, prop): 782 if prop == 'tags': 783 setattr(testgroup, prop, 784 eval(config.get(sect, prop))) 785 elif prop == 'failsafe': 786 failsafe = config.get(sect, prop) 787 setattr(testgroup, prop, 788 os.path.join(testdir, failsafe)) 789 elif prop == 'outputdir': 790 outputdir = config.get(sect, prop) 791 setattr(self, opt, 792 os.path.join(outputdir, 793 self.timestamp)) 794 else: 795 setattr(testgroup, prop, 796 config.get(sect, prop)) 797 798 # Repopulate tests using eval to convert the string to a list 799 testgroup.tests = eval(config.get(section, 'tests')) 800 801 if testgroup.verify(): 802 self.testgroups[section] = testgroup 803 else: 804 test = Test(section) 805 for prop in Test.props: 806 for sect in ['DEFAULT', section]: 807 if config.has_option(sect, prop): 808 if prop == 'failsafe': 809 failsafe = config.get(sect, prop) 810 setattr(test, prop, 811 os.path.join(testdir, failsafe)) 812 elif prop == 'outputdir': 813 outputdir = config.get(sect, prop) 814 setattr(self, opt, 815 os.path.join(outputdir, 816 self.timestamp)) 817 else: 818 setattr(test, prop, config.get(sect, prop)) 819 820 if test.verify(): 821 self.tests[section] = test 822 823 def write(self, options): 824 """ 825 Create a configuration file for editing and later use. The 826 'DEFAULT' section of the config file is created from the 827 properties that were specified on the command line. Tests are 828 simply added as sections that inherit everything from the 829 'DEFAULT' section. TestGroups are the same, except they get an 830 option including all the tests to run in that directory. 831 """ 832 833 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 834 self.defaults]) 835 config = configparser.RawConfigParser(defaults) 836 837 for test in sorted(self.tests.keys()): 838 config.add_section(test) 839 for prop in Test.props: 840 if prop not in self.props: 841 config.set(test, prop, 842 getattr(self.tests[test], prop)) 843 844 for testgroup in sorted(self.testgroups.keys()): 845 config.add_section(testgroup) 846 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 847 for prop in TestGroup.props: 848 if prop not in self.props: 849 config.set(testgroup, prop, 850 getattr(self.testgroups[testgroup], prop)) 851 852 try: 853 with open(options.template, 'w') as f: 854 return config.write(f) 855 except IOError: 856 fail('Could not open \'%s\' for writing.' % options.template) 857 858 def complete_outputdirs(self): 859 """ 860 Collect all the pathnames for Tests, and TestGroups. Work 861 backwards one pathname component at a time, to create a unique 862 directory name in which to deposit test output. Tests will be able 863 to write output files directly in the newly modified outputdir. 864 TestGroups will be able to create one subdirectory per test in the 865 outputdir, and are guaranteed uniqueness because a group can only 866 contain files in one directory. Pre and post tests will create a 867 directory rooted at the outputdir of the Test or TestGroup in 868 question for their output. Failsafe scripts will create a directory 869 rooted at the outputdir of each Test for their output. 870 """ 871 done = False 872 components = 0 873 tmp_dict = dict(list(self.tests.items()) + 874 list(self.testgroups.items())) 875 total = len(tmp_dict) 876 base = self.outputdir 877 878 while not done: 879 paths = [] 880 components -= 1 881 for testfile in list(tmp_dict.keys()): 882 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 883 if uniq not in paths: 884 paths.append(uniq) 885 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 886 else: 887 break 888 done = total == len(paths) 889 890 def setup_logging(self, options): 891 """ 892 This function creates the output directory and gets a file object 893 for the logfile. This function must be called before write_log() 894 can be used. 895 """ 896 if options.dryrun is True: 897 return 898 899 global LOG_FILE_OBJ 900 if not options.template: 901 try: 902 old = os.umask(0) 903 os.makedirs(self.outputdir, mode=0o777) 904 os.umask(old) 905 filename = os.path.join(self.outputdir, 'log') 906 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') 907 except OSError as e: 908 fail('%s' % e) 909 910 def run(self, options): 911 """ 912 Walk through all the Tests and TestGroups, calling run(). 913 """ 914 try: 915 os.chdir(self.outputdir) 916 except OSError: 917 fail('Could not change to directory %s' % self.outputdir) 918 # make a symlink to the output for the currently running test 919 logsymlink = os.path.join(self.outputdir, '../current') 920 if os.path.islink(logsymlink): 921 os.unlink(logsymlink) 922 if not os.path.exists(logsymlink): 923 os.symlink(self.outputdir, logsymlink) 924 else: 925 write_log('Could not make a symlink to directory %s\n' % 926 self.outputdir, LOG_ERR) 927 928 if options.kmemleak: 929 cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"' 930 check_output(cmd, shell=True) 931 932 iteration = 0 933 while iteration < options.iterations: 934 for test in sorted(self.tests.keys()): 935 self.tests[test].run(options) 936 for testgroup in sorted(self.testgroups.keys()): 937 self.testgroups[testgroup].run(options) 938 iteration += 1 939 940 def summary(self): 941 if Result.total == 0: 942 return 2 943 944 print('\nResults Summary') 945 for key in list(Result.runresults.keys()): 946 if Result.runresults[key] != 0: 947 print('%s\t% 4d' % (key, Result.runresults[key])) 948 949 m, s = divmod(time() - self.starttime, 60) 950 h, m = divmod(m, 60) 951 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 952 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 953 float(Result.total)) * 100)) 954 print('Log directory:\t%s' % self.outputdir) 955 956 if Result.runresults['FAIL'] > 0: 957 return 1 958 959 if Result.runresults['KILLED'] > 0: 960 return 1 961 962 if Result.runresults['RERAN'] > 0: 963 return 3 964 965 return 0 966 967 968def write_log(msg, target): 969 """ 970 Write the provided message to standard out, standard error or 971 the logfile. If specifying LOG_FILE, then `msg` must be a bytes 972 like object. This way we can still handle output from tests that 973 may be in unexpected encodings. 974 """ 975 if target == LOG_OUT: 976 os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8')) 977 elif target == LOG_ERR: 978 os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8')) 979 elif target == LOG_FILE: 980 os.write(LOG_FILE_OBJ.fileno(), msg) 981 else: 982 fail('log_msg called with unknown target "%s"' % target) 983 984 985def verify_file(pathname): 986 """ 987 Verify that the supplied pathname is an executable regular file. 988 """ 989 if os.path.isdir(pathname) or os.path.islink(pathname): 990 return False 991 992 for ext in '', '.ksh', '.sh': 993 script_path = pathname + ext 994 if os.path.isfile(script_path) and os.access(script_path, os.X_OK): 995 return True 996 997 return False 998 999 1000def verify_user(user): 1001 """ 1002 Verify that the specified user exists on this system, and can execute 1003 sudo without being prompted for a password. 1004 """ 1005 testcmd = [SUDO, '-n', '-u', user, TRUE] 1006 1007 if user in Cmd.verified_users: 1008 return True 1009 1010 try: 1011 getpwnam(user) 1012 except KeyError: 1013 write_log("Warning: user '%s' does not exist.\n" % user, 1014 LOG_ERR) 1015 return False 1016 1017 p = Popen(testcmd) 1018 p.wait() 1019 if p.returncode != 0: 1020 write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user, 1021 LOG_ERR) 1022 return False 1023 else: 1024 Cmd.verified_users.append(user) 1025 1026 return True 1027 1028 1029def find_tests(testrun, options): 1030 """ 1031 For the given list of pathnames, add files as Tests. For directories, 1032 if do_groups is True, add the directory as a TestGroup. If False, 1033 recursively search for executable files. 1034 """ 1035 1036 for p in sorted(options.pathnames): 1037 if os.path.isdir(p): 1038 for dirname, _, filenames in os.walk(p): 1039 if options.do_groups: 1040 testrun.addtestgroup(dirname, filenames, options) 1041 else: 1042 for f in sorted(filenames): 1043 testrun.addtest(os.path.join(dirname, f), options) 1044 else: 1045 testrun.addtest(p, options) 1046 1047 1048def filter_tests(testrun, options): 1049 try: 1050 fh = open(options.logfile, "r") 1051 except Exception as e: 1052 fail('%s' % e) 1053 1054 failed = {} 1055 while True: 1056 line = fh.readline() 1057 if not line: 1058 break 1059 m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line) 1060 if not m: 1061 continue 1062 group, test = m.group(1, 2) 1063 try: 1064 failed[group].append(test) 1065 except KeyError: 1066 failed[group] = [test] 1067 fh.close() 1068 1069 testrun.filter(failed) 1070 1071 1072def fail(retstr, ret=1): 1073 print('%s: %s' % (sys.argv[0], retstr)) 1074 exit(ret) 1075 1076 1077def kmemleak_cb(option, opt_str, value, parser): 1078 if not os.path.exists(KMEMLEAK_FILE): 1079 fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " + 1080 "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.") 1081 1082 setattr(parser.values, option.dest, True) 1083 1084 1085def options_cb(option, opt_str, value, parser): 1086 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 1087 1088 if opt_str in parser.rargs: 1089 fail('%s may only be specified once.' % opt_str) 1090 1091 if option.dest == 'runfiles': 1092 parser.values.cmd = 'rdconfig' 1093 value = set(os.path.abspath(p) for p in value.split(',')) 1094 if option.dest == 'tags': 1095 value = [x.strip() for x in value.split(',')] 1096 1097 if option.dest in path_options: 1098 setattr(parser.values, option.dest, os.path.abspath(value)) 1099 else: 1100 setattr(parser.values, option.dest, value) 1101 1102 1103def parse_args(): 1104 parser = OptionParser() 1105 parser.add_option('-c', action='callback', callback=options_cb, 1106 type='string', dest='runfiles', metavar='runfiles', 1107 help='Specify tests to run via config files.') 1108 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 1109 help='Dry run. Print tests, but take no other action.') 1110 parser.add_option('-D', action='store_true', default=False, dest='debug', 1111 help='Write all test output to stdout as it arrives.') 1112 parser.add_option('-l', action='callback', callback=options_cb, 1113 default=None, dest='logfile', metavar='logfile', 1114 type='string', 1115 help='Read logfile and re-run tests which failed.') 1116 parser.add_option('-g', action='store_true', default=False, 1117 dest='do_groups', help='Make directories TestGroups.') 1118 parser.add_option('-o', action='callback', callback=options_cb, 1119 default=BASEDIR, dest='outputdir', type='string', 1120 metavar='outputdir', help='Specify an output directory.') 1121 parser.add_option('-i', action='callback', callback=options_cb, 1122 default=TESTDIR, dest='testdir', type='string', 1123 metavar='testdir', help='Specify a test directory.') 1124 parser.add_option('-K', action='store_true', default=False, dest='kmsg', 1125 help='Log tests names to /dev/kmsg') 1126 parser.add_option('-m', action='callback', callback=kmemleak_cb, 1127 default=False, dest='kmemleak', 1128 help='Enable kmemleak reporting (Linux only)') 1129 parser.add_option('-p', action='callback', callback=options_cb, 1130 default='', dest='pre', metavar='script', 1131 type='string', help='Specify a pre script.') 1132 parser.add_option('-P', action='callback', callback=options_cb, 1133 default='', dest='post', metavar='script', 1134 type='string', help='Specify a post script.') 1135 parser.add_option('-q', action='store_true', default=False, dest='quiet', 1136 help='Silence on the console during a test run.') 1137 parser.add_option('-s', action='callback', callback=options_cb, 1138 default='', dest='failsafe', metavar='script', 1139 type='string', help='Specify a failsafe script.') 1140 parser.add_option('-S', action='callback', callback=options_cb, 1141 default='', dest='failsafe_user', 1142 metavar='failsafe_user', type='string', 1143 help='Specify a user to execute the failsafe script.') 1144 parser.add_option('-t', action='callback', callback=options_cb, default=60, 1145 dest='timeout', metavar='seconds', type='int', 1146 help='Timeout (in seconds) for an individual test.') 1147 parser.add_option('-u', action='callback', callback=options_cb, 1148 default='', dest='user', metavar='user', type='string', 1149 help='Specify a different user name to run as.') 1150 parser.add_option('-w', action='callback', callback=options_cb, 1151 default=None, dest='template', metavar='template', 1152 type='string', help='Create a new config file.') 1153 parser.add_option('-x', action='callback', callback=options_cb, default='', 1154 dest='pre_user', metavar='pre_user', type='string', 1155 help='Specify a user to execute the pre script.') 1156 parser.add_option('-X', action='callback', callback=options_cb, default='', 1157 dest='post_user', metavar='post_user', type='string', 1158 help='Specify a user to execute the post script.') 1159 parser.add_option('-T', action='callback', callback=options_cb, default='', 1160 dest='tags', metavar='tags', type='string', 1161 help='Specify tags to execute specific test groups.') 1162 parser.add_option('-I', action='callback', callback=options_cb, default=1, 1163 dest='iterations', metavar='iterations', type='int', 1164 help='Number of times to run the test run.') 1165 (options, pathnames) = parser.parse_args() 1166 1167 if options.runfiles and len(pathnames): 1168 fail('Extraneous arguments.') 1169 1170 options.pathnames = [os.path.abspath(path) for path in pathnames] 1171 1172 return options 1173 1174 1175def main(): 1176 options = parse_args() 1177 1178 testrun = TestRun(options) 1179 1180 if options.runfiles: 1181 testrun.read(options) 1182 else: 1183 find_tests(testrun, options) 1184 1185 if options.logfile: 1186 filter_tests(testrun, options) 1187 1188 if options.template: 1189 testrun.write(options) 1190 exit(0) 1191 1192 testrun.complete_outputdirs() 1193 testrun.run(options) 1194 exit(testrun.summary()) 1195 1196 1197if __name__ == '__main__': 1198 main() 1199