1#!/usr/bin/env @PYTHON_SHEBANG@ 2# SPDX-License-Identifier: CDDL-1.0 3 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# http://www.illumos.org/license/CDDL. 13# 14 15# 16# Copyright (c) 2012, 2018 by Delphix. All rights reserved. 17# Copyright (c) 2019 Datto Inc. 18# Copyright (c) 2025, Klara, Inc. 19# 20# This script must remain compatible with Python 3.6+. 21# 22 23import os 24import sys 25import ctypes 26import re 27import configparser 28 29from datetime import datetime 30from optparse import OptionParser 31from pwd import getpwnam 32from pwd import getpwuid 33from select import select 34from subprocess import PIPE 35from subprocess import Popen 36from subprocess import check_output 37from threading import Timer 38from time import time, CLOCK_MONOTONIC 39from os.path import exists 40 41BASEDIR = '/var/tmp/test_results' 42TESTDIR = '/usr/share/zfs/' 43KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak' 44KILL = 'kill' 45TRUE = 'true' 46SUDO = 'sudo' 47LOG_FILE = 'LOG_FILE' 48LOG_OUT = 'LOG_OUT' 49LOG_ERR = 'LOG_ERR' 50LOG_FILE_OBJ = None 51 52try: 53 from time import monotonic as monotonic_time 54except ImportError: 55 class timespec(ctypes.Structure): 56 _fields_ = [ 57 ('tv_sec', ctypes.c_long), 58 ('tv_nsec', ctypes.c_long) 59 ] 60 61 librt = ctypes.CDLL('librt.so.1', use_errno=True) 62 clock_gettime = librt.clock_gettime 63 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] 64 65 def monotonic_time(): 66 t = timespec() 67 if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0: 68 errno_ = ctypes.get_errno() 69 raise OSError(errno_, os.strerror(errno_)) 70 return t.tv_sec + t.tv_nsec * 1e-9 71 72 73class Result(object): 74 total = 0 75 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0} 76 77 def __init__(self): 78 self.starttime = None 79 self.returncode = None 80 self.runtime = '' 81 self.stdout = [] 82 self.stderr = [] 83 self.kmemleak = '' 84 self.result = '' 85 86 def done(self, proc, killed, reran): 87 """ 88 Finalize the results of this Cmd. 89 """ 90 Result.total += 1 91 m, s = divmod(monotonic_time() - self.starttime, 60) 92 self.runtime = '%02d:%02d' % (m, s) 93 self.returncode = proc.returncode 94 if reran is True: 95 Result.runresults['RERAN'] += 1 96 if killed: 97 self.result = 'KILLED' 98 Result.runresults['KILLED'] += 1 99 elif len(self.kmemleak) > 0: 100 self.result = 'FAIL' 101 Result.runresults['FAIL'] += 1 102 elif self.returncode == 0: 103 self.result = 'PASS' 104 Result.runresults['PASS'] += 1 105 elif self.returncode == 4: 106 self.result = 'SKIP' 107 Result.runresults['SKIP'] += 1 108 elif self.returncode != 0: 109 self.result = 'FAIL' 110 Result.runresults['FAIL'] += 1 111 112 113class Output(object): 114 """ 115 This class is a slightly modified version of the 'Stream' class found 116 here: https://stackoverflow.com/q/4984549/ 117 """ 118 def __init__(self, stream, debug=False): 119 self.stream = stream 120 self.debug = debug 121 self._buf = b'' 122 self.lines = [] 123 124 def fileno(self): 125 return self.stream.fileno() 126 127 def read(self, drain=0): 128 """ 129 Read from the file descriptor. If 'drain' set, read until EOF. 130 """ 131 while self._read() is not None: 132 if not drain: 133 break 134 135 def _read(self): 136 """ 137 Read up to 4k of data from this output stream. Collect the output 138 up to the last newline, and append it to any leftover data from a 139 previous call. The lines are stored as a (timestamp, data) tuple 140 for easy sorting/merging later. 141 """ 142 fd = self.fileno() 143 buf = os.read(fd, 4096) 144 if not buf: 145 return None 146 if self.debug: 147 os.write(sys.stderr.fileno(), buf) 148 if b'\n' not in buf: 149 self._buf += buf 150 return [] 151 152 buf = self._buf + buf 153 tmp, rest = buf.rsplit(b'\n', 1) 154 self._buf = rest 155 now = datetime.now() 156 rows = tmp.split(b'\n') 157 self.lines += [(now, r) for r in rows] 158 159 160class Cmd(object): 161 verified_users = [] 162 163 def __init__(self, pathname, identifier=None, outputdir=None, 164 timeout=None, user=None, tags=None): 165 self.pathname = pathname 166 self.identifier = identifier 167 self.outputdir = outputdir or 'BASEDIR' 168 """ 169 The timeout for tests is measured in wall-clock time 170 """ 171 self.timeout = timeout 172 self.user = user or '' 173 self.killed = False 174 self.reran = None 175 self.result = Result() 176 177 if self.timeout is None: 178 self.timeout = 60 179 180 def __str__(self): 181 return '''\ 182Pathname: %s 183Identifier: %s 184Outputdir: %s 185Timeout: %d 186User: %s 187''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 188 189 def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False): 190 """ 191 Kill a running command due to timeout, or ^C from the keyboard. If 192 sudo is required, this user was verified previously. 193 """ 194 self.killed = True 195 do_sudo = len(self.user) != 0 196 signal = '-TERM' 197 198 cmd = [SUDO, KILL, signal, str(proc.pid)] 199 if not do_sudo: 200 del cmd[0] 201 202 try: 203 kp = Popen(cmd) 204 kp.wait() 205 except Exception: 206 pass 207 208 """ 209 If this is not a user-initiated kill and the test has not been 210 reran before we consider if the test needs to be reran: 211 If the test has spent some time hibernating and didn't run the whole 212 length of time before being timed out we will rerun the test. 213 """ 214 if keyboard_interrupt is False and self.reran is None: 215 runtime = monotonic_time() - self.result.starttime 216 if int(self.timeout) > runtime: 217 self.killed = False 218 self.reran = False 219 self.run(options, dryrun=False, kmemleak=kmemleak) 220 self.reran = True 221 222 def update_cmd_privs(self, cmd, user): 223 """ 224 If a user has been specified to run this Cmd and we're not already 225 running as that user, prepend the appropriate sudo command to run 226 as that user. 227 """ 228 me = getpwuid(os.getuid()) 229 230 if not user or user is me: 231 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 232 cmd += '.ksh' 233 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 234 cmd += '.sh' 235 return cmd 236 237 if not os.path.isfile(cmd): 238 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 239 cmd += '.ksh' 240 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 241 cmd += '.sh' 242 243 # glibc (at least) will not pass TMPDIR through to setuid programs. 244 # if set, arrange for it to be reset before running the target cmd 245 tmpdir = os.getenv('TMPDIR') 246 if tmpdir: 247 tmpdirarg = 'env TMPDIR=%s' % tmpdir 248 else: 249 tmpdirarg = '' 250 251 ret = '%s -E -u %s %s %s' % (SUDO, user, tmpdirarg, cmd) 252 return ret.split(' ') 253 254 def collect_output(self, proc, debug=False): 255 """ 256 Read from stdout/stderr as data becomes available, until the 257 process is no longer running. Return the lines from the stdout and 258 stderr Output objects. 259 """ 260 out = Output(proc.stdout, debug) 261 err = Output(proc.stderr, debug) 262 res = [] 263 while proc.returncode is None: 264 proc.poll() 265 res = select([out, err], [], [], .1) 266 for fd in res[0]: 267 fd.read() 268 for fd in res[0]: 269 fd.read(drain=1) 270 271 return out.lines, err.lines 272 273 def run(self, options, dryrun=None, kmemleak=None): 274 """ 275 This is the main function that runs each individual test. 276 Determine whether or not the command requires sudo, and modify it 277 if needed. Run the command, and update the result object. 278 """ 279 if dryrun is None: 280 dryrun = options.dryrun 281 if dryrun is True: 282 print(self) 283 return 284 if kmemleak is None: 285 kmemleak = options.kmemleak 286 287 privcmd = self.update_cmd_privs(self.pathname, self.user) 288 try: 289 old = os.umask(0) 290 if not os.path.isdir(self.outputdir): 291 os.makedirs(self.outputdir, mode=0o777) 292 os.umask(old) 293 except OSError as e: 294 fail('%s' % e) 295 296 """ 297 Log each test we run to /dev/kmsg (on Linux), so if there's a kernel 298 warning we'll be able to match it up to a particular test. 299 """ 300 if options.kmsg is True and exists("/dev/kmsg"): 301 try: 302 kp = Popen([SUDO, "sh", "-c", 303 f"echo ZTS run {self.pathname} > /dev/kmsg"]) 304 kp.wait() 305 except Exception: 306 pass 307 308 """ 309 Log each test we run to /dev/ttyu0 (on FreeBSD), so if there's a kernel 310 warning we'll be able to match it up to a particular test. 311 """ 312 if options.kmsg is True and exists("/dev/ttyu0"): 313 try: 314 kp = Popen([SUDO, "sh", "-c", 315 f"echo ZTS run {self.pathname} > /dev/ttyu0"]) 316 kp.wait() 317 except Exception: 318 pass 319 320 self.result.starttime = monotonic_time() 321 322 if kmemleak: 323 cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"' 324 check_output(cmd, shell=True) 325 326 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) 327 # Allow a special timeout value of 0 to mean infinity 328 if int(self.timeout) == 0: 329 self.timeout = sys.maxsize / (10 ** 9) 330 t = Timer( 331 int(self.timeout), self.kill_cmd, [proc, options, kmemleak] 332 ) 333 334 try: 335 t.start() 336 337 out, err = self.collect_output(proc, options.debug) 338 self.result.stdout = out 339 self.result.stderr = err 340 341 if kmemleak: 342 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"' 343 check_output(cmd, shell=True) 344 cmd = f'{SUDO} cat {KMEMLEAK_FILE}' 345 self.result.kmemleak = check_output(cmd, shell=True) 346 except KeyboardInterrupt: 347 self.kill_cmd(proc, options, kmemleak, True) 348 fail('\nRun terminated at user request.') 349 finally: 350 t.cancel() 351 352 if self.reran is not False: 353 self.result.done(proc, self.killed, self.reran) 354 355 def skip(self): 356 """ 357 Initialize enough of the test result that we can log a skipped 358 command. 359 """ 360 Result.total += 1 361 Result.runresults['SKIP'] += 1 362 self.result.stdout = self.result.stderr = [] 363 self.result.starttime = monotonic_time() 364 m, s = divmod(monotonic_time() - self.result.starttime, 60) 365 self.result.runtime = '%02d:%02d' % (m, s) 366 self.result.result = 'SKIP' 367 368 def log(self, options, suppress_console=False): 369 """ 370 This function is responsible for writing all output. This includes 371 the console output, the logfile of all results (with timestamped 372 merged stdout and stderr), and for each test, the unmodified 373 stdout/stderr/merged in its own file. 374 """ 375 376 timeprefix = datetime.now().strftime('[%FT%T.%f] ') 377 378 logname = getpwuid(os.getuid()).pw_name 379 rer = '' 380 if self.reran is True: 381 rer = ' (RERAN)' 382 user = ' (run as %s)' % (self.user if len(self.user) else logname) 383 if self.identifier: 384 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 385 else: 386 msga = 'Test: %s%s ' % (self.pathname, user) 387 msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer) 388 pad = ' ' * (80 - (len(msga) + len(msgb))) 389 result_line = timeprefix + msga + pad + msgb 390 391 # The result line is always written to the log file. If -q was 392 # specified only failures are written to the console, otherwise 393 # the result line is written to the console. The console output 394 # may be suppressed by calling log() with suppress_console=True. 395 write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE) 396 if not suppress_console: 397 if not options.quiet: 398 write_log(result_line, LOG_OUT) 399 elif options.quiet and self.result.result != 'PASS': 400 write_log(result_line, LOG_OUT) 401 402 lines = sorted(self.result.stdout + self.result.stderr, 403 key=lambda x: x[0]) 404 405 # Write timestamped output (stdout and stderr) to the logfile 406 for dt, line in lines: 407 timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11], 408 encoding='utf-8') 409 write_log(b'%s %s\n' % (timestamp, line), LOG_FILE) 410 411 # Write the separate stdout/stderr/merged files, if the data exists 412 if len(self.result.stdout): 413 with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out: 414 for _, line in self.result.stdout: 415 os.write(out.fileno(), b'%s\n' % line) 416 if len(self.result.stderr): 417 with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err: 418 for _, line in self.result.stderr: 419 os.write(err.fileno(), b'%s\n' % line) 420 if len(self.result.stdout) and len(self.result.stderr): 421 with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged: 422 for _, line in lines: 423 os.write(merged.fileno(), b'%s\n' % line) 424 if len(self.result.kmemleak): 425 with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem: 426 kmem.write(self.result.kmemleak) 427 428 429class Test(Cmd): 430 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 431 'post_user', 'failsafe', 'failsafe_user', 'tags'] 432 433 def __init__(self, pathname, 434 pre=None, pre_user=None, post=None, post_user=None, 435 failsafe=None, failsafe_user=None, tags=None, **kwargs): 436 super(Test, self).__init__(pathname, **kwargs) 437 self.pre = pre or '' 438 self.pre_user = pre_user or '' 439 self.post = post or '' 440 self.post_user = post_user or '' 441 self.failsafe = failsafe or '' 442 self.failsafe_user = failsafe_user or '' 443 self.tags = tags or [] 444 445 def __str__(self): 446 post_user = pre_user = failsafe_user = '' 447 if len(self.pre_user): 448 pre_user = ' (as %s)' % (self.pre_user) 449 if len(self.post_user): 450 post_user = ' (as %s)' % (self.post_user) 451 if len(self.failsafe_user): 452 failsafe_user = ' (as %s)' % (self.failsafe_user) 453 return '''\ 454Pathname: %s 455Identifier: %s 456Outputdir: %s 457Timeout: %d 458User: %s 459Pre: %s%s 460Post: %s%s 461Failsafe: %s%s 462Tags: %s 463''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 464 self.pre, pre_user, self.post, post_user, self.failsafe, 465 failsafe_user, self.tags) 466 467 def verify(self): 468 """ 469 Check the pre/post/failsafe scripts, user and Test. Omit the Test from 470 this run if there are any problems. 471 """ 472 files = [self.pre, self.pathname, self.post, self.failsafe] 473 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 474 475 for f in [f for f in files if len(f)]: 476 if not verify_file(f): 477 write_log("Warning: Test '%s' not added to this run because" 478 " it failed verification.\n" % f, LOG_ERR) 479 return False 480 481 for user in [user for user in users if len(user)]: 482 if not verify_user(user): 483 write_log("Not adding Test '%s' to this run.\n" % 484 self.pathname, LOG_ERR) 485 return False 486 487 return True 488 489 def run(self, options, dryrun=None, kmemleak=None): 490 """ 491 Create Cmd instances for the pre/post/failsafe scripts. If the pre 492 script doesn't pass, skip this Test. Run the post script regardless. 493 If the Test is killed, also run the failsafe script. 494 """ 495 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 496 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 497 timeout=self.timeout, user=self.pre_user) 498 test = Cmd(self.pathname, identifier=self.identifier, 499 outputdir=self.outputdir, timeout=self.timeout, 500 user=self.user) 501 odir = os.path.join(self.outputdir, os.path.basename(self.failsafe)) 502 failsafe = Cmd(self.failsafe, identifier=self.identifier, 503 outputdir=odir, timeout=self.timeout, 504 user=self.failsafe_user) 505 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 506 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 507 timeout=self.timeout, user=self.post_user) 508 509 cont = True 510 if len(pretest.pathname): 511 pretest.run(options, kmemleak=False) 512 cont = pretest.result.result == 'PASS' 513 pretest.log(options) 514 515 if cont: 516 test.run(options, kmemleak=kmemleak) 517 if test.result.result == 'KILLED' and len(failsafe.pathname): 518 failsafe.run(options, kmemleak=False) 519 failsafe.log(options, suppress_console=True) 520 else: 521 test.skip() 522 523 test.log(options) 524 525 if len(posttest.pathname): 526 posttest.run(options, kmemleak=False) 527 posttest.log(options) 528 529 530class TestGroup(Test): 531 props = Test.props + ['tests'] 532 533 def __init__(self, pathname, tests=None, **kwargs): 534 super(TestGroup, self).__init__(pathname, **kwargs) 535 self.tests = tests or [] 536 537 def __str__(self): 538 post_user = pre_user = failsafe_user = '' 539 if len(self.pre_user): 540 pre_user = ' (as %s)' % (self.pre_user) 541 if len(self.post_user): 542 post_user = ' (as %s)' % (self.post_user) 543 if len(self.failsafe_user): 544 failsafe_user = ' (as %s)' % (self.failsafe_user) 545 return '''\ 546Pathname: %s 547Identifier: %s 548Outputdir: %s 549Tests: %s 550Timeout: %s 551User: %s 552Pre: %s%s 553Post: %s%s 554Failsafe: %s%s 555Tags: %s 556''' % (self.pathname, self.identifier, self.outputdir, self.tests, 557 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 558 self.failsafe, failsafe_user, self.tags) 559 560 def filter(self, keeplist): 561 self.tests = [x for x in self.tests if x in keeplist] 562 563 def verify(self): 564 """ 565 Check the pre/post/failsafe scripts, user and tests in this TestGroup. 566 Omit the TestGroup entirely, or simply delete the relevant tests in the 567 group, if that's all that's required. 568 """ 569 # If the pre/post/failsafe scripts are relative pathnames, convert to 570 # absolute, so they stand a chance of passing verification. 571 if len(self.pre) and not os.path.isabs(self.pre): 572 self.pre = os.path.join(self.pathname, self.pre) 573 if len(self.post) and not os.path.isabs(self.post): 574 self.post = os.path.join(self.pathname, self.post) 575 if len(self.failsafe) and not os.path.isabs(self.failsafe): 576 self.post = os.path.join(self.pathname, self.post) 577 578 auxfiles = [self.pre, self.post, self.failsafe] 579 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 580 581 for f in [f for f in auxfiles if len(f)]: 582 if f != self.failsafe and self.pathname != os.path.dirname(f): 583 write_log("Warning: TestGroup '%s' not added to this run. " 584 "Auxiliary script '%s' exists in a different " 585 "directory.\n" % (self.pathname, f), LOG_ERR) 586 return False 587 588 if not verify_file(f): 589 write_log("Warning: TestGroup '%s' not added to this run. " 590 "Auxiliary script '%s' failed verification.\n" % 591 (self.pathname, f), LOG_ERR) 592 return False 593 594 for user in [user for user in users if len(user)]: 595 if not verify_user(user): 596 write_log("Not adding TestGroup '%s' to this run.\n" % 597 self.pathname, LOG_ERR) 598 return False 599 600 # If one of the tests is invalid, delete it, log it, and drive on. 601 for test in self.tests: 602 if not verify_file(os.path.join(self.pathname, test)): 603 del self.tests[self.tests.index(test)] 604 write_log("Warning: Test '%s' removed from TestGroup '%s' " 605 "because it failed verification.\n" % 606 (test, self.pathname), LOG_ERR) 607 608 return len(self.tests) != 0 609 610 def run(self, options, dryrun=None, kmemleak=None): 611 """ 612 Create Cmd instances for the pre/post/failsafe scripts. If the pre 613 script doesn't pass, skip all the tests in this TestGroup. Run the 614 post script regardless. Run the failsafe script when a test is killed. 615 """ 616 # tags assigned to this test group also include the test names 617 if options.tags and not set(self.tags).intersection(set(options.tags)): 618 return 619 620 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 621 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 622 user=self.pre_user, identifier=self.identifier) 623 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 624 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 625 user=self.post_user, identifier=self.identifier) 626 627 cont = True 628 if len(pretest.pathname): 629 pretest.run(options, dryrun=dryrun, kmemleak=False) 630 cont = pretest.result.result == 'PASS' 631 pretest.log(options) 632 633 for fname in self.tests: 634 odir = os.path.join(self.outputdir, fname) 635 test = Cmd(os.path.join(self.pathname, fname), outputdir=odir, 636 timeout=self.timeout, user=self.user, 637 identifier=self.identifier) 638 odir = os.path.join(odir, os.path.basename(self.failsafe)) 639 failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout, 640 user=self.failsafe_user, identifier=self.identifier) 641 if cont: 642 test.run(options, dryrun=dryrun, kmemleak=kmemleak) 643 if test.result.result == 'KILLED' and len(failsafe.pathname): 644 failsafe.run(options, dryrun=dryrun, kmemleak=False) 645 failsafe.log(options, suppress_console=True) 646 else: 647 test.skip() 648 649 test.log(options) 650 651 if len(posttest.pathname): 652 posttest.run(options, dryrun=dryrun, kmemleak=False) 653 posttest.log(options) 654 655 656class TestRun(object): 657 props = ['quiet', 'outputdir', 'debug'] 658 659 def __init__(self, options): 660 self.tests = {} 661 self.testgroups = {} 662 self.starttime = time() 663 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 664 self.outputdir = os.path.join(options.outputdir, self.timestamp) 665 self.setup_logging(options) 666 self.defaults = [ 667 ('outputdir', BASEDIR), 668 ('quiet', False), 669 ('timeout', 60), 670 ('user', ''), 671 ('pre', ''), 672 ('pre_user', ''), 673 ('post', ''), 674 ('post_user', ''), 675 ('failsafe', ''), 676 ('failsafe_user', ''), 677 ('tags', []), 678 ('debug', False) 679 ] 680 681 def __str__(self): 682 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 683 s += 'TESTS:\n' 684 for key in sorted(self.tests.keys()): 685 s += '%s%s' % (self.tests[key].__str__(), '\n') 686 s += 'TESTGROUPS:\n' 687 for key in sorted(self.testgroups.keys()): 688 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 689 return s 690 691 def addtest(self, pathname, options): 692 """ 693 Create a new Test, and apply any properties that were passed in 694 from the command line. If it passes verification, add it to the 695 TestRun. 696 """ 697 test = Test(pathname) 698 for prop in Test.props: 699 setattr(test, prop, getattr(options, prop)) 700 701 if test.verify(): 702 self.tests[pathname] = test 703 704 def addtestgroup(self, dirname, filenames, options): 705 """ 706 Create a new TestGroup, and apply any properties that were passed 707 in from the command line. If it passes verification, add it to the 708 TestRun. 709 """ 710 if dirname not in self.testgroups: 711 testgroup = TestGroup(dirname) 712 for prop in Test.props: 713 setattr(testgroup, prop, getattr(options, prop)) 714 715 # Prevent pre/post/failsafe scripts from running as regular tests 716 for f in [testgroup.pre, testgroup.post, testgroup.failsafe]: 717 if f in filenames: 718 del filenames[filenames.index(f)] 719 720 self.testgroups[dirname] = testgroup 721 self.testgroups[dirname].tests = sorted(filenames) 722 723 testgroup.verify() 724 725 def filter(self, keeplist): 726 for group in list(self.testgroups.keys()): 727 if group not in keeplist: 728 del self.testgroups[group] 729 continue 730 731 g = self.testgroups[group] 732 733 if g.pre and os.path.basename(g.pre) in keeplist[group]: 734 continue 735 736 g.filter(keeplist[group]) 737 738 for test in list(self.tests.keys()): 739 directory, base = os.path.split(test) 740 if directory not in keeplist or base not in keeplist[directory]: 741 del self.tests[test] 742 743 def read(self, options): 744 """ 745 Read in the specified runfiles, and apply the TestRun properties 746 listed in the 'DEFAULT' section to our TestRun. Then read each 747 section, and apply the appropriate properties to the Test or 748 TestGroup. Properties from individual sections override those set 749 in the 'DEFAULT' section. If the Test or TestGroup passes 750 verification, add it to the TestRun. 751 """ 752 config = configparser.RawConfigParser() 753 parsed = config.read(options.runfiles) 754 failed = options.runfiles - set(parsed) 755 if len(failed): 756 files = ' '.join(sorted(failed)) 757 fail("Couldn't read config files: %s" % files) 758 759 for opt in TestRun.props: 760 if config.has_option('DEFAULT', opt): 761 if opt == 'outputdir': 762 outputdir = config.get('DEFAULT', opt) 763 setattr(self, opt, os.path.join(outputdir, self.timestamp)) 764 else: 765 setattr(self, opt, config.get('DEFAULT', opt)) 766 767 testdir = options.testdir 768 769 for section in config.sections(): 770 if 'tests' in config.options(section): 771 parts = section.split(':', 1) 772 sectiondir = parts[0] 773 identifier = parts[1] if len(parts) == 2 else None 774 if os.path.isdir(sectiondir): 775 pathname = sectiondir 776 elif os.path.isdir(os.path.join(testdir, sectiondir)): 777 pathname = os.path.join(testdir, sectiondir) 778 else: 779 pathname = sectiondir 780 781 testgroup = TestGroup(os.path.abspath(pathname), 782 identifier=identifier) 783 for prop in TestGroup.props: 784 for sect in ['DEFAULT', section]: 785 if config.has_option(sect, prop): 786 if prop == 'tags': 787 setattr(testgroup, prop, 788 eval(config.get(sect, prop))) 789 elif prop == 'failsafe': 790 failsafe = config.get(sect, prop) 791 setattr(testgroup, prop, 792 os.path.join(testdir, failsafe)) 793 elif prop == 'outputdir': 794 outputdir = config.get(sect, prop) 795 setattr(self, opt, 796 os.path.join(outputdir, 797 self.timestamp)) 798 else: 799 setattr(testgroup, prop, 800 config.get(sect, prop)) 801 802 # Repopulate tests using eval to convert the string to a list 803 testgroup.tests = eval(config.get(section, 'tests')) 804 805 if testgroup.verify(): 806 self.testgroups[section] = testgroup 807 else: 808 test = Test(section) 809 for prop in Test.props: 810 for sect in ['DEFAULT', section]: 811 if config.has_option(sect, prop): 812 if prop == 'failsafe': 813 failsafe = config.get(sect, prop) 814 setattr(test, prop, 815 os.path.join(testdir, failsafe)) 816 elif prop == 'outputdir': 817 outputdir = config.get(sect, prop) 818 setattr(self, opt, 819 os.path.join(outputdir, 820 self.timestamp)) 821 else: 822 setattr(test, prop, config.get(sect, prop)) 823 824 if test.verify(): 825 self.tests[section] = test 826 827 def write(self, options): 828 """ 829 Create a configuration file for editing and later use. The 830 'DEFAULT' section of the config file is created from the 831 properties that were specified on the command line. Tests are 832 simply added as sections that inherit everything from the 833 'DEFAULT' section. TestGroups are the same, except they get an 834 option including all the tests to run in that directory. 835 """ 836 837 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 838 self.defaults]) 839 config = configparser.RawConfigParser(defaults) 840 841 for test in sorted(self.tests.keys()): 842 config.add_section(test) 843 for prop in Test.props: 844 if prop not in self.props: 845 config.set(test, prop, 846 getattr(self.tests[test], prop)) 847 848 for testgroup in sorted(self.testgroups.keys()): 849 config.add_section(testgroup) 850 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 851 for prop in TestGroup.props: 852 if prop not in self.props: 853 config.set(testgroup, prop, 854 getattr(self.testgroups[testgroup], prop)) 855 856 try: 857 with open(options.template, 'w') as f: 858 return config.write(f) 859 except IOError: 860 fail('Could not open \'%s\' for writing.' % options.template) 861 862 def complete_outputdirs(self): 863 """ 864 Collect all the pathnames for Tests, and TestGroups. Strip off all 865 common leading path components, and append what remains to the top 866 "output" dir, to create a tree of output directories that match 867 the test and group names in structure. Tests will be able 868 to write output files directly in the newly modified outputdir. 869 TestGroups will be able to create one subdirectory per test in the 870 outputdir, and are guaranteed uniqueness because a group can only 871 contain files in one directory. Pre and post tests will create a 872 directory rooted at the outputdir of the Test or TestGroup in 873 question for their output. Failsafe scripts will create a directory 874 rooted at the outputdir of each Test for their output. 875 """ 876 877 alltests = dict(list(self.tests.items()) + 878 list(self.testgroups.items())) 879 base = os.path.join(self.outputdir, 'output') 880 881 seen = [] 882 883 for path in list(alltests.keys()): 884 frag = path.split('/') 885 for i in range(0, len(frag)): 886 if len(seen) == i: 887 seen.append({}) 888 seen[i][frag[i]] = 1 889 890 cut = 0 891 for i in range(0, len(seen)): 892 if len(list(seen[i].keys())) == 1: 893 cut += 1 894 else: 895 break 896 897 for path in list(alltests.keys()): 898 uniq = path.split('/', cut)[-1] 899 alltests[path].outputdir = os.path.join(base, uniq) 900 901 def setup_logging(self, options): 902 """ 903 This function creates the output directory and gets a file object 904 for the logfile. This function must be called before write_log() 905 can be used. 906 """ 907 if options.dryrun is True: 908 return 909 910 global LOG_FILE_OBJ 911 if not options.template: 912 try: 913 old = os.umask(0) 914 os.makedirs(self.outputdir, mode=0o777) 915 os.umask(old) 916 filename = os.path.join(self.outputdir, 'log') 917 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') 918 except OSError as e: 919 fail('%s' % e) 920 921 def run(self, options): 922 """ 923 Walk through all the Tests and TestGroups, calling run(). 924 """ 925 try: 926 os.chdir(self.outputdir) 927 except OSError: 928 fail('Could not change to directory %s' % self.outputdir) 929 # make a symlink to the output for the currently running test 930 logsymlink = os.path.join(self.outputdir, '../current') 931 if os.path.islink(logsymlink): 932 os.unlink(logsymlink) 933 if not os.path.exists(logsymlink): 934 os.symlink(self.outputdir, logsymlink) 935 else: 936 write_log('Could not make a symlink to directory %s\n' % 937 self.outputdir, LOG_ERR) 938 939 if options.kmemleak: 940 cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"' 941 check_output(cmd, shell=True) 942 943 iteration = 0 944 while iteration < options.iterations: 945 for test in sorted(self.tests.keys()): 946 self.tests[test].run(options) 947 for testgroup in sorted(self.testgroups.keys()): 948 self.testgroups[testgroup].run(options) 949 iteration += 1 950 951 def summary(self): 952 if Result.total == 0: 953 return 2 954 955 print('\nResults Summary') 956 for key in list(Result.runresults.keys()): 957 if Result.runresults[key] != 0: 958 print('%s\t% 4d' % (key, Result.runresults[key])) 959 960 m, s = divmod(time() - self.starttime, 60) 961 h, m = divmod(m, 60) 962 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 963 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 964 float(Result.total)) * 100)) 965 print('Log directory:\t%s' % self.outputdir) 966 967 if Result.runresults['FAIL'] > 0: 968 return 1 969 970 if Result.runresults['KILLED'] > 0: 971 return 1 972 973 if Result.runresults['RERAN'] > 0: 974 return 3 975 976 return 0 977 978 979def write_log(msg, target): 980 """ 981 Write the provided message to standard out, standard error or 982 the logfile. If specifying LOG_FILE, then `msg` must be a bytes 983 like object. This way we can still handle output from tests that 984 may be in unexpected encodings. 985 """ 986 if target == LOG_OUT: 987 os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8')) 988 elif target == LOG_ERR: 989 os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8')) 990 elif target == LOG_FILE: 991 os.write(LOG_FILE_OBJ.fileno(), msg) 992 else: 993 fail('log_msg called with unknown target "%s"' % target) 994 995 996def verify_file(pathname): 997 """ 998 Verify that the supplied pathname is an executable regular file. 999 """ 1000 if os.path.isdir(pathname) or os.path.islink(pathname): 1001 return False 1002 1003 for ext in '', '.ksh', '.sh': 1004 script_path = pathname + ext 1005 if os.path.isfile(script_path) and os.access(script_path, os.X_OK): 1006 return True 1007 1008 return False 1009 1010 1011def verify_user(user): 1012 """ 1013 Verify that the specified user exists on this system, and can execute 1014 sudo without being prompted for a password. 1015 """ 1016 testcmd = [SUDO, '-n', '-u', user, TRUE] 1017 1018 if user in Cmd.verified_users: 1019 return True 1020 1021 try: 1022 getpwnam(user) 1023 except KeyError: 1024 write_log("Warning: user '%s' does not exist.\n" % user, 1025 LOG_ERR) 1026 return False 1027 1028 p = Popen(testcmd) 1029 p.wait() 1030 if p.returncode != 0: 1031 write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user, 1032 LOG_ERR) 1033 return False 1034 else: 1035 Cmd.verified_users.append(user) 1036 1037 return True 1038 1039 1040def find_tests(testrun, options): 1041 """ 1042 For the given list of pathnames, add files as Tests. For directories, 1043 if do_groups is True, add the directory as a TestGroup. If False, 1044 recursively search for executable files. 1045 """ 1046 1047 for p in sorted(options.pathnames): 1048 if os.path.isdir(p): 1049 for dirname, _, filenames in os.walk(p): 1050 if options.do_groups: 1051 testrun.addtestgroup(dirname, filenames, options) 1052 else: 1053 for f in sorted(filenames): 1054 testrun.addtest(os.path.join(dirname, f), options) 1055 else: 1056 testrun.addtest(p, options) 1057 1058 1059def filter_tests(testrun, options): 1060 try: 1061 fh = open(options.logfile, "r") 1062 except Exception as e: 1063 fail('%s' % e) 1064 1065 failed = {} 1066 while True: 1067 line = fh.readline() 1068 if not line: 1069 break 1070 m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line) 1071 if not m: 1072 continue 1073 group, test = m.group(1, 2) 1074 try: 1075 failed[group].append(test) 1076 except KeyError: 1077 failed[group] = [test] 1078 fh.close() 1079 1080 testrun.filter(failed) 1081 1082 1083def fail(retstr, ret=1): 1084 print('%s: %s' % (sys.argv[0], retstr)) 1085 exit(ret) 1086 1087 1088def kmemleak_cb(option, opt_str, value, parser): 1089 if not os.path.exists(KMEMLEAK_FILE): 1090 fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " + 1091 "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.") 1092 1093 setattr(parser.values, option.dest, True) 1094 1095 1096def options_cb(option, opt_str, value, parser): 1097 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 1098 1099 if opt_str in parser.rargs: 1100 fail('%s may only be specified once.' % opt_str) 1101 1102 if option.dest == 'runfiles': 1103 parser.values.cmd = 'rdconfig' 1104 value = set(os.path.abspath(p) for p in value.split(',')) 1105 if option.dest == 'tags': 1106 value = [x.strip() for x in value.split(',')] 1107 1108 if option.dest in path_options: 1109 setattr(parser.values, option.dest, os.path.abspath(value)) 1110 else: 1111 setattr(parser.values, option.dest, value) 1112 1113 1114def parse_args(): 1115 parser = OptionParser() 1116 parser.add_option('-c', action='callback', callback=options_cb, 1117 type='string', dest='runfiles', metavar='runfiles', 1118 help='Specify tests to run via config files.') 1119 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 1120 help='Dry run. Print tests, but take no other action.') 1121 parser.add_option('-D', action='store_true', default=False, dest='debug', 1122 help='Write all test output to stdout as it arrives.') 1123 parser.add_option('-l', action='callback', callback=options_cb, 1124 default=None, dest='logfile', metavar='logfile', 1125 type='string', 1126 help='Read logfile and re-run tests which failed.') 1127 parser.add_option('-g', action='store_true', default=False, 1128 dest='do_groups', help='Make directories TestGroups.') 1129 parser.add_option('-o', action='callback', callback=options_cb, 1130 default=BASEDIR, dest='outputdir', type='string', 1131 metavar='outputdir', help='Specify an output directory.') 1132 parser.add_option('-i', action='callback', callback=options_cb, 1133 default=TESTDIR, dest='testdir', type='string', 1134 metavar='testdir', help='Specify a test directory.') 1135 parser.add_option('-K', action='store_true', default=False, dest='kmsg', 1136 help='Log tests names to /dev/kmsg') 1137 parser.add_option('-m', action='callback', callback=kmemleak_cb, 1138 default=False, dest='kmemleak', 1139 help='Enable kmemleak reporting (Linux only)') 1140 parser.add_option('-p', action='callback', callback=options_cb, 1141 default='', dest='pre', metavar='script', 1142 type='string', help='Specify a pre script.') 1143 parser.add_option('-P', action='callback', callback=options_cb, 1144 default='', dest='post', metavar='script', 1145 type='string', help='Specify a post script.') 1146 parser.add_option('-q', action='store_true', default=False, dest='quiet', 1147 help='Silence on the console during a test run.') 1148 parser.add_option('-s', action='callback', callback=options_cb, 1149 default='', dest='failsafe', metavar='script', 1150 type='string', help='Specify a failsafe script.') 1151 parser.add_option('-S', action='callback', callback=options_cb, 1152 default='', dest='failsafe_user', 1153 metavar='failsafe_user', type='string', 1154 help='Specify a user to execute the failsafe script.') 1155 parser.add_option('-t', action='callback', callback=options_cb, default=60, 1156 dest='timeout', metavar='seconds', type='int', 1157 help='Timeout (in seconds) for an individual test.') 1158 parser.add_option('-u', action='callback', callback=options_cb, 1159 default='', dest='user', metavar='user', type='string', 1160 help='Specify a different user name to run as.') 1161 parser.add_option('-w', action='callback', callback=options_cb, 1162 default=None, dest='template', metavar='template', 1163 type='string', help='Create a new config file.') 1164 parser.add_option('-x', action='callback', callback=options_cb, default='', 1165 dest='pre_user', metavar='pre_user', type='string', 1166 help='Specify a user to execute the pre script.') 1167 parser.add_option('-X', action='callback', callback=options_cb, default='', 1168 dest='post_user', metavar='post_user', type='string', 1169 help='Specify a user to execute the post script.') 1170 parser.add_option('-T', action='callback', callback=options_cb, default='', 1171 dest='tags', metavar='tags', type='string', 1172 help='Specify tags to execute specific test groups.') 1173 parser.add_option('-I', action='callback', callback=options_cb, default=1, 1174 dest='iterations', metavar='iterations', type='int', 1175 help='Number of times to run the test run.') 1176 (options, pathnames) = parser.parse_args() 1177 1178 if options.runfiles and len(pathnames): 1179 fail('Extraneous arguments.') 1180 1181 options.pathnames = [os.path.abspath(path) for path in pathnames] 1182 1183 return options 1184 1185 1186def main(): 1187 options = parse_args() 1188 1189 testrun = TestRun(options) 1190 1191 if options.runfiles: 1192 testrun.read(options) 1193 else: 1194 find_tests(testrun, options) 1195 1196 if options.logfile: 1197 filter_tests(testrun, options) 1198 1199 if options.template: 1200 testrun.write(options) 1201 exit(0) 1202 1203 testrun.complete_outputdirs() 1204 testrun.run(options) 1205 exit(testrun.summary()) 1206 1207 1208if __name__ == '__main__': 1209 main() 1210