1#!/usr/bin/env @PYTHON_SHEBANG@ 2# SPDX-License-Identifier: CDDL-1.0 3 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# http://www.illumos.org/license/CDDL. 13# 14 15# 16# Copyright (c) 2012, 2018 by Delphix. All rights reserved. 17# Copyright (c) 2019 Datto Inc. 18# Copyright (c) 2025, Klara, Inc. 19# 20# This script must remain compatible with Python 3.6+. 21# 22 23import os 24import sys 25import ctypes 26import re 27import configparser 28 29from datetime import datetime 30from optparse import OptionParser 31from pwd import getpwnam 32from pwd import getpwuid 33from select import select 34from subprocess import PIPE 35from subprocess import Popen 36from subprocess import check_output 37from subprocess import run 38from threading import Timer 39from time import time, CLOCK_MONOTONIC 40from os.path import exists 41 42BASEDIR = '/var/tmp/test_results' 43TESTDIR = '/usr/share/zfs/' 44KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak' 45KILL = 'kill' 46TRUE = 'true' 47SUDO = 'sudo' 48LOG_FILE = 'LOG_FILE' 49LOG_OUT = 'LOG_OUT' 50LOG_ERR = 'LOG_ERR' 51LOG_FILE_OBJ = None 52 53try: 54 from time import monotonic as monotonic_time 55except ImportError: 56 class timespec(ctypes.Structure): 57 _fields_ = [ 58 ('tv_sec', ctypes.c_long), 59 ('tv_nsec', ctypes.c_long) 60 ] 61 62 librt = ctypes.CDLL('librt.so.1', use_errno=True) 63 clock_gettime = librt.clock_gettime 64 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] 65 66 def monotonic_time(): 67 t = timespec() 68 if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0: 69 errno_ = ctypes.get_errno() 70 raise OSError(errno_, os.strerror(errno_)) 71 return t.tv_sec + t.tv_nsec * 1e-9 72 73 74class Result(object): 75 total = 0 76 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0} 77 78 def __init__(self): 79 self.starttime = None 80 self.returncode = None 81 self.runtime = '' 82 self.stdout = [] 83 self.stderr = [] 84 self.kmemleak = '' 85 self.result = '' 86 87 def done(self, proc, killed, reran): 88 """ 89 Finalize the results of this Cmd. 90 """ 91 Result.total += 1 92 m, s = divmod(monotonic_time() - self.starttime, 60) 93 self.runtime = '%02d:%02d' % (m, s) 94 self.returncode = proc.returncode 95 if reran is True: 96 Result.runresults['RERAN'] += 1 97 if killed: 98 self.result = 'KILLED' 99 Result.runresults['KILLED'] += 1 100 elif len(self.kmemleak) > 0: 101 self.result = 'FAIL' 102 Result.runresults['FAIL'] += 1 103 elif self.returncode == 0: 104 self.result = 'PASS' 105 Result.runresults['PASS'] += 1 106 elif self.returncode == 4: 107 self.result = 'SKIP' 108 Result.runresults['SKIP'] += 1 109 elif self.returncode != 0: 110 self.result = 'FAIL' 111 Result.runresults['FAIL'] += 1 112 113 114class Output(object): 115 """ 116 This class is a slightly modified version of the 'Stream' class found 117 here: https://stackoverflow.com/q/4984549/ 118 """ 119 def __init__(self, stream, debug=False): 120 self.stream = stream 121 self.debug = debug 122 self._buf = b'' 123 self.lines = [] 124 125 def fileno(self): 126 return self.stream.fileno() 127 128 def read(self, drain=0): 129 """ 130 Read from the file descriptor. If 'drain' set, read until EOF. 131 """ 132 while self._read() is not None: 133 if not drain: 134 break 135 136 def _read(self): 137 """ 138 Read up to 4k of data from this output stream. Collect the output 139 up to the last newline, and append it to any leftover data from a 140 previous call. The lines are stored as a (timestamp, data) tuple 141 for easy sorting/merging later. 142 """ 143 fd = self.fileno() 144 buf = os.read(fd, 4096) 145 if not buf: 146 return None 147 if self.debug: 148 os.write(sys.stderr.fileno(), buf) 149 if b'\n' not in buf: 150 self._buf += buf 151 return [] 152 153 buf = self._buf + buf 154 tmp, rest = buf.rsplit(b'\n', 1) 155 self._buf = rest 156 now = datetime.now() 157 rows = tmp.split(b'\n') 158 self.lines += [(now, r) for r in rows] 159 160 161class Cmd(object): 162 verified_users = [] 163 164 def __init__(self, pathname, identifier=None, outputdir=None, 165 timeout=None, user=None, tags=None): 166 self.pathname = pathname 167 self.identifier = identifier 168 self.outputdir = outputdir or 'BASEDIR' 169 """ 170 The timeout for tests is measured in wall-clock time 171 """ 172 self.timeout = timeout 173 self.user = user or '' 174 self.killed = False 175 self.reran = None 176 self.result = Result() 177 178 if self.timeout is None: 179 self.timeout = 60 180 181 def __str__(self): 182 return '''\ 183Pathname: %s 184Identifier: %s 185Outputdir: %s 186Timeout: %d 187User: %s 188''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 189 190 def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False): 191 192 """ 193 We're about to kill a command due to a timeout. 194 If we're running with the -O option, then dump debug info about the 195 process with the highest CPU usage to /dev/kmsg (Linux only). This can 196 help debug the timeout. 197 198 Debug info includes: 199 - 30 lines from 'top' 200 - /proc/<PID>/stack output of process with highest CPU usage 201 - Last lines strace-ing process with highest CPU usage 202 """ 203 if exists("/dev/kmsg"): 204 c = """ 205TOP_OUT="$(COLUMNS=160 top -b -n 1 | head -n 30)" 206read -r PID CMD <<< $(echo "$TOP_OUT" | /usr/bin/awk \ 207"/COMMAND/{ 208 print_next=1 209 next 210} 211{ 212 if (print_next == 1) { 213 print \\$1\\" \\"\\$12 214 exit 215 } 216}") 217echo "##### ZTS timeout debug #####" 218echo "----- top -----" 219echo "$TOP_OUT" 220echo "----- /proc/$PID/stack ($CMD)) -----" 221cat /proc/$PID/stack 222echo "----- strace ($CMD) -----" 223TMPFILE="$(mktemp --suffix=ZTS)" 224/usr/bin/strace -k --stack-traces -p $PID &> "$TMPFILE" & 225sleep 0.1 226killall strace 227tail -n 30 $TMPFILE 228rm "$TMPFILE" 229echo "##### /proc/sysrq-trigger stack #####" 230""" 231 c = "sudo bash -c '" + c + "'" 232 data = run(c, capture_output=True, shell=True, text=True) 233 out = data.stdout 234 try: 235 kp = Popen([SUDO, "sh", "-c", 236 "echo '" + out + "' > /dev/kmsg"]) 237 kp.wait() 238 239 """ 240 Trigger kernel stack traces 241 """ 242 kp = Popen([SUDO, "sh", "-c", 243 "echo l > /proc/sysrq-trigger"]) 244 kp.wait() 245 except Exception: 246 pass 247 248 """ 249 Kill a running command due to timeout, or ^C from the keyboard. If 250 sudo is required, this user was verified previously. 251 """ 252 self.killed = True 253 do_sudo = len(self.user) != 0 254 signal = '-TERM' 255 256 cmd = [SUDO, KILL, signal, str(proc.pid)] 257 if not do_sudo: 258 del cmd[0] 259 260 try: 261 kp = Popen(cmd) 262 kp.wait() 263 except Exception: 264 pass 265 266 """ 267 If this is not a user-initiated kill and the test has not been 268 reran before we consider if the test needs to be reran: 269 If the test has spent some time hibernating and didn't run the whole 270 length of time before being timed out we will rerun the test. 271 """ 272 if keyboard_interrupt is False and self.reran is None: 273 runtime = monotonic_time() - self.result.starttime 274 if int(self.timeout) > runtime: 275 self.killed = False 276 self.reran = False 277 self.run(options, dryrun=False, kmemleak=kmemleak) 278 self.reran = True 279 280 def update_cmd_privs(self, cmd, user): 281 """ 282 If a user has been specified to run this Cmd and we're not already 283 running as that user, prepend the appropriate sudo command to run 284 as that user. 285 """ 286 me = getpwuid(os.getuid()) 287 288 if not user or user is me: 289 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 290 cmd += '.ksh' 291 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 292 cmd += '.sh' 293 return cmd 294 295 if not os.path.isfile(cmd): 296 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 297 cmd += '.ksh' 298 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 299 cmd += '.sh' 300 301 # glibc (at least) will not pass TMPDIR through to setuid programs. 302 # if set, arrange for it to be reset before running the target cmd 303 tmpdir = os.getenv('TMPDIR') 304 if tmpdir: 305 tmpdirarg = 'env TMPDIR=%s' % tmpdir 306 else: 307 tmpdirarg = '' 308 309 ret = '%s -E -u %s %s %s' % (SUDO, user, tmpdirarg, cmd) 310 return ret.split(' ') 311 312 def collect_output(self, proc, debug=False): 313 """ 314 Read from stdout/stderr as data becomes available, until the 315 process is no longer running. Return the lines from the stdout and 316 stderr Output objects. 317 """ 318 out = Output(proc.stdout, debug) 319 err = Output(proc.stderr, debug) 320 res = [] 321 while proc.returncode is None: 322 proc.poll() 323 res = select([out, err], [], [], .1) 324 for fd in res[0]: 325 fd.read() 326 for fd in res[0]: 327 fd.read(drain=1) 328 329 return out.lines, err.lines 330 331 def run(self, options, dryrun=None, kmemleak=None): 332 """ 333 This is the main function that runs each individual test. 334 Determine whether or not the command requires sudo, and modify it 335 if needed. Run the command, and update the result object. 336 """ 337 if dryrun is None: 338 dryrun = options.dryrun 339 if dryrun is True: 340 print(self) 341 return 342 if kmemleak is None: 343 kmemleak = options.kmemleak 344 345 privcmd = self.update_cmd_privs(self.pathname, self.user) 346 try: 347 old = os.umask(0) 348 if not os.path.isdir(self.outputdir): 349 os.makedirs(self.outputdir, mode=0o777) 350 os.umask(old) 351 except OSError as e: 352 fail('%s' % e) 353 354 """ 355 Log each test we run to /dev/kmsg (on Linux), so if there's a kernel 356 warning we'll be able to match it up to a particular test. 357 """ 358 if options.kmsg is True and exists("/dev/kmsg"): 359 try: 360 kp = Popen([SUDO, "sh", "-c", 361 f"echo ZTS run {self.pathname} > /dev/kmsg"]) 362 kp.wait() 363 except Exception: 364 pass 365 366 """ 367 Log each test we run to /dev/ttyu0 (on FreeBSD), so if there's a kernel 368 warning we'll be able to match it up to a particular test. 369 """ 370 if options.kmsg is True and exists("/dev/ttyu0"): 371 try: 372 kp = Popen([SUDO, "sh", "-c", 373 f"echo ZTS run {self.pathname} > /dev/ttyu0"]) 374 kp.wait() 375 except Exception: 376 pass 377 378 self.result.starttime = monotonic_time() 379 380 if kmemleak: 381 cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"' 382 check_output(cmd, shell=True) 383 384 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) 385 # Allow a special timeout value of 0 to mean infinity 386 if int(self.timeout) == 0: 387 self.timeout = sys.maxsize / (10 ** 9) 388 t = Timer( 389 int(self.timeout), self.kill_cmd, [proc, options, kmemleak] 390 ) 391 392 try: 393 t.start() 394 395 out, err = self.collect_output(proc, options.debug) 396 self.result.stdout = out 397 self.result.stderr = err 398 399 if kmemleak: 400 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"' 401 check_output(cmd, shell=True) 402 cmd = f'{SUDO} cat {KMEMLEAK_FILE}' 403 self.result.kmemleak = check_output(cmd, shell=True) 404 except KeyboardInterrupt: 405 self.kill_cmd(proc, options, kmemleak, True) 406 fail('\nRun terminated at user request.') 407 finally: 408 t.cancel() 409 410 if self.reran is not False: 411 self.result.done(proc, self.killed, self.reran) 412 413 def skip(self): 414 """ 415 Initialize enough of the test result that we can log a skipped 416 command. 417 """ 418 Result.total += 1 419 Result.runresults['SKIP'] += 1 420 self.result.stdout = self.result.stderr = [] 421 self.result.starttime = monotonic_time() 422 m, s = divmod(monotonic_time() - self.result.starttime, 60) 423 self.result.runtime = '%02d:%02d' % (m, s) 424 self.result.result = 'SKIP' 425 426 def log(self, options, suppress_console=False): 427 """ 428 This function is responsible for writing all output. This includes 429 the console output, the logfile of all results (with timestamped 430 merged stdout and stderr), and for each test, the unmodified 431 stdout/stderr/merged in its own file. 432 """ 433 434 timeprefix = datetime.now().strftime('[%FT%T.%f] ') 435 436 logname = getpwuid(os.getuid()).pw_name 437 rer = '' 438 if self.reran is True: 439 rer = ' (RERAN)' 440 user = ' (run as %s)' % (self.user if len(self.user) else logname) 441 if self.identifier: 442 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 443 else: 444 msga = 'Test: %s%s ' % (self.pathname, user) 445 msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer) 446 pad = ' ' * (80 - (len(msga) + len(msgb))) 447 result_line = timeprefix + msga + pad + msgb 448 449 # The result line is always written to the log file. If -q was 450 # specified only failures are written to the console, otherwise 451 # the result line is written to the console. The console output 452 # may be suppressed by calling log() with suppress_console=True. 453 write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE) 454 if not suppress_console: 455 if not options.quiet: 456 write_log(result_line, LOG_OUT) 457 elif options.quiet and self.result.result != 'PASS': 458 write_log(result_line, LOG_OUT) 459 460 lines = sorted(self.result.stdout + self.result.stderr, 461 key=lambda x: x[0]) 462 463 # Write timestamped output (stdout and stderr) to the logfile 464 for dt, line in lines: 465 timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11], 466 encoding='utf-8') 467 write_log(b'%s %s\n' % (timestamp, line), LOG_FILE) 468 469 # Write the separate stdout/stderr/merged files, if the data exists 470 if len(self.result.stdout): 471 with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out: 472 for _, line in self.result.stdout: 473 os.write(out.fileno(), b'%s\n' % line) 474 if len(self.result.stderr): 475 with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err: 476 for _, line in self.result.stderr: 477 os.write(err.fileno(), b'%s\n' % line) 478 if len(self.result.stdout) and len(self.result.stderr): 479 with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged: 480 for _, line in lines: 481 os.write(merged.fileno(), b'%s\n' % line) 482 if len(self.result.kmemleak): 483 with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem: 484 kmem.write(self.result.kmemleak) 485 486 487class Test(Cmd): 488 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 489 'post_user', 'failsafe', 'failsafe_user', 'tags'] 490 491 def __init__(self, pathname, 492 pre=None, pre_user=None, post=None, post_user=None, 493 failsafe=None, failsafe_user=None, tags=None, **kwargs): 494 super(Test, self).__init__(pathname, **kwargs) 495 self.pre = pre or '' 496 self.pre_user = pre_user or '' 497 self.post = post or '' 498 self.post_user = post_user or '' 499 self.failsafe = failsafe or '' 500 self.failsafe_user = failsafe_user or '' 501 self.tags = tags or [] 502 503 def __str__(self): 504 post_user = pre_user = failsafe_user = '' 505 if len(self.pre_user): 506 pre_user = ' (as %s)' % (self.pre_user) 507 if len(self.post_user): 508 post_user = ' (as %s)' % (self.post_user) 509 if len(self.failsafe_user): 510 failsafe_user = ' (as %s)' % (self.failsafe_user) 511 return '''\ 512Pathname: %s 513Identifier: %s 514Outputdir: %s 515Timeout: %d 516User: %s 517Pre: %s%s 518Post: %s%s 519Failsafe: %s%s 520Tags: %s 521''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 522 self.pre, pre_user, self.post, post_user, self.failsafe, 523 failsafe_user, self.tags) 524 525 def verify(self): 526 """ 527 Check the pre/post/failsafe scripts, user and Test. Omit the Test from 528 this run if there are any problems. 529 """ 530 files = [self.pre, self.pathname, self.post, self.failsafe] 531 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 532 533 for f in [f for f in files if len(f)]: 534 if not verify_file(f): 535 write_log("Warning: Test '%s' not added to this run because" 536 " it failed verification.\n" % f, LOG_ERR) 537 return False 538 539 for user in [user for user in users if len(user)]: 540 if not verify_user(user): 541 write_log("Not adding Test '%s' to this run.\n" % 542 self.pathname, LOG_ERR) 543 return False 544 545 return True 546 547 def run(self, options, dryrun=None, kmemleak=None): 548 """ 549 Create Cmd instances for the pre/post/failsafe scripts. If the pre 550 script doesn't pass, skip this Test. Run the post script regardless. 551 If the Test is killed, also run the failsafe script. 552 """ 553 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 554 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 555 timeout=self.timeout, user=self.pre_user) 556 test = Cmd(self.pathname, identifier=self.identifier, 557 outputdir=self.outputdir, timeout=self.timeout, 558 user=self.user) 559 odir = os.path.join(self.outputdir, os.path.basename(self.failsafe)) 560 failsafe = Cmd(self.failsafe, identifier=self.identifier, 561 outputdir=odir, timeout=self.timeout, 562 user=self.failsafe_user) 563 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 564 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 565 timeout=self.timeout, user=self.post_user) 566 567 cont = True 568 if len(pretest.pathname): 569 pretest.run(options, kmemleak=False) 570 cont = pretest.result.result == 'PASS' 571 pretest.log(options) 572 573 if cont: 574 test.run(options, kmemleak=kmemleak) 575 if test.result.result == 'KILLED' and len(failsafe.pathname): 576 failsafe.run(options, kmemleak=False) 577 failsafe.log(options, suppress_console=True) 578 else: 579 test.skip() 580 581 test.log(options) 582 583 if len(posttest.pathname): 584 posttest.run(options, kmemleak=False) 585 posttest.log(options) 586 587 588class TestGroup(Test): 589 props = Test.props + ['tests'] 590 591 def __init__(self, pathname, tests=None, **kwargs): 592 super(TestGroup, self).__init__(pathname, **kwargs) 593 self.tests = tests or [] 594 595 def __str__(self): 596 post_user = pre_user = failsafe_user = '' 597 if len(self.pre_user): 598 pre_user = ' (as %s)' % (self.pre_user) 599 if len(self.post_user): 600 post_user = ' (as %s)' % (self.post_user) 601 if len(self.failsafe_user): 602 failsafe_user = ' (as %s)' % (self.failsafe_user) 603 return '''\ 604Pathname: %s 605Identifier: %s 606Outputdir: %s 607Tests: %s 608Timeout: %s 609User: %s 610Pre: %s%s 611Post: %s%s 612Failsafe: %s%s 613Tags: %s 614''' % (self.pathname, self.identifier, self.outputdir, self.tests, 615 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 616 self.failsafe, failsafe_user, self.tags) 617 618 def filter(self, keeplist): 619 self.tests = [x for x in self.tests if x in keeplist] 620 621 def verify(self): 622 """ 623 Check the pre/post/failsafe scripts, user and tests in this TestGroup. 624 Omit the TestGroup entirely, or simply delete the relevant tests in the 625 group, if that's all that's required. 626 """ 627 # If the pre/post/failsafe scripts are relative pathnames, convert to 628 # absolute, so they stand a chance of passing verification. 629 if len(self.pre) and not os.path.isabs(self.pre): 630 self.pre = os.path.join(self.pathname, self.pre) 631 if len(self.post) and not os.path.isabs(self.post): 632 self.post = os.path.join(self.pathname, self.post) 633 if len(self.failsafe) and not os.path.isabs(self.failsafe): 634 self.post = os.path.join(self.pathname, self.post) 635 636 auxfiles = [self.pre, self.post, self.failsafe] 637 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 638 639 for f in [f for f in auxfiles if len(f)]: 640 if f != self.failsafe and self.pathname != os.path.dirname(f): 641 write_log("Warning: TestGroup '%s' not added to this run. " 642 "Auxiliary script '%s' exists in a different " 643 "directory.\n" % (self.pathname, f), LOG_ERR) 644 return False 645 646 if not verify_file(f): 647 write_log("Warning: TestGroup '%s' not added to this run. " 648 "Auxiliary script '%s' failed verification.\n" % 649 (self.pathname, f), LOG_ERR) 650 return False 651 652 for user in [user for user in users if len(user)]: 653 if not verify_user(user): 654 write_log("Not adding TestGroup '%s' to this run.\n" % 655 self.pathname, LOG_ERR) 656 return False 657 658 # If one of the tests is invalid, delete it, log it, and drive on. 659 for test in self.tests: 660 if not verify_file(os.path.join(self.pathname, test)): 661 del self.tests[self.tests.index(test)] 662 write_log("Warning: Test '%s' removed from TestGroup '%s' " 663 "because it failed verification.\n" % 664 (test, self.pathname), LOG_ERR) 665 666 return len(self.tests) != 0 667 668 def run(self, options, dryrun=None, kmemleak=None): 669 """ 670 Create Cmd instances for the pre/post/failsafe scripts. If the pre 671 script doesn't pass, skip all the tests in this TestGroup. Run the 672 post script regardless. Run the failsafe script when a test is killed. 673 """ 674 # tags assigned to this test group also include the test names 675 if options.tags and not set(self.tags).intersection(set(options.tags)): 676 return 677 678 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 679 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 680 user=self.pre_user, identifier=self.identifier) 681 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 682 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 683 user=self.post_user, identifier=self.identifier) 684 685 cont = True 686 if len(pretest.pathname): 687 pretest.run(options, dryrun=dryrun, kmemleak=False) 688 cont = pretest.result.result == 'PASS' 689 pretest.log(options) 690 691 for fname in self.tests: 692 odir = os.path.join(self.outputdir, fname) 693 test = Cmd(os.path.join(self.pathname, fname), outputdir=odir, 694 timeout=self.timeout, user=self.user, 695 identifier=self.identifier) 696 odir = os.path.join(odir, os.path.basename(self.failsafe)) 697 failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout, 698 user=self.failsafe_user, identifier=self.identifier) 699 if cont: 700 test.run(options, dryrun=dryrun, kmemleak=kmemleak) 701 if test.result.result == 'KILLED' and len(failsafe.pathname): 702 failsafe.run(options, dryrun=dryrun, kmemleak=False) 703 failsafe.log(options, suppress_console=True) 704 else: 705 test.skip() 706 707 test.log(options) 708 709 if len(posttest.pathname): 710 posttest.run(options, dryrun=dryrun, kmemleak=False) 711 posttest.log(options) 712 713 714class TestRun(object): 715 props = ['quiet', 'outputdir', 'debug'] 716 717 def __init__(self, options): 718 self.tests = {} 719 self.testgroups = {} 720 self.starttime = time() 721 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 722 self.outputdir = os.path.join(options.outputdir, self.timestamp) 723 self.setup_logging(options) 724 self.defaults = [ 725 ('outputdir', BASEDIR), 726 ('quiet', False), 727 ('timeout', 60), 728 ('user', ''), 729 ('pre', ''), 730 ('pre_user', ''), 731 ('post', ''), 732 ('post_user', ''), 733 ('failsafe', ''), 734 ('failsafe_user', ''), 735 ('tags', []), 736 ('debug', False) 737 ] 738 739 def __str__(self): 740 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 741 s += 'TESTS:\n' 742 for key in sorted(self.tests.keys()): 743 s += '%s%s' % (self.tests[key].__str__(), '\n') 744 s += 'TESTGROUPS:\n' 745 for key in sorted(self.testgroups.keys()): 746 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 747 return s 748 749 def addtest(self, pathname, options): 750 """ 751 Create a new Test, and apply any properties that were passed in 752 from the command line. If it passes verification, add it to the 753 TestRun. 754 """ 755 test = Test(pathname) 756 for prop in Test.props: 757 setattr(test, prop, getattr(options, prop)) 758 759 if test.verify(): 760 self.tests[pathname] = test 761 762 def addtestgroup(self, dirname, filenames, options): 763 """ 764 Create a new TestGroup, and apply any properties that were passed 765 in from the command line. If it passes verification, add it to the 766 TestRun. 767 """ 768 if dirname not in self.testgroups: 769 testgroup = TestGroup(dirname) 770 for prop in Test.props: 771 setattr(testgroup, prop, getattr(options, prop)) 772 773 # Prevent pre/post/failsafe scripts from running as regular tests 774 for f in [testgroup.pre, testgroup.post, testgroup.failsafe]: 775 if f in filenames: 776 del filenames[filenames.index(f)] 777 778 self.testgroups[dirname] = testgroup 779 self.testgroups[dirname].tests = sorted(filenames) 780 781 testgroup.verify() 782 783 def filter(self, keeplist): 784 for group in list(self.testgroups.keys()): 785 if group not in keeplist: 786 del self.testgroups[group] 787 continue 788 789 g = self.testgroups[group] 790 791 if g.pre and os.path.basename(g.pre) in keeplist[group]: 792 continue 793 794 g.filter(keeplist[group]) 795 796 for test in list(self.tests.keys()): 797 directory, base = os.path.split(test) 798 if directory not in keeplist or base not in keeplist[directory]: 799 del self.tests[test] 800 801 def read(self, options): 802 """ 803 Read in the specified runfiles, and apply the TestRun properties 804 listed in the 'DEFAULT' section to our TestRun. Then read each 805 section, and apply the appropriate properties to the Test or 806 TestGroup. Properties from individual sections override those set 807 in the 'DEFAULT' section. If the Test or TestGroup passes 808 verification, add it to the TestRun. 809 """ 810 config = configparser.RawConfigParser() 811 parsed = config.read(options.runfiles) 812 failed = options.runfiles - set(parsed) 813 if len(failed): 814 files = ' '.join(sorted(failed)) 815 fail("Couldn't read config files: %s" % files) 816 817 for opt in TestRun.props: 818 if config.has_option('DEFAULT', opt): 819 if opt == 'outputdir': 820 outputdir = config.get('DEFAULT', opt) 821 setattr(self, opt, os.path.join(outputdir, self.timestamp)) 822 else: 823 setattr(self, opt, config.get('DEFAULT', opt)) 824 825 testdir = options.testdir 826 827 for section in config.sections(): 828 if 'tests' in config.options(section): 829 parts = section.split(':', 1) 830 sectiondir = parts[0] 831 identifier = parts[1] if len(parts) == 2 else None 832 if os.path.isdir(sectiondir): 833 pathname = sectiondir 834 elif os.path.isdir(os.path.join(testdir, sectiondir)): 835 pathname = os.path.join(testdir, sectiondir) 836 else: 837 pathname = sectiondir 838 839 testgroup = TestGroup(os.path.abspath(pathname), 840 identifier=identifier) 841 for prop in TestGroup.props: 842 for sect in ['DEFAULT', section]: 843 if config.has_option(sect, prop): 844 if prop == 'tags': 845 setattr(testgroup, prop, 846 eval(config.get(sect, prop))) 847 elif prop == 'failsafe': 848 failsafe = config.get(sect, prop) 849 setattr(testgroup, prop, 850 os.path.join(testdir, failsafe)) 851 elif prop == 'outputdir': 852 outputdir = config.get(sect, prop) 853 setattr(self, opt, 854 os.path.join(outputdir, 855 self.timestamp)) 856 else: 857 setattr(testgroup, prop, 858 config.get(sect, prop)) 859 860 # Repopulate tests using eval to convert the string to a list 861 testgroup.tests = eval(config.get(section, 'tests')) 862 863 if testgroup.verify(): 864 self.testgroups[section] = testgroup 865 else: 866 test = Test(section) 867 for prop in Test.props: 868 for sect in ['DEFAULT', section]: 869 if config.has_option(sect, prop): 870 if prop == 'failsafe': 871 failsafe = config.get(sect, prop) 872 setattr(test, prop, 873 os.path.join(testdir, failsafe)) 874 elif prop == 'outputdir': 875 outputdir = config.get(sect, prop) 876 setattr(self, opt, 877 os.path.join(outputdir, 878 self.timestamp)) 879 else: 880 setattr(test, prop, config.get(sect, prop)) 881 882 if test.verify(): 883 self.tests[section] = test 884 885 def write(self, options): 886 """ 887 Create a configuration file for editing and later use. The 888 'DEFAULT' section of the config file is created from the 889 properties that were specified on the command line. Tests are 890 simply added as sections that inherit everything from the 891 'DEFAULT' section. TestGroups are the same, except they get an 892 option including all the tests to run in that directory. 893 """ 894 895 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 896 self.defaults]) 897 config = configparser.RawConfigParser(defaults) 898 899 for test in sorted(self.tests.keys()): 900 config.add_section(test) 901 for prop in Test.props: 902 if prop not in self.props: 903 config.set(test, prop, 904 getattr(self.tests[test], prop)) 905 906 for testgroup in sorted(self.testgroups.keys()): 907 config.add_section(testgroup) 908 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 909 for prop in TestGroup.props: 910 if prop not in self.props: 911 config.set(testgroup, prop, 912 getattr(self.testgroups[testgroup], prop)) 913 914 try: 915 with open(options.template, 'w') as f: 916 return config.write(f) 917 except IOError: 918 fail('Could not open \'%s\' for writing.' % options.template) 919 920 def complete_outputdirs(self): 921 """ 922 Collect all the pathnames for Tests, and TestGroups. Strip off all 923 common leading path components, and append what remains to the top 924 "output" dir, to create a tree of output directories that match 925 the test and group names in structure. Tests will be able 926 to write output files directly in the newly modified outputdir. 927 TestGroups will be able to create one subdirectory per test in the 928 outputdir, and are guaranteed uniqueness because a group can only 929 contain files in one directory. Pre and post tests will create a 930 directory rooted at the outputdir of the Test or TestGroup in 931 question for their output. Failsafe scripts will create a directory 932 rooted at the outputdir of each Test for their output. 933 """ 934 935 alltests = dict(list(self.tests.items()) + 936 list(self.testgroups.items())) 937 base = os.path.join(self.outputdir, 'output') 938 939 seen = [] 940 941 for path in list(alltests.keys()): 942 frag = path.split('/') 943 for i in range(0, len(frag)): 944 if len(seen) == i: 945 seen.append({}) 946 seen[i][frag[i]] = 1 947 948 cut = 0 949 for i in range(0, len(seen)): 950 if len(list(seen[i].keys())) == 1: 951 cut += 1 952 else: 953 break 954 955 for path in list(alltests.keys()): 956 uniq = path.split('/', cut)[-1] 957 alltests[path].outputdir = os.path.join(base, uniq) 958 959 def setup_logging(self, options): 960 """ 961 This function creates the output directory and gets a file object 962 for the logfile. This function must be called before write_log() 963 can be used. 964 """ 965 if options.dryrun is True: 966 return 967 968 global LOG_FILE_OBJ 969 if not options.template: 970 try: 971 old = os.umask(0) 972 os.makedirs(self.outputdir, mode=0o777) 973 os.umask(old) 974 filename = os.path.join(self.outputdir, 'log') 975 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') 976 except OSError as e: 977 fail('%s' % e) 978 979 def run(self, options): 980 """ 981 Walk through all the Tests and TestGroups, calling run(). 982 """ 983 try: 984 os.chdir(self.outputdir) 985 except OSError: 986 fail('Could not change to directory %s' % self.outputdir) 987 # make a symlink to the output for the currently running test 988 logsymlink = os.path.join(self.outputdir, '../current') 989 if os.path.islink(logsymlink): 990 os.unlink(logsymlink) 991 if not os.path.exists(logsymlink): 992 os.symlink(self.outputdir, logsymlink) 993 else: 994 write_log('Could not make a symlink to directory %s\n' % 995 self.outputdir, LOG_ERR) 996 997 if options.kmemleak: 998 cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"' 999 check_output(cmd, shell=True) 1000 1001 iteration = 0 1002 while iteration < options.iterations: 1003 for test in sorted(self.tests.keys()): 1004 self.tests[test].run(options) 1005 for testgroup in sorted(self.testgroups.keys()): 1006 self.testgroups[testgroup].run(options) 1007 iteration += 1 1008 1009 def summary(self): 1010 if Result.total == 0: 1011 return 2 1012 1013 print('\nResults Summary') 1014 for key in list(Result.runresults.keys()): 1015 if Result.runresults[key] != 0: 1016 print('%s\t% 4d' % (key, Result.runresults[key])) 1017 1018 m, s = divmod(time() - self.starttime, 60) 1019 h, m = divmod(m, 60) 1020 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 1021 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 1022 float(Result.total)) * 100)) 1023 print('Log directory:\t%s' % self.outputdir) 1024 1025 if Result.runresults['FAIL'] > 0: 1026 return 1 1027 1028 if Result.runresults['KILLED'] > 0: 1029 return 1 1030 1031 if Result.runresults['RERAN'] > 0: 1032 return 3 1033 1034 return 0 1035 1036 1037def write_log(msg, target): 1038 """ 1039 Write the provided message to standard out, standard error or 1040 the logfile. If specifying LOG_FILE, then `msg` must be a bytes 1041 like object. This way we can still handle output from tests that 1042 may be in unexpected encodings. 1043 """ 1044 if target == LOG_OUT: 1045 os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8')) 1046 elif target == LOG_ERR: 1047 os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8')) 1048 elif target == LOG_FILE: 1049 os.write(LOG_FILE_OBJ.fileno(), msg) 1050 else: 1051 fail('log_msg called with unknown target "%s"' % target) 1052 1053 1054def verify_file(pathname): 1055 """ 1056 Verify that the supplied pathname is an executable regular file. 1057 """ 1058 if os.path.isdir(pathname) or os.path.islink(pathname): 1059 return False 1060 1061 for ext in '', '.ksh', '.sh': 1062 script_path = pathname + ext 1063 if os.path.isfile(script_path) and os.access(script_path, os.X_OK): 1064 return True 1065 1066 return False 1067 1068 1069def verify_user(user): 1070 """ 1071 Verify that the specified user exists on this system, and can execute 1072 sudo without being prompted for a password. 1073 """ 1074 testcmd = [SUDO, '-n', '-u', user, TRUE] 1075 1076 if user in Cmd.verified_users: 1077 return True 1078 1079 try: 1080 getpwnam(user) 1081 except KeyError: 1082 write_log("Warning: user '%s' does not exist.\n" % user, 1083 LOG_ERR) 1084 return False 1085 1086 p = Popen(testcmd) 1087 p.wait() 1088 if p.returncode != 0: 1089 write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user, 1090 LOG_ERR) 1091 return False 1092 else: 1093 Cmd.verified_users.append(user) 1094 1095 return True 1096 1097 1098def find_tests(testrun, options): 1099 """ 1100 For the given list of pathnames, add files as Tests. For directories, 1101 if do_groups is True, add the directory as a TestGroup. If False, 1102 recursively search for executable files. 1103 """ 1104 1105 for p in sorted(options.pathnames): 1106 if os.path.isdir(p): 1107 for dirname, _, filenames in os.walk(p): 1108 if options.do_groups: 1109 testrun.addtestgroup(dirname, filenames, options) 1110 else: 1111 for f in sorted(filenames): 1112 testrun.addtest(os.path.join(dirname, f), options) 1113 else: 1114 testrun.addtest(p, options) 1115 1116 1117def filter_tests(testrun, options): 1118 try: 1119 fh = open(options.logfile, "r") 1120 except Exception as e: 1121 fail('%s' % e) 1122 1123 failed = {} 1124 while True: 1125 line = fh.readline() 1126 if not line: 1127 break 1128 m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line) 1129 if not m: 1130 continue 1131 group, test = m.group(1, 2) 1132 try: 1133 failed[group].append(test) 1134 except KeyError: 1135 failed[group] = [test] 1136 fh.close() 1137 1138 testrun.filter(failed) 1139 1140 1141def fail(retstr, ret=1): 1142 print('%s: %s' % (sys.argv[0], retstr)) 1143 exit(ret) 1144 1145 1146def kmemleak_cb(option, opt_str, value, parser): 1147 if not os.path.exists(KMEMLEAK_FILE): 1148 fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " + 1149 "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.") 1150 1151 setattr(parser.values, option.dest, True) 1152 1153 1154def options_cb(option, opt_str, value, parser): 1155 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 1156 1157 if opt_str in parser.rargs: 1158 fail('%s may only be specified once.' % opt_str) 1159 1160 if option.dest == 'runfiles': 1161 parser.values.cmd = 'rdconfig' 1162 value = set(os.path.abspath(p) for p in value.split(',')) 1163 if option.dest == 'tags': 1164 value = [x.strip() for x in value.split(',')] 1165 1166 if option.dest in path_options: 1167 setattr(parser.values, option.dest, os.path.abspath(value)) 1168 else: 1169 setattr(parser.values, option.dest, value) 1170 1171 1172def parse_args(): 1173 parser = OptionParser() 1174 parser.add_option('-c', action='callback', callback=options_cb, 1175 type='string', dest='runfiles', metavar='runfiles', 1176 help='Specify tests to run via config files.') 1177 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 1178 help='Dry run. Print tests, but take no other action.') 1179 parser.add_option('-D', action='store_true', default=False, dest='debug', 1180 help='Write all test output to stdout as it arrives.') 1181 parser.add_option('-l', action='callback', callback=options_cb, 1182 default=None, dest='logfile', metavar='logfile', 1183 type='string', 1184 help='Read logfile and re-run tests which failed.') 1185 parser.add_option('-g', action='store_true', default=False, 1186 dest='do_groups', help='Make directories TestGroups.') 1187 parser.add_option('-o', action='callback', callback=options_cb, 1188 default=BASEDIR, dest='outputdir', type='string', 1189 metavar='outputdir', help='Specify an output directory.') 1190 parser.add_option('-O', action='store_true', default=False, 1191 dest='timeout_debug', 1192 help='Dump debugging info to /dev/kmsg on test timeout') 1193 parser.add_option('-i', action='callback', callback=options_cb, 1194 default=TESTDIR, dest='testdir', type='string', 1195 metavar='testdir', help='Specify a test directory.') 1196 parser.add_option('-K', action='store_true', default=False, dest='kmsg', 1197 help='Log tests names to /dev/kmsg') 1198 parser.add_option('-m', action='callback', callback=kmemleak_cb, 1199 default=False, dest='kmemleak', 1200 help='Enable kmemleak reporting (Linux only)') 1201 parser.add_option('-p', action='callback', callback=options_cb, 1202 default='', dest='pre', metavar='script', 1203 type='string', help='Specify a pre script.') 1204 parser.add_option('-P', action='callback', callback=options_cb, 1205 default='', dest='post', metavar='script', 1206 type='string', help='Specify a post script.') 1207 parser.add_option('-q', action='store_true', default=False, dest='quiet', 1208 help='Silence on the console during a test run.') 1209 parser.add_option('-s', action='callback', callback=options_cb, 1210 default='', dest='failsafe', metavar='script', 1211 type='string', help='Specify a failsafe script.') 1212 parser.add_option('-S', action='callback', callback=options_cb, 1213 default='', dest='failsafe_user', 1214 metavar='failsafe_user', type='string', 1215 help='Specify a user to execute the failsafe script.') 1216 parser.add_option('-t', action='callback', callback=options_cb, default=60, 1217 dest='timeout', metavar='seconds', type='int', 1218 help='Timeout (in seconds) for an individual test.') 1219 parser.add_option('-u', action='callback', callback=options_cb, 1220 default='', dest='user', metavar='user', type='string', 1221 help='Specify a different user name to run as.') 1222 parser.add_option('-w', action='callback', callback=options_cb, 1223 default=None, dest='template', metavar='template', 1224 type='string', help='Create a new config file.') 1225 parser.add_option('-x', action='callback', callback=options_cb, default='', 1226 dest='pre_user', metavar='pre_user', type='string', 1227 help='Specify a user to execute the pre script.') 1228 parser.add_option('-X', action='callback', callback=options_cb, default='', 1229 dest='post_user', metavar='post_user', type='string', 1230 help='Specify a user to execute the post script.') 1231 parser.add_option('-T', action='callback', callback=options_cb, default='', 1232 dest='tags', metavar='tags', type='string', 1233 help='Specify tags to execute specific test groups.') 1234 parser.add_option('-I', action='callback', callback=options_cb, default=1, 1235 dest='iterations', metavar='iterations', type='int', 1236 help='Number of times to run the test run.') 1237 (options, pathnames) = parser.parse_args() 1238 1239 if options.runfiles and len(pathnames): 1240 fail('Extraneous arguments.') 1241 1242 options.pathnames = [os.path.abspath(path) for path in pathnames] 1243 1244 return options 1245 1246 1247def main(): 1248 options = parse_args() 1249 1250 testrun = TestRun(options) 1251 1252 if options.runfiles: 1253 testrun.read(options) 1254 else: 1255 find_tests(testrun, options) 1256 1257 if options.logfile: 1258 filter_tests(testrun, options) 1259 1260 if options.template: 1261 testrun.write(options) 1262 exit(0) 1263 1264 testrun.complete_outputdirs() 1265 testrun.run(options) 1266 exit(testrun.summary()) 1267 1268 1269if __name__ == '__main__': 1270 main() 1271