1#!/usr/bin/env @PYTHON_SHEBANG@ 2# SPDX-License-Identifier: CDDL-1.0 3 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# http://www.illumos.org/license/CDDL. 13# 14 15# 16# Copyright (c) 2012, 2018 by Delphix. All rights reserved. 17# Copyright (c) 2019 Datto Inc. 18# 19# This script must remain compatible with Python 3.6+. 20# 21 22import os 23import sys 24import ctypes 25import re 26import configparser 27 28from datetime import datetime 29from optparse import OptionParser 30from pwd import getpwnam 31from pwd import getpwuid 32from select import select 33from subprocess import PIPE 34from subprocess import Popen 35from subprocess import check_output 36from threading import Timer 37from time import time, CLOCK_MONOTONIC 38from os.path import exists 39 40BASEDIR = '/var/tmp/test_results' 41TESTDIR = '/usr/share/zfs/' 42KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak' 43KILL = 'kill' 44TRUE = 'true' 45SUDO = 'sudo' 46LOG_FILE = 'LOG_FILE' 47LOG_OUT = 'LOG_OUT' 48LOG_ERR = 'LOG_ERR' 49LOG_FILE_OBJ = None 50 51try: 52 from time import monotonic as monotonic_time 53except ImportError: 54 class timespec(ctypes.Structure): 55 _fields_ = [ 56 ('tv_sec', ctypes.c_long), 57 ('tv_nsec', ctypes.c_long) 58 ] 59 60 librt = ctypes.CDLL('librt.so.1', use_errno=True) 61 clock_gettime = librt.clock_gettime 62 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] 63 64 def monotonic_time(): 65 t = timespec() 66 if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0: 67 errno_ = ctypes.get_errno() 68 raise OSError(errno_, os.strerror(errno_)) 69 return t.tv_sec + t.tv_nsec * 1e-9 70 71 72class Result(object): 73 total = 0 74 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0} 75 76 def __init__(self): 77 self.starttime = None 78 self.returncode = None 79 self.runtime = '' 80 self.stdout = [] 81 self.stderr = [] 82 self.kmemleak = '' 83 self.result = '' 84 85 def done(self, proc, killed, reran): 86 """ 87 Finalize the results of this Cmd. 88 """ 89 Result.total += 1 90 m, s = divmod(monotonic_time() - self.starttime, 60) 91 self.runtime = '%02d:%02d' % (m, s) 92 self.returncode = proc.returncode 93 if reran is True: 94 Result.runresults['RERAN'] += 1 95 if killed: 96 self.result = 'KILLED' 97 Result.runresults['KILLED'] += 1 98 elif len(self.kmemleak) > 0: 99 self.result = 'FAIL' 100 Result.runresults['FAIL'] += 1 101 elif self.returncode == 0: 102 self.result = 'PASS' 103 Result.runresults['PASS'] += 1 104 elif self.returncode == 4: 105 self.result = 'SKIP' 106 Result.runresults['SKIP'] += 1 107 elif self.returncode != 0: 108 self.result = 'FAIL' 109 Result.runresults['FAIL'] += 1 110 111 112class Output(object): 113 """ 114 This class is a slightly modified version of the 'Stream' class found 115 here: https://stackoverflow.com/q/4984549/ 116 """ 117 def __init__(self, stream, debug=False): 118 self.stream = stream 119 self.debug = debug 120 self._buf = b'' 121 self.lines = [] 122 123 def fileno(self): 124 return self.stream.fileno() 125 126 def read(self, drain=0): 127 """ 128 Read from the file descriptor. If 'drain' set, read until EOF. 129 """ 130 while self._read() is not None: 131 if not drain: 132 break 133 134 def _read(self): 135 """ 136 Read up to 4k of data from this output stream. Collect the output 137 up to the last newline, and append it to any leftover data from a 138 previous call. The lines are stored as a (timestamp, data) tuple 139 for easy sorting/merging later. 140 """ 141 fd = self.fileno() 142 buf = os.read(fd, 4096) 143 if not buf: 144 return None 145 if self.debug: 146 os.write(sys.stderr.fileno(), buf) 147 if b'\n' not in buf: 148 self._buf += buf 149 return [] 150 151 buf = self._buf + buf 152 tmp, rest = buf.rsplit(b'\n', 1) 153 self._buf = rest 154 now = datetime.now() 155 rows = tmp.split(b'\n') 156 self.lines += [(now, r) for r in rows] 157 158 159class Cmd(object): 160 verified_users = [] 161 162 def __init__(self, pathname, identifier=None, outputdir=None, 163 timeout=None, user=None, tags=None): 164 self.pathname = pathname 165 self.identifier = identifier 166 self.outputdir = outputdir or 'BASEDIR' 167 """ 168 The timeout for tests is measured in wall-clock time 169 """ 170 self.timeout = timeout 171 self.user = user or '' 172 self.killed = False 173 self.reran = None 174 self.result = Result() 175 176 if self.timeout is None: 177 self.timeout = 60 178 179 def __str__(self): 180 return '''\ 181Pathname: %s 182Identifier: %s 183Outputdir: %s 184Timeout: %d 185User: %s 186''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 187 188 def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False): 189 """ 190 Kill a running command due to timeout, or ^C from the keyboard. If 191 sudo is required, this user was verified previously. 192 """ 193 self.killed = True 194 do_sudo = len(self.user) != 0 195 signal = '-TERM' 196 197 cmd = [SUDO, KILL, signal, str(proc.pid)] 198 if not do_sudo: 199 del cmd[0] 200 201 try: 202 kp = Popen(cmd) 203 kp.wait() 204 except Exception: 205 pass 206 207 """ 208 If this is not a user-initiated kill and the test has not been 209 reran before we consider if the test needs to be reran: 210 If the test has spent some time hibernating and didn't run the whole 211 length of time before being timed out we will rerun the test. 212 """ 213 if keyboard_interrupt is False and self.reran is None: 214 runtime = monotonic_time() - self.result.starttime 215 if int(self.timeout) > runtime: 216 self.killed = False 217 self.reran = False 218 self.run(options, dryrun=False, kmemleak=kmemleak) 219 self.reran = True 220 221 def update_cmd_privs(self, cmd, user): 222 """ 223 If a user has been specified to run this Cmd and we're not already 224 running as that user, prepend the appropriate sudo command to run 225 as that user. 226 """ 227 me = getpwuid(os.getuid()) 228 229 if not user or user is me: 230 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 231 cmd += '.ksh' 232 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 233 cmd += '.sh' 234 return cmd 235 236 if not os.path.isfile(cmd): 237 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 238 cmd += '.ksh' 239 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 240 cmd += '.sh' 241 242 # glibc (at least) will not pass TMPDIR through to setuid programs. 243 # if set, arrange for it to be reset before running the target cmd 244 tmpdir = os.getenv('TMPDIR') 245 if tmpdir: 246 tmpdirarg = 'env TMPDIR=%s' % tmpdir 247 else: 248 tmpdirarg = '' 249 250 ret = '%s -E -u %s %s %s' % (SUDO, user, tmpdirarg, cmd) 251 return ret.split(' ') 252 253 def collect_output(self, proc, debug=False): 254 """ 255 Read from stdout/stderr as data becomes available, until the 256 process is no longer running. Return the lines from the stdout and 257 stderr Output objects. 258 """ 259 out = Output(proc.stdout, debug) 260 err = Output(proc.stderr, debug) 261 res = [] 262 while proc.returncode is None: 263 proc.poll() 264 res = select([out, err], [], [], .1) 265 for fd in res[0]: 266 fd.read() 267 for fd in res[0]: 268 fd.read(drain=1) 269 270 return out.lines, err.lines 271 272 def run(self, options, dryrun=None, kmemleak=None): 273 """ 274 This is the main function that runs each individual test. 275 Determine whether or not the command requires sudo, and modify it 276 if needed. Run the command, and update the result object. 277 """ 278 if dryrun is None: 279 dryrun = options.dryrun 280 if dryrun is True: 281 print(self) 282 return 283 if kmemleak is None: 284 kmemleak = options.kmemleak 285 286 privcmd = self.update_cmd_privs(self.pathname, self.user) 287 try: 288 old = os.umask(0) 289 if not os.path.isdir(self.outputdir): 290 os.makedirs(self.outputdir, mode=0o777) 291 os.umask(old) 292 except OSError as e: 293 fail('%s' % e) 294 295 """ 296 Log each test we run to /dev/kmsg (on Linux), so if there's a kernel 297 warning we'll be able to match it up to a particular test. 298 """ 299 if options.kmsg is True and exists("/dev/kmsg"): 300 try: 301 kp = Popen([SUDO, "sh", "-c", 302 f"echo ZTS run {self.pathname} > /dev/kmsg"]) 303 kp.wait() 304 except Exception: 305 pass 306 307 """ 308 Log each test we run to /dev/ttyu0 (on FreeBSD), so if there's a kernel 309 warning we'll be able to match it up to a particular test. 310 """ 311 if options.kmsg is True and exists("/dev/ttyu0"): 312 try: 313 kp = Popen([SUDO, "sh", "-c", 314 f"echo ZTS run {self.pathname} > /dev/ttyu0"]) 315 kp.wait() 316 except Exception: 317 pass 318 319 self.result.starttime = monotonic_time() 320 321 if kmemleak: 322 cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"' 323 check_output(cmd, shell=True) 324 325 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) 326 # Allow a special timeout value of 0 to mean infinity 327 if int(self.timeout) == 0: 328 self.timeout = sys.maxsize / (10 ** 9) 329 t = Timer( 330 int(self.timeout), self.kill_cmd, [proc, options, kmemleak] 331 ) 332 333 try: 334 t.start() 335 336 out, err = self.collect_output(proc, options.debug) 337 self.result.stdout = out 338 self.result.stderr = err 339 340 if kmemleak: 341 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"' 342 check_output(cmd, shell=True) 343 cmd = f'{SUDO} cat {KMEMLEAK_FILE}' 344 self.result.kmemleak = check_output(cmd, shell=True) 345 except KeyboardInterrupt: 346 self.kill_cmd(proc, options, kmemleak, True) 347 fail('\nRun terminated at user request.') 348 finally: 349 t.cancel() 350 351 if self.reran is not False: 352 self.result.done(proc, self.killed, self.reran) 353 354 def skip(self): 355 """ 356 Initialize enough of the test result that we can log a skipped 357 command. 358 """ 359 Result.total += 1 360 Result.runresults['SKIP'] += 1 361 self.result.stdout = self.result.stderr = [] 362 self.result.starttime = monotonic_time() 363 m, s = divmod(monotonic_time() - self.result.starttime, 60) 364 self.result.runtime = '%02d:%02d' % (m, s) 365 self.result.result = 'SKIP' 366 367 def log(self, options, suppress_console=False): 368 """ 369 This function is responsible for writing all output. This includes 370 the console output, the logfile of all results (with timestamped 371 merged stdout and stderr), and for each test, the unmodified 372 stdout/stderr/merged in its own file. 373 """ 374 375 logname = getpwuid(os.getuid()).pw_name 376 rer = '' 377 if self.reran is True: 378 rer = ' (RERAN)' 379 user = ' (run as %s)' % (self.user if len(self.user) else logname) 380 if self.identifier: 381 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 382 else: 383 msga = 'Test: %s%s ' % (self.pathname, user) 384 msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer) 385 pad = ' ' * (80 - (len(msga) + len(msgb))) 386 result_line = msga + pad + msgb 387 388 # The result line is always written to the log file. If -q was 389 # specified only failures are written to the console, otherwise 390 # the result line is written to the console. The console output 391 # may be suppressed by calling log() with suppress_console=True. 392 write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE) 393 if not suppress_console: 394 if not options.quiet: 395 write_log(result_line, LOG_OUT) 396 elif options.quiet and self.result.result != 'PASS': 397 write_log(result_line, LOG_OUT) 398 399 lines = sorted(self.result.stdout + self.result.stderr, 400 key=lambda x: x[0]) 401 402 # Write timestamped output (stdout and stderr) to the logfile 403 for dt, line in lines: 404 timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11], 405 encoding='utf-8') 406 write_log(b'%s %s\n' % (timestamp, line), LOG_FILE) 407 408 # Write the separate stdout/stderr/merged files, if the data exists 409 if len(self.result.stdout): 410 with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out: 411 for _, line in self.result.stdout: 412 os.write(out.fileno(), b'%s\n' % line) 413 if len(self.result.stderr): 414 with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err: 415 for _, line in self.result.stderr: 416 os.write(err.fileno(), b'%s\n' % line) 417 if len(self.result.stdout) and len(self.result.stderr): 418 with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged: 419 for _, line in lines: 420 os.write(merged.fileno(), b'%s\n' % line) 421 if len(self.result.kmemleak): 422 with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem: 423 kmem.write(self.result.kmemleak) 424 425 426class Test(Cmd): 427 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 428 'post_user', 'failsafe', 'failsafe_user', 'tags'] 429 430 def __init__(self, pathname, 431 pre=None, pre_user=None, post=None, post_user=None, 432 failsafe=None, failsafe_user=None, tags=None, **kwargs): 433 super(Test, self).__init__(pathname, **kwargs) 434 self.pre = pre or '' 435 self.pre_user = pre_user or '' 436 self.post = post or '' 437 self.post_user = post_user or '' 438 self.failsafe = failsafe or '' 439 self.failsafe_user = failsafe_user or '' 440 self.tags = tags or [] 441 442 def __str__(self): 443 post_user = pre_user = failsafe_user = '' 444 if len(self.pre_user): 445 pre_user = ' (as %s)' % (self.pre_user) 446 if len(self.post_user): 447 post_user = ' (as %s)' % (self.post_user) 448 if len(self.failsafe_user): 449 failsafe_user = ' (as %s)' % (self.failsafe_user) 450 return '''\ 451Pathname: %s 452Identifier: %s 453Outputdir: %s 454Timeout: %d 455User: %s 456Pre: %s%s 457Post: %s%s 458Failsafe: %s%s 459Tags: %s 460''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 461 self.pre, pre_user, self.post, post_user, self.failsafe, 462 failsafe_user, self.tags) 463 464 def verify(self): 465 """ 466 Check the pre/post/failsafe scripts, user and Test. Omit the Test from 467 this run if there are any problems. 468 """ 469 files = [self.pre, self.pathname, self.post, self.failsafe] 470 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 471 472 for f in [f for f in files if len(f)]: 473 if not verify_file(f): 474 write_log("Warning: Test '%s' not added to this run because" 475 " it failed verification.\n" % f, LOG_ERR) 476 return False 477 478 for user in [user for user in users if len(user)]: 479 if not verify_user(user): 480 write_log("Not adding Test '%s' to this run.\n" % 481 self.pathname, LOG_ERR) 482 return False 483 484 return True 485 486 def run(self, options, dryrun=None, kmemleak=None): 487 """ 488 Create Cmd instances for the pre/post/failsafe scripts. If the pre 489 script doesn't pass, skip this Test. Run the post script regardless. 490 If the Test is killed, also run the failsafe script. 491 """ 492 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 493 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 494 timeout=self.timeout, user=self.pre_user) 495 test = Cmd(self.pathname, identifier=self.identifier, 496 outputdir=self.outputdir, timeout=self.timeout, 497 user=self.user) 498 odir = os.path.join(self.outputdir, os.path.basename(self.failsafe)) 499 failsafe = Cmd(self.failsafe, identifier=self.identifier, 500 outputdir=odir, timeout=self.timeout, 501 user=self.failsafe_user) 502 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 503 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 504 timeout=self.timeout, user=self.post_user) 505 506 cont = True 507 if len(pretest.pathname): 508 pretest.run(options, kmemleak=False) 509 cont = pretest.result.result == 'PASS' 510 pretest.log(options) 511 512 if cont: 513 test.run(options, kmemleak=kmemleak) 514 if test.result.result == 'KILLED' and len(failsafe.pathname): 515 failsafe.run(options, kmemleak=False) 516 failsafe.log(options, suppress_console=True) 517 else: 518 test.skip() 519 520 test.log(options) 521 522 if len(posttest.pathname): 523 posttest.run(options, kmemleak=False) 524 posttest.log(options) 525 526 527class TestGroup(Test): 528 props = Test.props + ['tests'] 529 530 def __init__(self, pathname, tests=None, **kwargs): 531 super(TestGroup, self).__init__(pathname, **kwargs) 532 self.tests = tests or [] 533 534 def __str__(self): 535 post_user = pre_user = failsafe_user = '' 536 if len(self.pre_user): 537 pre_user = ' (as %s)' % (self.pre_user) 538 if len(self.post_user): 539 post_user = ' (as %s)' % (self.post_user) 540 if len(self.failsafe_user): 541 failsafe_user = ' (as %s)' % (self.failsafe_user) 542 return '''\ 543Pathname: %s 544Identifier: %s 545Outputdir: %s 546Tests: %s 547Timeout: %s 548User: %s 549Pre: %s%s 550Post: %s%s 551Failsafe: %s%s 552Tags: %s 553''' % (self.pathname, self.identifier, self.outputdir, self.tests, 554 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 555 self.failsafe, failsafe_user, self.tags) 556 557 def filter(self, keeplist): 558 self.tests = [x for x in self.tests if x in keeplist] 559 560 def verify(self): 561 """ 562 Check the pre/post/failsafe scripts, user and tests in this TestGroup. 563 Omit the TestGroup entirely, or simply delete the relevant tests in the 564 group, if that's all that's required. 565 """ 566 # If the pre/post/failsafe scripts are relative pathnames, convert to 567 # absolute, so they stand a chance of passing verification. 568 if len(self.pre) and not os.path.isabs(self.pre): 569 self.pre = os.path.join(self.pathname, self.pre) 570 if len(self.post) and not os.path.isabs(self.post): 571 self.post = os.path.join(self.pathname, self.post) 572 if len(self.failsafe) and not os.path.isabs(self.failsafe): 573 self.post = os.path.join(self.pathname, self.post) 574 575 auxfiles = [self.pre, self.post, self.failsafe] 576 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 577 578 for f in [f for f in auxfiles if len(f)]: 579 if f != self.failsafe and self.pathname != os.path.dirname(f): 580 write_log("Warning: TestGroup '%s' not added to this run. " 581 "Auxiliary script '%s' exists in a different " 582 "directory.\n" % (self.pathname, f), LOG_ERR) 583 return False 584 585 if not verify_file(f): 586 write_log("Warning: TestGroup '%s' not added to this run. " 587 "Auxiliary script '%s' failed verification.\n" % 588 (self.pathname, f), LOG_ERR) 589 return False 590 591 for user in [user for user in users if len(user)]: 592 if not verify_user(user): 593 write_log("Not adding TestGroup '%s' to this run.\n" % 594 self.pathname, LOG_ERR) 595 return False 596 597 # If one of the tests is invalid, delete it, log it, and drive on. 598 for test in self.tests: 599 if not verify_file(os.path.join(self.pathname, test)): 600 del self.tests[self.tests.index(test)] 601 write_log("Warning: Test '%s' removed from TestGroup '%s' " 602 "because it failed verification.\n" % 603 (test, self.pathname), LOG_ERR) 604 605 return len(self.tests) != 0 606 607 def run(self, options, dryrun=None, kmemleak=None): 608 """ 609 Create Cmd instances for the pre/post/failsafe scripts. If the pre 610 script doesn't pass, skip all the tests in this TestGroup. Run the 611 post script regardless. Run the failsafe script when a test is killed. 612 """ 613 # tags assigned to this test group also include the test names 614 if options.tags and not set(self.tags).intersection(set(options.tags)): 615 return 616 617 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 618 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 619 user=self.pre_user, identifier=self.identifier) 620 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 621 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 622 user=self.post_user, identifier=self.identifier) 623 624 cont = True 625 if len(pretest.pathname): 626 pretest.run(options, dryrun=dryrun, kmemleak=False) 627 cont = pretest.result.result == 'PASS' 628 pretest.log(options) 629 630 for fname in self.tests: 631 odir = os.path.join(self.outputdir, fname) 632 test = Cmd(os.path.join(self.pathname, fname), outputdir=odir, 633 timeout=self.timeout, user=self.user, 634 identifier=self.identifier) 635 odir = os.path.join(odir, os.path.basename(self.failsafe)) 636 failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout, 637 user=self.failsafe_user, identifier=self.identifier) 638 if cont: 639 test.run(options, dryrun=dryrun, kmemleak=kmemleak) 640 if test.result.result == 'KILLED' and len(failsafe.pathname): 641 failsafe.run(options, dryrun=dryrun, kmemleak=False) 642 failsafe.log(options, suppress_console=True) 643 else: 644 test.skip() 645 646 test.log(options) 647 648 if len(posttest.pathname): 649 posttest.run(options, dryrun=dryrun, kmemleak=False) 650 posttest.log(options) 651 652 653class TestRun(object): 654 props = ['quiet', 'outputdir', 'debug'] 655 656 def __init__(self, options): 657 self.tests = {} 658 self.testgroups = {} 659 self.starttime = time() 660 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 661 self.outputdir = os.path.join(options.outputdir, self.timestamp) 662 self.setup_logging(options) 663 self.defaults = [ 664 ('outputdir', BASEDIR), 665 ('quiet', False), 666 ('timeout', 60), 667 ('user', ''), 668 ('pre', ''), 669 ('pre_user', ''), 670 ('post', ''), 671 ('post_user', ''), 672 ('failsafe', ''), 673 ('failsafe_user', ''), 674 ('tags', []), 675 ('debug', False) 676 ] 677 678 def __str__(self): 679 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 680 s += 'TESTS:\n' 681 for key in sorted(self.tests.keys()): 682 s += '%s%s' % (self.tests[key].__str__(), '\n') 683 s += 'TESTGROUPS:\n' 684 for key in sorted(self.testgroups.keys()): 685 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 686 return s 687 688 def addtest(self, pathname, options): 689 """ 690 Create a new Test, and apply any properties that were passed in 691 from the command line. If it passes verification, add it to the 692 TestRun. 693 """ 694 test = Test(pathname) 695 for prop in Test.props: 696 setattr(test, prop, getattr(options, prop)) 697 698 if test.verify(): 699 self.tests[pathname] = test 700 701 def addtestgroup(self, dirname, filenames, options): 702 """ 703 Create a new TestGroup, and apply any properties that were passed 704 in from the command line. If it passes verification, add it to the 705 TestRun. 706 """ 707 if dirname not in self.testgroups: 708 testgroup = TestGroup(dirname) 709 for prop in Test.props: 710 setattr(testgroup, prop, getattr(options, prop)) 711 712 # Prevent pre/post/failsafe scripts from running as regular tests 713 for f in [testgroup.pre, testgroup.post, testgroup.failsafe]: 714 if f in filenames: 715 del filenames[filenames.index(f)] 716 717 self.testgroups[dirname] = testgroup 718 self.testgroups[dirname].tests = sorted(filenames) 719 720 testgroup.verify() 721 722 def filter(self, keeplist): 723 for group in list(self.testgroups.keys()): 724 if group not in keeplist: 725 del self.testgroups[group] 726 continue 727 728 g = self.testgroups[group] 729 730 if g.pre and os.path.basename(g.pre) in keeplist[group]: 731 continue 732 733 g.filter(keeplist[group]) 734 735 for test in list(self.tests.keys()): 736 directory, base = os.path.split(test) 737 if directory not in keeplist or base not in keeplist[directory]: 738 del self.tests[test] 739 740 def read(self, options): 741 """ 742 Read in the specified runfiles, and apply the TestRun properties 743 listed in the 'DEFAULT' section to our TestRun. Then read each 744 section, and apply the appropriate properties to the Test or 745 TestGroup. Properties from individual sections override those set 746 in the 'DEFAULT' section. If the Test or TestGroup passes 747 verification, add it to the TestRun. 748 """ 749 config = configparser.RawConfigParser() 750 parsed = config.read(options.runfiles) 751 failed = options.runfiles - set(parsed) 752 if len(failed): 753 files = ' '.join(sorted(failed)) 754 fail("Couldn't read config files: %s" % files) 755 756 for opt in TestRun.props: 757 if config.has_option('DEFAULT', opt): 758 if opt == 'outputdir': 759 outputdir = config.get('DEFAULT', opt) 760 setattr(self, opt, os.path.join(outputdir, self.timestamp)) 761 else: 762 setattr(self, opt, config.get('DEFAULT', opt)) 763 764 testdir = options.testdir 765 766 for section in config.sections(): 767 if 'tests' in config.options(section): 768 parts = section.split(':', 1) 769 sectiondir = parts[0] 770 identifier = parts[1] if len(parts) == 2 else None 771 if os.path.isdir(sectiondir): 772 pathname = sectiondir 773 elif os.path.isdir(os.path.join(testdir, sectiondir)): 774 pathname = os.path.join(testdir, sectiondir) 775 else: 776 pathname = sectiondir 777 778 testgroup = TestGroup(os.path.abspath(pathname), 779 identifier=identifier) 780 for prop in TestGroup.props: 781 for sect in ['DEFAULT', section]: 782 if config.has_option(sect, prop): 783 if prop == 'tags': 784 setattr(testgroup, prop, 785 eval(config.get(sect, prop))) 786 elif prop == 'failsafe': 787 failsafe = config.get(sect, prop) 788 setattr(testgroup, prop, 789 os.path.join(testdir, failsafe)) 790 elif prop == 'outputdir': 791 outputdir = config.get(sect, prop) 792 setattr(self, opt, 793 os.path.join(outputdir, 794 self.timestamp)) 795 else: 796 setattr(testgroup, prop, 797 config.get(sect, prop)) 798 799 # Repopulate tests using eval to convert the string to a list 800 testgroup.tests = eval(config.get(section, 'tests')) 801 802 if testgroup.verify(): 803 self.testgroups[section] = testgroup 804 else: 805 test = Test(section) 806 for prop in Test.props: 807 for sect in ['DEFAULT', section]: 808 if config.has_option(sect, prop): 809 if prop == 'failsafe': 810 failsafe = config.get(sect, prop) 811 setattr(test, prop, 812 os.path.join(testdir, failsafe)) 813 elif prop == 'outputdir': 814 outputdir = config.get(sect, prop) 815 setattr(self, opt, 816 os.path.join(outputdir, 817 self.timestamp)) 818 else: 819 setattr(test, prop, config.get(sect, prop)) 820 821 if test.verify(): 822 self.tests[section] = test 823 824 def write(self, options): 825 """ 826 Create a configuration file for editing and later use. The 827 'DEFAULT' section of the config file is created from the 828 properties that were specified on the command line. Tests are 829 simply added as sections that inherit everything from the 830 'DEFAULT' section. TestGroups are the same, except they get an 831 option including all the tests to run in that directory. 832 """ 833 834 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 835 self.defaults]) 836 config = configparser.RawConfigParser(defaults) 837 838 for test in sorted(self.tests.keys()): 839 config.add_section(test) 840 for prop in Test.props: 841 if prop not in self.props: 842 config.set(test, prop, 843 getattr(self.tests[test], prop)) 844 845 for testgroup in sorted(self.testgroups.keys()): 846 config.add_section(testgroup) 847 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 848 for prop in TestGroup.props: 849 if prop not in self.props: 850 config.set(testgroup, prop, 851 getattr(self.testgroups[testgroup], prop)) 852 853 try: 854 with open(options.template, 'w') as f: 855 return config.write(f) 856 except IOError: 857 fail('Could not open \'%s\' for writing.' % options.template) 858 859 def complete_outputdirs(self): 860 """ 861 Collect all the pathnames for Tests, and TestGroups. Work 862 backwards one pathname component at a time, to create a unique 863 directory name in which to deposit test output. Tests will be able 864 to write output files directly in the newly modified outputdir. 865 TestGroups will be able to create one subdirectory per test in the 866 outputdir, and are guaranteed uniqueness because a group can only 867 contain files in one directory. Pre and post tests will create a 868 directory rooted at the outputdir of the Test or TestGroup in 869 question for their output. Failsafe scripts will create a directory 870 rooted at the outputdir of each Test for their output. 871 """ 872 done = False 873 components = 0 874 tmp_dict = dict(list(self.tests.items()) + 875 list(self.testgroups.items())) 876 total = len(tmp_dict) 877 base = self.outputdir 878 879 while not done: 880 paths = [] 881 components -= 1 882 for testfile in list(tmp_dict.keys()): 883 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') 884 if uniq not in paths: 885 paths.append(uniq) 886 tmp_dict[testfile].outputdir = os.path.join(base, uniq) 887 else: 888 break 889 done = total == len(paths) 890 891 def setup_logging(self, options): 892 """ 893 This function creates the output directory and gets a file object 894 for the logfile. This function must be called before write_log() 895 can be used. 896 """ 897 if options.dryrun is True: 898 return 899 900 global LOG_FILE_OBJ 901 if not options.template: 902 try: 903 old = os.umask(0) 904 os.makedirs(self.outputdir, mode=0o777) 905 os.umask(old) 906 filename = os.path.join(self.outputdir, 'log') 907 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') 908 except OSError as e: 909 fail('%s' % e) 910 911 def run(self, options): 912 """ 913 Walk through all the Tests and TestGroups, calling run(). 914 """ 915 try: 916 os.chdir(self.outputdir) 917 except OSError: 918 fail('Could not change to directory %s' % self.outputdir) 919 # make a symlink to the output for the currently running test 920 logsymlink = os.path.join(self.outputdir, '../current') 921 if os.path.islink(logsymlink): 922 os.unlink(logsymlink) 923 if not os.path.exists(logsymlink): 924 os.symlink(self.outputdir, logsymlink) 925 else: 926 write_log('Could not make a symlink to directory %s\n' % 927 self.outputdir, LOG_ERR) 928 929 if options.kmemleak: 930 cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"' 931 check_output(cmd, shell=True) 932 933 iteration = 0 934 while iteration < options.iterations: 935 for test in sorted(self.tests.keys()): 936 self.tests[test].run(options) 937 for testgroup in sorted(self.testgroups.keys()): 938 self.testgroups[testgroup].run(options) 939 iteration += 1 940 941 def summary(self): 942 if Result.total == 0: 943 return 2 944 945 print('\nResults Summary') 946 for key in list(Result.runresults.keys()): 947 if Result.runresults[key] != 0: 948 print('%s\t% 4d' % (key, Result.runresults[key])) 949 950 m, s = divmod(time() - self.starttime, 60) 951 h, m = divmod(m, 60) 952 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 953 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 954 float(Result.total)) * 100)) 955 print('Log directory:\t%s' % self.outputdir) 956 957 if Result.runresults['FAIL'] > 0: 958 return 1 959 960 if Result.runresults['KILLED'] > 0: 961 return 1 962 963 if Result.runresults['RERAN'] > 0: 964 return 3 965 966 return 0 967 968 969def write_log(msg, target): 970 """ 971 Write the provided message to standard out, standard error or 972 the logfile. If specifying LOG_FILE, then `msg` must be a bytes 973 like object. This way we can still handle output from tests that 974 may be in unexpected encodings. 975 """ 976 if target == LOG_OUT: 977 os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8')) 978 elif target == LOG_ERR: 979 os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8')) 980 elif target == LOG_FILE: 981 os.write(LOG_FILE_OBJ.fileno(), msg) 982 else: 983 fail('log_msg called with unknown target "%s"' % target) 984 985 986def verify_file(pathname): 987 """ 988 Verify that the supplied pathname is an executable regular file. 989 """ 990 if os.path.isdir(pathname) or os.path.islink(pathname): 991 return False 992 993 for ext in '', '.ksh', '.sh': 994 script_path = pathname + ext 995 if os.path.isfile(script_path) and os.access(script_path, os.X_OK): 996 return True 997 998 return False 999 1000 1001def verify_user(user): 1002 """ 1003 Verify that the specified user exists on this system, and can execute 1004 sudo without being prompted for a password. 1005 """ 1006 testcmd = [SUDO, '-n', '-u', user, TRUE] 1007 1008 if user in Cmd.verified_users: 1009 return True 1010 1011 try: 1012 getpwnam(user) 1013 except KeyError: 1014 write_log("Warning: user '%s' does not exist.\n" % user, 1015 LOG_ERR) 1016 return False 1017 1018 p = Popen(testcmd) 1019 p.wait() 1020 if p.returncode != 0: 1021 write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user, 1022 LOG_ERR) 1023 return False 1024 else: 1025 Cmd.verified_users.append(user) 1026 1027 return True 1028 1029 1030def find_tests(testrun, options): 1031 """ 1032 For the given list of pathnames, add files as Tests. For directories, 1033 if do_groups is True, add the directory as a TestGroup. If False, 1034 recursively search for executable files. 1035 """ 1036 1037 for p in sorted(options.pathnames): 1038 if os.path.isdir(p): 1039 for dirname, _, filenames in os.walk(p): 1040 if options.do_groups: 1041 testrun.addtestgroup(dirname, filenames, options) 1042 else: 1043 for f in sorted(filenames): 1044 testrun.addtest(os.path.join(dirname, f), options) 1045 else: 1046 testrun.addtest(p, options) 1047 1048 1049def filter_tests(testrun, options): 1050 try: 1051 fh = open(options.logfile, "r") 1052 except Exception as e: 1053 fail('%s' % e) 1054 1055 failed = {} 1056 while True: 1057 line = fh.readline() 1058 if not line: 1059 break 1060 m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line) 1061 if not m: 1062 continue 1063 group, test = m.group(1, 2) 1064 try: 1065 failed[group].append(test) 1066 except KeyError: 1067 failed[group] = [test] 1068 fh.close() 1069 1070 testrun.filter(failed) 1071 1072 1073def fail(retstr, ret=1): 1074 print('%s: %s' % (sys.argv[0], retstr)) 1075 exit(ret) 1076 1077 1078def kmemleak_cb(option, opt_str, value, parser): 1079 if not os.path.exists(KMEMLEAK_FILE): 1080 fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " + 1081 "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.") 1082 1083 setattr(parser.values, option.dest, True) 1084 1085 1086def options_cb(option, opt_str, value, parser): 1087 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 1088 1089 if opt_str in parser.rargs: 1090 fail('%s may only be specified once.' % opt_str) 1091 1092 if option.dest == 'runfiles': 1093 parser.values.cmd = 'rdconfig' 1094 value = set(os.path.abspath(p) for p in value.split(',')) 1095 if option.dest == 'tags': 1096 value = [x.strip() for x in value.split(',')] 1097 1098 if option.dest in path_options: 1099 setattr(parser.values, option.dest, os.path.abspath(value)) 1100 else: 1101 setattr(parser.values, option.dest, value) 1102 1103 1104def parse_args(): 1105 parser = OptionParser() 1106 parser.add_option('-c', action='callback', callback=options_cb, 1107 type='string', dest='runfiles', metavar='runfiles', 1108 help='Specify tests to run via config files.') 1109 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 1110 help='Dry run. Print tests, but take no other action.') 1111 parser.add_option('-D', action='store_true', default=False, dest='debug', 1112 help='Write all test output to stdout as it arrives.') 1113 parser.add_option('-l', action='callback', callback=options_cb, 1114 default=None, dest='logfile', metavar='logfile', 1115 type='string', 1116 help='Read logfile and re-run tests which failed.') 1117 parser.add_option('-g', action='store_true', default=False, 1118 dest='do_groups', help='Make directories TestGroups.') 1119 parser.add_option('-o', action='callback', callback=options_cb, 1120 default=BASEDIR, dest='outputdir', type='string', 1121 metavar='outputdir', help='Specify an output directory.') 1122 parser.add_option('-i', action='callback', callback=options_cb, 1123 default=TESTDIR, dest='testdir', type='string', 1124 metavar='testdir', help='Specify a test directory.') 1125 parser.add_option('-K', action='store_true', default=False, dest='kmsg', 1126 help='Log tests names to /dev/kmsg') 1127 parser.add_option('-m', action='callback', callback=kmemleak_cb, 1128 default=False, dest='kmemleak', 1129 help='Enable kmemleak reporting (Linux only)') 1130 parser.add_option('-p', action='callback', callback=options_cb, 1131 default='', dest='pre', metavar='script', 1132 type='string', help='Specify a pre script.') 1133 parser.add_option('-P', action='callback', callback=options_cb, 1134 default='', dest='post', metavar='script', 1135 type='string', help='Specify a post script.') 1136 parser.add_option('-q', action='store_true', default=False, dest='quiet', 1137 help='Silence on the console during a test run.') 1138 parser.add_option('-s', action='callback', callback=options_cb, 1139 default='', dest='failsafe', metavar='script', 1140 type='string', help='Specify a failsafe script.') 1141 parser.add_option('-S', action='callback', callback=options_cb, 1142 default='', dest='failsafe_user', 1143 metavar='failsafe_user', type='string', 1144 help='Specify a user to execute the failsafe script.') 1145 parser.add_option('-t', action='callback', callback=options_cb, default=60, 1146 dest='timeout', metavar='seconds', type='int', 1147 help='Timeout (in seconds) for an individual test.') 1148 parser.add_option('-u', action='callback', callback=options_cb, 1149 default='', dest='user', metavar='user', type='string', 1150 help='Specify a different user name to run as.') 1151 parser.add_option('-w', action='callback', callback=options_cb, 1152 default=None, dest='template', metavar='template', 1153 type='string', help='Create a new config file.') 1154 parser.add_option('-x', action='callback', callback=options_cb, default='', 1155 dest='pre_user', metavar='pre_user', type='string', 1156 help='Specify a user to execute the pre script.') 1157 parser.add_option('-X', action='callback', callback=options_cb, default='', 1158 dest='post_user', metavar='post_user', type='string', 1159 help='Specify a user to execute the post script.') 1160 parser.add_option('-T', action='callback', callback=options_cb, default='', 1161 dest='tags', metavar='tags', type='string', 1162 help='Specify tags to execute specific test groups.') 1163 parser.add_option('-I', action='callback', callback=options_cb, default=1, 1164 dest='iterations', metavar='iterations', type='int', 1165 help='Number of times to run the test run.') 1166 (options, pathnames) = parser.parse_args() 1167 1168 if options.runfiles and len(pathnames): 1169 fail('Extraneous arguments.') 1170 1171 options.pathnames = [os.path.abspath(path) for path in pathnames] 1172 1173 return options 1174 1175 1176def main(): 1177 options = parse_args() 1178 1179 testrun = TestRun(options) 1180 1181 if options.runfiles: 1182 testrun.read(options) 1183 else: 1184 find_tests(testrun, options) 1185 1186 if options.logfile: 1187 filter_tests(testrun, options) 1188 1189 if options.template: 1190 testrun.write(options) 1191 exit(0) 1192 1193 testrun.complete_outputdirs() 1194 testrun.run(options) 1195 exit(testrun.summary()) 1196 1197 1198if __name__ == '__main__': 1199 main() 1200