1#!/usr/bin/env @PYTHON_SHEBANG@ 2# SPDX-License-Identifier: CDDL-1.0 3 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# http://www.illumos.org/license/CDDL. 13# 14 15# 16# Copyright (c) 2012, 2018 by Delphix. All rights reserved. 17# Copyright (c) 2019 Datto Inc. 18# 19# This script must remain compatible with Python 3.6+. 20# 21 22import os 23import sys 24import ctypes 25import re 26import configparser 27 28from datetime import datetime 29from optparse import OptionParser 30from pwd import getpwnam 31from pwd import getpwuid 32from select import select 33from subprocess import PIPE 34from subprocess import Popen 35from subprocess import check_output 36from threading import Timer 37from time import time, CLOCK_MONOTONIC 38from os.path import exists 39 40BASEDIR = '/var/tmp/test_results' 41TESTDIR = '/usr/share/zfs/' 42KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak' 43KILL = 'kill' 44TRUE = 'true' 45SUDO = 'sudo' 46LOG_FILE = 'LOG_FILE' 47LOG_OUT = 'LOG_OUT' 48LOG_ERR = 'LOG_ERR' 49LOG_FILE_OBJ = None 50 51try: 52 from time import monotonic as monotonic_time 53except ImportError: 54 class timespec(ctypes.Structure): 55 _fields_ = [ 56 ('tv_sec', ctypes.c_long), 57 ('tv_nsec', ctypes.c_long) 58 ] 59 60 librt = ctypes.CDLL('librt.so.1', use_errno=True) 61 clock_gettime = librt.clock_gettime 62 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] 63 64 def monotonic_time(): 65 t = timespec() 66 if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0: 67 errno_ = ctypes.get_errno() 68 raise OSError(errno_, os.strerror(errno_)) 69 return t.tv_sec + t.tv_nsec * 1e-9 70 71 72class Result(object): 73 total = 0 74 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0} 75 76 def __init__(self): 77 self.starttime = None 78 self.returncode = None 79 self.runtime = '' 80 self.stdout = [] 81 self.stderr = [] 82 self.kmemleak = '' 83 self.result = '' 84 85 def done(self, proc, killed, reran): 86 """ 87 Finalize the results of this Cmd. 88 """ 89 Result.total += 1 90 m, s = divmod(monotonic_time() - self.starttime, 60) 91 self.runtime = '%02d:%02d' % (m, s) 92 self.returncode = proc.returncode 93 if reran is True: 94 Result.runresults['RERAN'] += 1 95 if killed: 96 self.result = 'KILLED' 97 Result.runresults['KILLED'] += 1 98 elif len(self.kmemleak) > 0: 99 self.result = 'FAIL' 100 Result.runresults['FAIL'] += 1 101 elif self.returncode == 0: 102 self.result = 'PASS' 103 Result.runresults['PASS'] += 1 104 elif self.returncode == 4: 105 self.result = 'SKIP' 106 Result.runresults['SKIP'] += 1 107 elif self.returncode != 0: 108 self.result = 'FAIL' 109 Result.runresults['FAIL'] += 1 110 111 112class Output(object): 113 """ 114 This class is a slightly modified version of the 'Stream' class found 115 here: https://stackoverflow.com/q/4984549/ 116 """ 117 def __init__(self, stream, debug=False): 118 self.stream = stream 119 self.debug = debug 120 self._buf = b'' 121 self.lines = [] 122 123 def fileno(self): 124 return self.stream.fileno() 125 126 def read(self, drain=0): 127 """ 128 Read from the file descriptor. If 'drain' set, read until EOF. 129 """ 130 while self._read() is not None: 131 if not drain: 132 break 133 134 def _read(self): 135 """ 136 Read up to 4k of data from this output stream. Collect the output 137 up to the last newline, and append it to any leftover data from a 138 previous call. The lines are stored as a (timestamp, data) tuple 139 for easy sorting/merging later. 140 """ 141 fd = self.fileno() 142 buf = os.read(fd, 4096) 143 if not buf: 144 return None 145 if self.debug: 146 os.write(sys.stderr.fileno(), buf) 147 if b'\n' not in buf: 148 self._buf += buf 149 return [] 150 151 buf = self._buf + buf 152 tmp, rest = buf.rsplit(b'\n', 1) 153 self._buf = rest 154 now = datetime.now() 155 rows = tmp.split(b'\n') 156 self.lines += [(now, r) for r in rows] 157 158 159class Cmd(object): 160 verified_users = [] 161 162 def __init__(self, pathname, identifier=None, outputdir=None, 163 timeout=None, user=None, tags=None): 164 self.pathname = pathname 165 self.identifier = identifier 166 self.outputdir = outputdir or 'BASEDIR' 167 """ 168 The timeout for tests is measured in wall-clock time 169 """ 170 self.timeout = timeout 171 self.user = user or '' 172 self.killed = False 173 self.reran = None 174 self.result = Result() 175 176 if self.timeout is None: 177 self.timeout = 60 178 179 def __str__(self): 180 return '''\ 181Pathname: %s 182Identifier: %s 183Outputdir: %s 184Timeout: %d 185User: %s 186''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user) 187 188 def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False): 189 """ 190 Kill a running command due to timeout, or ^C from the keyboard. If 191 sudo is required, this user was verified previously. 192 """ 193 self.killed = True 194 do_sudo = len(self.user) != 0 195 signal = '-TERM' 196 197 cmd = [SUDO, KILL, signal, str(proc.pid)] 198 if not do_sudo: 199 del cmd[0] 200 201 try: 202 kp = Popen(cmd) 203 kp.wait() 204 except Exception: 205 pass 206 207 """ 208 If this is not a user-initiated kill and the test has not been 209 reran before we consider if the test needs to be reran: 210 If the test has spent some time hibernating and didn't run the whole 211 length of time before being timed out we will rerun the test. 212 """ 213 if keyboard_interrupt is False and self.reran is None: 214 runtime = monotonic_time() - self.result.starttime 215 if int(self.timeout) > runtime: 216 self.killed = False 217 self.reran = False 218 self.run(options, dryrun=False, kmemleak=kmemleak) 219 self.reran = True 220 221 def update_cmd_privs(self, cmd, user): 222 """ 223 If a user has been specified to run this Cmd and we're not already 224 running as that user, prepend the appropriate sudo command to run 225 as that user. 226 """ 227 me = getpwuid(os.getuid()) 228 229 if not user or user is me: 230 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 231 cmd += '.ksh' 232 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 233 cmd += '.sh' 234 return cmd 235 236 if not os.path.isfile(cmd): 237 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): 238 cmd += '.ksh' 239 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): 240 cmd += '.sh' 241 242 # glibc (at least) will not pass TMPDIR through to setuid programs. 243 # if set, arrange for it to be reset before running the target cmd 244 tmpdir = os.getenv('TMPDIR') 245 if tmpdir: 246 tmpdirarg = 'env TMPDIR=%s' % tmpdir 247 else: 248 tmpdirarg = '' 249 250 ret = '%s -E -u %s %s %s' % (SUDO, user, tmpdirarg, cmd) 251 return ret.split(' ') 252 253 def collect_output(self, proc, debug=False): 254 """ 255 Read from stdout/stderr as data becomes available, until the 256 process is no longer running. Return the lines from the stdout and 257 stderr Output objects. 258 """ 259 out = Output(proc.stdout, debug) 260 err = Output(proc.stderr, debug) 261 res = [] 262 while proc.returncode is None: 263 proc.poll() 264 res = select([out, err], [], [], .1) 265 for fd in res[0]: 266 fd.read() 267 for fd in res[0]: 268 fd.read(drain=1) 269 270 return out.lines, err.lines 271 272 def run(self, options, dryrun=None, kmemleak=None): 273 """ 274 This is the main function that runs each individual test. 275 Determine whether or not the command requires sudo, and modify it 276 if needed. Run the command, and update the result object. 277 """ 278 if dryrun is None: 279 dryrun = options.dryrun 280 if dryrun is True: 281 print(self) 282 return 283 if kmemleak is None: 284 kmemleak = options.kmemleak 285 286 privcmd = self.update_cmd_privs(self.pathname, self.user) 287 try: 288 old = os.umask(0) 289 if not os.path.isdir(self.outputdir): 290 os.makedirs(self.outputdir, mode=0o777) 291 os.umask(old) 292 except OSError as e: 293 fail('%s' % e) 294 295 """ 296 Log each test we run to /dev/kmsg (on Linux), so if there's a kernel 297 warning we'll be able to match it up to a particular test. 298 """ 299 if options.kmsg is True and exists("/dev/kmsg"): 300 try: 301 kp = Popen([SUDO, "sh", "-c", 302 f"echo ZTS run {self.pathname} > /dev/kmsg"]) 303 kp.wait() 304 except Exception: 305 pass 306 307 """ 308 Log each test we run to /dev/ttyu0 (on FreeBSD), so if there's a kernel 309 warning we'll be able to match it up to a particular test. 310 """ 311 if options.kmsg is True and exists("/dev/ttyu0"): 312 try: 313 kp = Popen([SUDO, "sh", "-c", 314 f"echo ZTS run {self.pathname} > /dev/ttyu0"]) 315 kp.wait() 316 except Exception: 317 pass 318 319 self.result.starttime = monotonic_time() 320 321 if kmemleak: 322 cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"' 323 check_output(cmd, shell=True) 324 325 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) 326 # Allow a special timeout value of 0 to mean infinity 327 if int(self.timeout) == 0: 328 self.timeout = sys.maxsize / (10 ** 9) 329 t = Timer( 330 int(self.timeout), self.kill_cmd, [proc, options, kmemleak] 331 ) 332 333 try: 334 t.start() 335 336 out, err = self.collect_output(proc, options.debug) 337 self.result.stdout = out 338 self.result.stderr = err 339 340 if kmemleak: 341 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"' 342 check_output(cmd, shell=True) 343 cmd = f'{SUDO} cat {KMEMLEAK_FILE}' 344 self.result.kmemleak = check_output(cmd, shell=True) 345 except KeyboardInterrupt: 346 self.kill_cmd(proc, options, kmemleak, True) 347 fail('\nRun terminated at user request.') 348 finally: 349 t.cancel() 350 351 if self.reran is not False: 352 self.result.done(proc, self.killed, self.reran) 353 354 def skip(self): 355 """ 356 Initialize enough of the test result that we can log a skipped 357 command. 358 """ 359 Result.total += 1 360 Result.runresults['SKIP'] += 1 361 self.result.stdout = self.result.stderr = [] 362 self.result.starttime = monotonic_time() 363 m, s = divmod(monotonic_time() - self.result.starttime, 60) 364 self.result.runtime = '%02d:%02d' % (m, s) 365 self.result.result = 'SKIP' 366 367 def log(self, options, suppress_console=False): 368 """ 369 This function is responsible for writing all output. This includes 370 the console output, the logfile of all results (with timestamped 371 merged stdout and stderr), and for each test, the unmodified 372 stdout/stderr/merged in its own file. 373 """ 374 375 logname = getpwuid(os.getuid()).pw_name 376 rer = '' 377 if self.reran is True: 378 rer = ' (RERAN)' 379 user = ' (run as %s)' % (self.user if len(self.user) else logname) 380 if self.identifier: 381 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user) 382 else: 383 msga = 'Test: %s%s ' % (self.pathname, user) 384 msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer) 385 pad = ' ' * (80 - (len(msga) + len(msgb))) 386 result_line = msga + pad + msgb 387 388 # The result line is always written to the log file. If -q was 389 # specified only failures are written to the console, otherwise 390 # the result line is written to the console. The console output 391 # may be suppressed by calling log() with suppress_console=True. 392 write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE) 393 if not suppress_console: 394 if not options.quiet: 395 write_log(result_line, LOG_OUT) 396 elif options.quiet and self.result.result != 'PASS': 397 write_log(result_line, LOG_OUT) 398 399 lines = sorted(self.result.stdout + self.result.stderr, 400 key=lambda x: x[0]) 401 402 # Write timestamped output (stdout and stderr) to the logfile 403 for dt, line in lines: 404 timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11], 405 encoding='utf-8') 406 write_log(b'%s %s\n' % (timestamp, line), LOG_FILE) 407 408 # Write the separate stdout/stderr/merged files, if the data exists 409 if len(self.result.stdout): 410 with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out: 411 for _, line in self.result.stdout: 412 os.write(out.fileno(), b'%s\n' % line) 413 if len(self.result.stderr): 414 with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err: 415 for _, line in self.result.stderr: 416 os.write(err.fileno(), b'%s\n' % line) 417 if len(self.result.stdout) and len(self.result.stderr): 418 with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged: 419 for _, line in lines: 420 os.write(merged.fileno(), b'%s\n' % line) 421 if len(self.result.kmemleak): 422 with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem: 423 kmem.write(self.result.kmemleak) 424 425 426class Test(Cmd): 427 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', 428 'post_user', 'failsafe', 'failsafe_user', 'tags'] 429 430 def __init__(self, pathname, 431 pre=None, pre_user=None, post=None, post_user=None, 432 failsafe=None, failsafe_user=None, tags=None, **kwargs): 433 super(Test, self).__init__(pathname, **kwargs) 434 self.pre = pre or '' 435 self.pre_user = pre_user or '' 436 self.post = post or '' 437 self.post_user = post_user or '' 438 self.failsafe = failsafe or '' 439 self.failsafe_user = failsafe_user or '' 440 self.tags = tags or [] 441 442 def __str__(self): 443 post_user = pre_user = failsafe_user = '' 444 if len(self.pre_user): 445 pre_user = ' (as %s)' % (self.pre_user) 446 if len(self.post_user): 447 post_user = ' (as %s)' % (self.post_user) 448 if len(self.failsafe_user): 449 failsafe_user = ' (as %s)' % (self.failsafe_user) 450 return '''\ 451Pathname: %s 452Identifier: %s 453Outputdir: %s 454Timeout: %d 455User: %s 456Pre: %s%s 457Post: %s%s 458Failsafe: %s%s 459Tags: %s 460''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user, 461 self.pre, pre_user, self.post, post_user, self.failsafe, 462 failsafe_user, self.tags) 463 464 def verify(self): 465 """ 466 Check the pre/post/failsafe scripts, user and Test. Omit the Test from 467 this run if there are any problems. 468 """ 469 files = [self.pre, self.pathname, self.post, self.failsafe] 470 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 471 472 for f in [f for f in files if len(f)]: 473 if not verify_file(f): 474 write_log("Warning: Test '%s' not added to this run because" 475 " it failed verification.\n" % f, LOG_ERR) 476 return False 477 478 for user in [user for user in users if len(user)]: 479 if not verify_user(user): 480 write_log("Not adding Test '%s' to this run.\n" % 481 self.pathname, LOG_ERR) 482 return False 483 484 return True 485 486 def run(self, options, dryrun=None, kmemleak=None): 487 """ 488 Create Cmd instances for the pre/post/failsafe scripts. If the pre 489 script doesn't pass, skip this Test. Run the post script regardless. 490 If the Test is killed, also run the failsafe script. 491 """ 492 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 493 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir, 494 timeout=self.timeout, user=self.pre_user) 495 test = Cmd(self.pathname, identifier=self.identifier, 496 outputdir=self.outputdir, timeout=self.timeout, 497 user=self.user) 498 odir = os.path.join(self.outputdir, os.path.basename(self.failsafe)) 499 failsafe = Cmd(self.failsafe, identifier=self.identifier, 500 outputdir=odir, timeout=self.timeout, 501 user=self.failsafe_user) 502 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 503 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir, 504 timeout=self.timeout, user=self.post_user) 505 506 cont = True 507 if len(pretest.pathname): 508 pretest.run(options, kmemleak=False) 509 cont = pretest.result.result == 'PASS' 510 pretest.log(options) 511 512 if cont: 513 test.run(options, kmemleak=kmemleak) 514 if test.result.result == 'KILLED' and len(failsafe.pathname): 515 failsafe.run(options, kmemleak=False) 516 failsafe.log(options, suppress_console=True) 517 else: 518 test.skip() 519 520 test.log(options) 521 522 if len(posttest.pathname): 523 posttest.run(options, kmemleak=False) 524 posttest.log(options) 525 526 527class TestGroup(Test): 528 props = Test.props + ['tests'] 529 530 def __init__(self, pathname, tests=None, **kwargs): 531 super(TestGroup, self).__init__(pathname, **kwargs) 532 self.tests = tests or [] 533 534 def __str__(self): 535 post_user = pre_user = failsafe_user = '' 536 if len(self.pre_user): 537 pre_user = ' (as %s)' % (self.pre_user) 538 if len(self.post_user): 539 post_user = ' (as %s)' % (self.post_user) 540 if len(self.failsafe_user): 541 failsafe_user = ' (as %s)' % (self.failsafe_user) 542 return '''\ 543Pathname: %s 544Identifier: %s 545Outputdir: %s 546Tests: %s 547Timeout: %s 548User: %s 549Pre: %s%s 550Post: %s%s 551Failsafe: %s%s 552Tags: %s 553''' % (self.pathname, self.identifier, self.outputdir, self.tests, 554 self.timeout, self.user, self.pre, pre_user, self.post, post_user, 555 self.failsafe, failsafe_user, self.tags) 556 557 def filter(self, keeplist): 558 self.tests = [x for x in self.tests if x in keeplist] 559 560 def verify(self): 561 """ 562 Check the pre/post/failsafe scripts, user and tests in this TestGroup. 563 Omit the TestGroup entirely, or simply delete the relevant tests in the 564 group, if that's all that's required. 565 """ 566 # If the pre/post/failsafe scripts are relative pathnames, convert to 567 # absolute, so they stand a chance of passing verification. 568 if len(self.pre) and not os.path.isabs(self.pre): 569 self.pre = os.path.join(self.pathname, self.pre) 570 if len(self.post) and not os.path.isabs(self.post): 571 self.post = os.path.join(self.pathname, self.post) 572 if len(self.failsafe) and not os.path.isabs(self.failsafe): 573 self.post = os.path.join(self.pathname, self.post) 574 575 auxfiles = [self.pre, self.post, self.failsafe] 576 users = [self.pre_user, self.user, self.post_user, self.failsafe_user] 577 578 for f in [f for f in auxfiles if len(f)]: 579 if f != self.failsafe and self.pathname != os.path.dirname(f): 580 write_log("Warning: TestGroup '%s' not added to this run. " 581 "Auxiliary script '%s' exists in a different " 582 "directory.\n" % (self.pathname, f), LOG_ERR) 583 return False 584 585 if not verify_file(f): 586 write_log("Warning: TestGroup '%s' not added to this run. " 587 "Auxiliary script '%s' failed verification.\n" % 588 (self.pathname, f), LOG_ERR) 589 return False 590 591 for user in [user for user in users if len(user)]: 592 if not verify_user(user): 593 write_log("Not adding TestGroup '%s' to this run.\n" % 594 self.pathname, LOG_ERR) 595 return False 596 597 # If one of the tests is invalid, delete it, log it, and drive on. 598 for test in self.tests: 599 if not verify_file(os.path.join(self.pathname, test)): 600 del self.tests[self.tests.index(test)] 601 write_log("Warning: Test '%s' removed from TestGroup '%s' " 602 "because it failed verification.\n" % 603 (test, self.pathname), LOG_ERR) 604 605 return len(self.tests) != 0 606 607 def run(self, options, dryrun=None, kmemleak=None): 608 """ 609 Create Cmd instances for the pre/post/failsafe scripts. If the pre 610 script doesn't pass, skip all the tests in this TestGroup. Run the 611 post script regardless. Run the failsafe script when a test is killed. 612 """ 613 # tags assigned to this test group also include the test names 614 if options.tags and not set(self.tags).intersection(set(options.tags)): 615 return 616 617 odir = os.path.join(self.outputdir, os.path.basename(self.pre)) 618 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, 619 user=self.pre_user, identifier=self.identifier) 620 odir = os.path.join(self.outputdir, os.path.basename(self.post)) 621 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, 622 user=self.post_user, identifier=self.identifier) 623 624 cont = True 625 if len(pretest.pathname): 626 pretest.run(options, dryrun=dryrun, kmemleak=False) 627 cont = pretest.result.result == 'PASS' 628 pretest.log(options) 629 630 for fname in self.tests: 631 odir = os.path.join(self.outputdir, fname) 632 test = Cmd(os.path.join(self.pathname, fname), outputdir=odir, 633 timeout=self.timeout, user=self.user, 634 identifier=self.identifier) 635 odir = os.path.join(odir, os.path.basename(self.failsafe)) 636 failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout, 637 user=self.failsafe_user, identifier=self.identifier) 638 if cont: 639 test.run(options, dryrun=dryrun, kmemleak=kmemleak) 640 if test.result.result == 'KILLED' and len(failsafe.pathname): 641 failsafe.run(options, dryrun=dryrun, kmemleak=False) 642 failsafe.log(options, suppress_console=True) 643 else: 644 test.skip() 645 646 test.log(options) 647 648 if len(posttest.pathname): 649 posttest.run(options, dryrun=dryrun, kmemleak=False) 650 posttest.log(options) 651 652 653class TestRun(object): 654 props = ['quiet', 'outputdir', 'debug'] 655 656 def __init__(self, options): 657 self.tests = {} 658 self.testgroups = {} 659 self.starttime = time() 660 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') 661 self.outputdir = os.path.join(options.outputdir, self.timestamp) 662 self.setup_logging(options) 663 self.defaults = [ 664 ('outputdir', BASEDIR), 665 ('quiet', False), 666 ('timeout', 60), 667 ('user', ''), 668 ('pre', ''), 669 ('pre_user', ''), 670 ('post', ''), 671 ('post_user', ''), 672 ('failsafe', ''), 673 ('failsafe_user', ''), 674 ('tags', []), 675 ('debug', False) 676 ] 677 678 def __str__(self): 679 s = 'TestRun:\n outputdir: %s\n' % self.outputdir 680 s += 'TESTS:\n' 681 for key in sorted(self.tests.keys()): 682 s += '%s%s' % (self.tests[key].__str__(), '\n') 683 s += 'TESTGROUPS:\n' 684 for key in sorted(self.testgroups.keys()): 685 s += '%s%s' % (self.testgroups[key].__str__(), '\n') 686 return s 687 688 def addtest(self, pathname, options): 689 """ 690 Create a new Test, and apply any properties that were passed in 691 from the command line. If it passes verification, add it to the 692 TestRun. 693 """ 694 test = Test(pathname) 695 for prop in Test.props: 696 setattr(test, prop, getattr(options, prop)) 697 698 if test.verify(): 699 self.tests[pathname] = test 700 701 def addtestgroup(self, dirname, filenames, options): 702 """ 703 Create a new TestGroup, and apply any properties that were passed 704 in from the command line. If it passes verification, add it to the 705 TestRun. 706 """ 707 if dirname not in self.testgroups: 708 testgroup = TestGroup(dirname) 709 for prop in Test.props: 710 setattr(testgroup, prop, getattr(options, prop)) 711 712 # Prevent pre/post/failsafe scripts from running as regular tests 713 for f in [testgroup.pre, testgroup.post, testgroup.failsafe]: 714 if f in filenames: 715 del filenames[filenames.index(f)] 716 717 self.testgroups[dirname] = testgroup 718 self.testgroups[dirname].tests = sorted(filenames) 719 720 testgroup.verify() 721 722 def filter(self, keeplist): 723 for group in list(self.testgroups.keys()): 724 if group not in keeplist: 725 del self.testgroups[group] 726 continue 727 728 g = self.testgroups[group] 729 730 if g.pre and os.path.basename(g.pre) in keeplist[group]: 731 continue 732 733 g.filter(keeplist[group]) 734 735 for test in list(self.tests.keys()): 736 directory, base = os.path.split(test) 737 if directory not in keeplist or base not in keeplist[directory]: 738 del self.tests[test] 739 740 def read(self, options): 741 """ 742 Read in the specified runfiles, and apply the TestRun properties 743 listed in the 'DEFAULT' section to our TestRun. Then read each 744 section, and apply the appropriate properties to the Test or 745 TestGroup. Properties from individual sections override those set 746 in the 'DEFAULT' section. If the Test or TestGroup passes 747 verification, add it to the TestRun. 748 """ 749 config = configparser.RawConfigParser() 750 parsed = config.read(options.runfiles) 751 failed = options.runfiles - set(parsed) 752 if len(failed): 753 files = ' '.join(sorted(failed)) 754 fail("Couldn't read config files: %s" % files) 755 756 for opt in TestRun.props: 757 if config.has_option('DEFAULT', opt): 758 if opt == 'outputdir': 759 outputdir = config.get('DEFAULT', opt) 760 setattr(self, opt, os.path.join(outputdir, self.timestamp)) 761 else: 762 setattr(self, opt, config.get('DEFAULT', opt)) 763 764 testdir = options.testdir 765 766 for section in config.sections(): 767 if 'tests' in config.options(section): 768 parts = section.split(':', 1) 769 sectiondir = parts[0] 770 identifier = parts[1] if len(parts) == 2 else None 771 if os.path.isdir(sectiondir): 772 pathname = sectiondir 773 elif os.path.isdir(os.path.join(testdir, sectiondir)): 774 pathname = os.path.join(testdir, sectiondir) 775 else: 776 pathname = sectiondir 777 778 testgroup = TestGroup(os.path.abspath(pathname), 779 identifier=identifier) 780 for prop in TestGroup.props: 781 for sect in ['DEFAULT', section]: 782 if config.has_option(sect, prop): 783 if prop == 'tags': 784 setattr(testgroup, prop, 785 eval(config.get(sect, prop))) 786 elif prop == 'failsafe': 787 failsafe = config.get(sect, prop) 788 setattr(testgroup, prop, 789 os.path.join(testdir, failsafe)) 790 elif prop == 'outputdir': 791 outputdir = config.get(sect, prop) 792 setattr(self, opt, 793 os.path.join(outputdir, 794 self.timestamp)) 795 else: 796 setattr(testgroup, prop, 797 config.get(sect, prop)) 798 799 # Repopulate tests using eval to convert the string to a list 800 testgroup.tests = eval(config.get(section, 'tests')) 801 802 if testgroup.verify(): 803 self.testgroups[section] = testgroup 804 else: 805 test = Test(section) 806 for prop in Test.props: 807 for sect in ['DEFAULT', section]: 808 if config.has_option(sect, prop): 809 if prop == 'failsafe': 810 failsafe = config.get(sect, prop) 811 setattr(test, prop, 812 os.path.join(testdir, failsafe)) 813 elif prop == 'outputdir': 814 outputdir = config.get(sect, prop) 815 setattr(self, opt, 816 os.path.join(outputdir, 817 self.timestamp)) 818 else: 819 setattr(test, prop, config.get(sect, prop)) 820 821 if test.verify(): 822 self.tests[section] = test 823 824 def write(self, options): 825 """ 826 Create a configuration file for editing and later use. The 827 'DEFAULT' section of the config file is created from the 828 properties that were specified on the command line. Tests are 829 simply added as sections that inherit everything from the 830 'DEFAULT' section. TestGroups are the same, except they get an 831 option including all the tests to run in that directory. 832 """ 833 834 defaults = dict([(prop, getattr(options, prop)) for prop, _ in 835 self.defaults]) 836 config = configparser.RawConfigParser(defaults) 837 838 for test in sorted(self.tests.keys()): 839 config.add_section(test) 840 for prop in Test.props: 841 if prop not in self.props: 842 config.set(test, prop, 843 getattr(self.tests[test], prop)) 844 845 for testgroup in sorted(self.testgroups.keys()): 846 config.add_section(testgroup) 847 config.set(testgroup, 'tests', self.testgroups[testgroup].tests) 848 for prop in TestGroup.props: 849 if prop not in self.props: 850 config.set(testgroup, prop, 851 getattr(self.testgroups[testgroup], prop)) 852 853 try: 854 with open(options.template, 'w') as f: 855 return config.write(f) 856 except IOError: 857 fail('Could not open \'%s\' for writing.' % options.template) 858 859 def complete_outputdirs(self): 860 """ 861 Collect all the pathnames for Tests, and TestGroups. Strip off all 862 common leading path components, and append what remains to the top 863 "output" dir, to create a tree of output directories that match 864 the test and group names in structure. Tests will be able 865 to write output files directly in the newly modified outputdir. 866 TestGroups will be able to create one subdirectory per test in the 867 outputdir, and are guaranteed uniqueness because a group can only 868 contain files in one directory. Pre and post tests will create a 869 directory rooted at the outputdir of the Test or TestGroup in 870 question for their output. Failsafe scripts will create a directory 871 rooted at the outputdir of each Test for their output. 872 """ 873 874 alltests = dict(list(self.tests.items()) + 875 list(self.testgroups.items())) 876 base = os.path.join(self.outputdir, 'output') 877 878 seen = [] 879 880 for path in list(alltests.keys()): 881 frag = path.split('/') 882 for i in range(0, len(frag)): 883 if len(seen) == i: 884 seen.append({}) 885 seen[i][frag[i]] = 1 886 887 cut = 0 888 for i in range(0, len(seen)): 889 if len(list(seen[i].keys())) == 1: 890 cut += 1 891 else: 892 break 893 894 for path in list(alltests.keys()): 895 uniq = path.split('/', cut)[-1] 896 alltests[path].outputdir = os.path.join(base, uniq) 897 898 def setup_logging(self, options): 899 """ 900 This function creates the output directory and gets a file object 901 for the logfile. This function must be called before write_log() 902 can be used. 903 """ 904 if options.dryrun is True: 905 return 906 907 global LOG_FILE_OBJ 908 if not options.template: 909 try: 910 old = os.umask(0) 911 os.makedirs(self.outputdir, mode=0o777) 912 os.umask(old) 913 filename = os.path.join(self.outputdir, 'log') 914 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') 915 except OSError as e: 916 fail('%s' % e) 917 918 def run(self, options): 919 """ 920 Walk through all the Tests and TestGroups, calling run(). 921 """ 922 try: 923 os.chdir(self.outputdir) 924 except OSError: 925 fail('Could not change to directory %s' % self.outputdir) 926 # make a symlink to the output for the currently running test 927 logsymlink = os.path.join(self.outputdir, '../current') 928 if os.path.islink(logsymlink): 929 os.unlink(logsymlink) 930 if not os.path.exists(logsymlink): 931 os.symlink(self.outputdir, logsymlink) 932 else: 933 write_log('Could not make a symlink to directory %s\n' % 934 self.outputdir, LOG_ERR) 935 936 if options.kmemleak: 937 cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"' 938 check_output(cmd, shell=True) 939 940 iteration = 0 941 while iteration < options.iterations: 942 for test in sorted(self.tests.keys()): 943 self.tests[test].run(options) 944 for testgroup in sorted(self.testgroups.keys()): 945 self.testgroups[testgroup].run(options) 946 iteration += 1 947 948 def summary(self): 949 if Result.total == 0: 950 return 2 951 952 print('\nResults Summary') 953 for key in list(Result.runresults.keys()): 954 if Result.runresults[key] != 0: 955 print('%s\t% 4d' % (key, Result.runresults[key])) 956 957 m, s = divmod(time() - self.starttime, 60) 958 h, m = divmod(m, 60) 959 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) 960 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / 961 float(Result.total)) * 100)) 962 print('Log directory:\t%s' % self.outputdir) 963 964 if Result.runresults['FAIL'] > 0: 965 return 1 966 967 if Result.runresults['KILLED'] > 0: 968 return 1 969 970 if Result.runresults['RERAN'] > 0: 971 return 3 972 973 return 0 974 975 976def write_log(msg, target): 977 """ 978 Write the provided message to standard out, standard error or 979 the logfile. If specifying LOG_FILE, then `msg` must be a bytes 980 like object. This way we can still handle output from tests that 981 may be in unexpected encodings. 982 """ 983 if target == LOG_OUT: 984 os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8')) 985 elif target == LOG_ERR: 986 os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8')) 987 elif target == LOG_FILE: 988 os.write(LOG_FILE_OBJ.fileno(), msg) 989 else: 990 fail('log_msg called with unknown target "%s"' % target) 991 992 993def verify_file(pathname): 994 """ 995 Verify that the supplied pathname is an executable regular file. 996 """ 997 if os.path.isdir(pathname) or os.path.islink(pathname): 998 return False 999 1000 for ext in '', '.ksh', '.sh': 1001 script_path = pathname + ext 1002 if os.path.isfile(script_path) and os.access(script_path, os.X_OK): 1003 return True 1004 1005 return False 1006 1007 1008def verify_user(user): 1009 """ 1010 Verify that the specified user exists on this system, and can execute 1011 sudo without being prompted for a password. 1012 """ 1013 testcmd = [SUDO, '-n', '-u', user, TRUE] 1014 1015 if user in Cmd.verified_users: 1016 return True 1017 1018 try: 1019 getpwnam(user) 1020 except KeyError: 1021 write_log("Warning: user '%s' does not exist.\n" % user, 1022 LOG_ERR) 1023 return False 1024 1025 p = Popen(testcmd) 1026 p.wait() 1027 if p.returncode != 0: 1028 write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user, 1029 LOG_ERR) 1030 return False 1031 else: 1032 Cmd.verified_users.append(user) 1033 1034 return True 1035 1036 1037def find_tests(testrun, options): 1038 """ 1039 For the given list of pathnames, add files as Tests. For directories, 1040 if do_groups is True, add the directory as a TestGroup. If False, 1041 recursively search for executable files. 1042 """ 1043 1044 for p in sorted(options.pathnames): 1045 if os.path.isdir(p): 1046 for dirname, _, filenames in os.walk(p): 1047 if options.do_groups: 1048 testrun.addtestgroup(dirname, filenames, options) 1049 else: 1050 for f in sorted(filenames): 1051 testrun.addtest(os.path.join(dirname, f), options) 1052 else: 1053 testrun.addtest(p, options) 1054 1055 1056def filter_tests(testrun, options): 1057 try: 1058 fh = open(options.logfile, "r") 1059 except Exception as e: 1060 fail('%s' % e) 1061 1062 failed = {} 1063 while True: 1064 line = fh.readline() 1065 if not line: 1066 break 1067 m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line) 1068 if not m: 1069 continue 1070 group, test = m.group(1, 2) 1071 try: 1072 failed[group].append(test) 1073 except KeyError: 1074 failed[group] = [test] 1075 fh.close() 1076 1077 testrun.filter(failed) 1078 1079 1080def fail(retstr, ret=1): 1081 print('%s: %s' % (sys.argv[0], retstr)) 1082 exit(ret) 1083 1084 1085def kmemleak_cb(option, opt_str, value, parser): 1086 if not os.path.exists(KMEMLEAK_FILE): 1087 fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " + 1088 "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.") 1089 1090 setattr(parser.values, option.dest, True) 1091 1092 1093def options_cb(option, opt_str, value, parser): 1094 path_options = ['outputdir', 'template', 'testdir', 'logfile'] 1095 1096 if opt_str in parser.rargs: 1097 fail('%s may only be specified once.' % opt_str) 1098 1099 if option.dest == 'runfiles': 1100 parser.values.cmd = 'rdconfig' 1101 value = set(os.path.abspath(p) for p in value.split(',')) 1102 if option.dest == 'tags': 1103 value = [x.strip() for x in value.split(',')] 1104 1105 if option.dest in path_options: 1106 setattr(parser.values, option.dest, os.path.abspath(value)) 1107 else: 1108 setattr(parser.values, option.dest, value) 1109 1110 1111def parse_args(): 1112 parser = OptionParser() 1113 parser.add_option('-c', action='callback', callback=options_cb, 1114 type='string', dest='runfiles', metavar='runfiles', 1115 help='Specify tests to run via config files.') 1116 parser.add_option('-d', action='store_true', default=False, dest='dryrun', 1117 help='Dry run. Print tests, but take no other action.') 1118 parser.add_option('-D', action='store_true', default=False, dest='debug', 1119 help='Write all test output to stdout as it arrives.') 1120 parser.add_option('-l', action='callback', callback=options_cb, 1121 default=None, dest='logfile', metavar='logfile', 1122 type='string', 1123 help='Read logfile and re-run tests which failed.') 1124 parser.add_option('-g', action='store_true', default=False, 1125 dest='do_groups', help='Make directories TestGroups.') 1126 parser.add_option('-o', action='callback', callback=options_cb, 1127 default=BASEDIR, dest='outputdir', type='string', 1128 metavar='outputdir', help='Specify an output directory.') 1129 parser.add_option('-i', action='callback', callback=options_cb, 1130 default=TESTDIR, dest='testdir', type='string', 1131 metavar='testdir', help='Specify a test directory.') 1132 parser.add_option('-K', action='store_true', default=False, dest='kmsg', 1133 help='Log tests names to /dev/kmsg') 1134 parser.add_option('-m', action='callback', callback=kmemleak_cb, 1135 default=False, dest='kmemleak', 1136 help='Enable kmemleak reporting (Linux only)') 1137 parser.add_option('-p', action='callback', callback=options_cb, 1138 default='', dest='pre', metavar='script', 1139 type='string', help='Specify a pre script.') 1140 parser.add_option('-P', action='callback', callback=options_cb, 1141 default='', dest='post', metavar='script', 1142 type='string', help='Specify a post script.') 1143 parser.add_option('-q', action='store_true', default=False, dest='quiet', 1144 help='Silence on the console during a test run.') 1145 parser.add_option('-s', action='callback', callback=options_cb, 1146 default='', dest='failsafe', metavar='script', 1147 type='string', help='Specify a failsafe script.') 1148 parser.add_option('-S', action='callback', callback=options_cb, 1149 default='', dest='failsafe_user', 1150 metavar='failsafe_user', type='string', 1151 help='Specify a user to execute the failsafe script.') 1152 parser.add_option('-t', action='callback', callback=options_cb, default=60, 1153 dest='timeout', metavar='seconds', type='int', 1154 help='Timeout (in seconds) for an individual test.') 1155 parser.add_option('-u', action='callback', callback=options_cb, 1156 default='', dest='user', metavar='user', type='string', 1157 help='Specify a different user name to run as.') 1158 parser.add_option('-w', action='callback', callback=options_cb, 1159 default=None, dest='template', metavar='template', 1160 type='string', help='Create a new config file.') 1161 parser.add_option('-x', action='callback', callback=options_cb, default='', 1162 dest='pre_user', metavar='pre_user', type='string', 1163 help='Specify a user to execute the pre script.') 1164 parser.add_option('-X', action='callback', callback=options_cb, default='', 1165 dest='post_user', metavar='post_user', type='string', 1166 help='Specify a user to execute the post script.') 1167 parser.add_option('-T', action='callback', callback=options_cb, default='', 1168 dest='tags', metavar='tags', type='string', 1169 help='Specify tags to execute specific test groups.') 1170 parser.add_option('-I', action='callback', callback=options_cb, default=1, 1171 dest='iterations', metavar='iterations', type='int', 1172 help='Number of times to run the test run.') 1173 (options, pathnames) = parser.parse_args() 1174 1175 if options.runfiles and len(pathnames): 1176 fail('Extraneous arguments.') 1177 1178 options.pathnames = [os.path.abspath(path) for path in pathnames] 1179 1180 return options 1181 1182 1183def main(): 1184 options = parse_args() 1185 1186 testrun = TestRun(options) 1187 1188 if options.runfiles: 1189 testrun.read(options) 1190 else: 1191 find_tests(testrun, options) 1192 1193 if options.logfile: 1194 filter_tests(testrun, options) 1195 1196 if options.template: 1197 testrun.write(options) 1198 exit(0) 1199 1200 testrun.complete_outputdirs() 1201 testrun.run(options) 1202 exit(testrun.summary()) 1203 1204 1205if __name__ == '__main__': 1206 main() 1207