Lines Matching +full:self +full:- +full:test
2 # SPDX-License-Identifier: CDDL-1.0
72 return t.tv_sec + t.tv_nsec * 1e-9
79 def __init__(self): argument
80 self.starttime = None
81 self.returncode = None
82 self.runtime = ''
83 self.stdout = []
84 self.stderr = []
85 self.kmemleak = ''
86 self.result = ''
88 def done(self, proc, killed, reran): argument
93 m, s = divmod(monotonic_time() - self.starttime, 60)
94 self.runtime = '%02d:%02d' % (m, s)
95 self.returncode = proc.returncode
99 self.result = 'KILLED'
101 elif len(self.kmemleak) > 0:
102 self.result = 'FAIL'
104 elif self.returncode == 0:
105 self.result = 'PASS'
107 elif self.returncode == 4:
108 self.result = 'SKIP'
110 elif self.returncode != 0:
111 self.result = 'FAIL'
120 def __init__(self, stream, debug=False): argument
121 self.stream = stream
122 self.debug = debug
123 self._buf = b''
124 self.lines = []
126 def fileno(self): argument
127 return self.stream.fileno()
129 def read(self, drain=0): argument
133 while self._read() is not None:
137 def _read(self): argument
144 fd = self.fileno()
148 if self.debug:
151 self._buf += buf
154 buf = self._buf + buf
156 self._buf = rest
159 self.lines += [(now, r) for r in rows]
165 def __init__(self, pathname, identifier=None, outputdir=None, argument
167 self.pathname = pathname
168 self.identifier = identifier
169 self.outputdir = outputdir or 'BASEDIR'
171 The timeout for tests is measured in wall-clock time
173 self.timeout = timeout
174 self.user = user or ''
175 self.killed = False
176 self.reran = None
177 self.result = Result()
179 if self.timeout is None:
180 self.timeout = 60
182 def __str__(self): argument
189 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
191 def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False): argument
195 If we're running with the -O option, then dump debug info about the
200 - 30 lines from 'top'
201 - /proc/<PID>/stack output of process with highest CPU usage
202 - Last lines strace-ing process with highest CPU usage
206 TOP_OUT="$(COLUMNS=160 top -b -n 1 | head -n 30)"
207 read -r PID CMD <<< $(echo "$TOP_OUT" | /usr/bin/awk \
219 echo "----- top -----"
221 echo "----- /proc/$PID/stack ($CMD)) -----"
223 echo "----- strace ($CMD) -----"
224 TMPFILE="$(mktemp --suffix=ZTS)"
225 /usr/bin/strace -k --stack-traces -p $PID &> "$TMPFILE" &
228 tail -n 30 $TMPFILE
230 echo "##### /proc/sysrq-trigger stack #####"
232 c = "sudo bash -c '" + c + "'"
236 kp = Popen([SUDO, "sh", "-c",
243 kp = Popen([SUDO, "sh", "-c",
244 "echo l > /proc/sysrq-trigger"])
253 self.killed = True
254 do_sudo = len(self.user) != 0
255 signal = '-TERM'
268 If this is not a user-initiated kill and the test has not been
269 reran before we consider if the test needs to be reran:
270 If the test has spent some time hibernating and didn't run the whole
271 length of time before being timed out we will rerun the test.
273 if keyboard_interrupt is False and self.reran is None:
274 runtime = monotonic_time() - self.result.starttime
275 if int(self.timeout) > runtime:
276 self.killed = False
277 self.reran = False
278 self.run(options, dryrun=False, kmemleak=kmemleak)
279 self.reran = True
281 def update_cmd_privs(self, cmd, user): argument
310 ret = '%s -E -u %s %s %s' % (SUDO, user, tmpdirarg, cmd)
313 def collect_output(self, proc, debug=False): argument
332 def run(self, options, dryrun=None, kmemleak=None): argument
334 This is the main function that runs each individual test.
341 print(self)
346 privcmd = self.update_cmd_privs(self.pathname, self.user)
349 if not os.path.isdir(self.outputdir):
350 os.makedirs(self.outputdir, mode=0o777)
356 Log each test we run to /dev/kmsg (on Linux), so if there's a kernel
357 warning we'll be able to match it up to a particular test.
361 kp = Popen([SUDO, "sh", "-c",
362 f"echo ZTS run {self.pathname} > /dev/kmsg"])
368 Log each test we run to /dev/ttyu0 (on FreeBSD), so if there's a kernel
369 warning we'll be able to match it up to a particular test.
373 kp = Popen([SUDO, "sh", "-c",
374 f"echo ZTS run {self.pathname} > /dev/ttyu0"])
379 self.result.starttime = monotonic_time()
382 cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"'
387 if int(self.timeout) == 0:
388 self.timeout = sys.maxsize / (10 ** 9)
390 int(self.timeout), self.kill_cmd, [proc, options, kmemleak]
396 out, err = self.collect_output(proc, options.debug)
397 self.result.stdout = out
398 self.result.stderr = err
401 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"'
404 self.result.kmemleak = check_output(cmd, shell=True)
406 self.kill_cmd(proc, options, kmemleak, True)
411 if self.reran is not False:
412 self.result.done(proc, self.killed, self.reran)
414 def skip(self): argument
416 Initialize enough of the test result that we can log a skipped
421 self.result.stdout = self.result.stderr = []
422 self.result.starttime = monotonic_time()
423 m, s = divmod(monotonic_time() - self.result.starttime, 60)
424 self.result.runtime = '%02d:%02d' % (m, s)
425 self.result.result = 'SKIP'
427 def log(self, options, suppress_console=False): argument
431 merged stdout and stderr), and for each test, the unmodified
439 if self.reran is True:
441 user = ' (run as %s)' % (self.user if len(self.user) else logname)
442 if self.identifier:
443 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
445 msga = 'Test: %s%s ' % (self.pathname, user)
446 msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer)
447 pad = ' ' * (80 - (len(msga) + len(msgb)))
450 # The result line is always written to the log file. If -q was
454 write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
458 elif options.quiet and self.result.result != 'PASS':
461 lines = sorted(self.result.stdout + self.result.stderr,
467 encoding='utf-8')
471 if len(self.result.stdout):
472 with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
473 for _, line in self.result.stdout:
475 if len(self.result.stderr):
476 with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
477 for _, line in self.result.stderr:
479 if len(self.result.stdout) and len(self.result.stderr):
480 with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
483 if len(self.result.kmemleak):
484 with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem:
485 kmem.write(self.result.kmemleak)
488 class Test(Cmd): class
492 def __init__(self, pathname, argument
495 super(Test, self).__init__(pathname, **kwargs)
496 self.pre = pre or ''
497 self.pre_user = pre_user or ''
498 self.post = post or ''
499 self.post_user = post_user or ''
500 self.failsafe = failsafe or ''
501 self.failsafe_user = failsafe_user or ''
502 self.tags = tags or []
504 def __str__(self): argument
506 if len(self.pre_user):
507 pre_user = ' (as %s)' % (self.pre_user)
508 if len(self.post_user):
509 post_user = ' (as %s)' % (self.post_user)
510 if len(self.failsafe_user):
511 failsafe_user = ' (as %s)' % (self.failsafe_user)
522 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
523 self.pre, pre_user, self.post, post_user, self.failsafe,
524 failsafe_user, self.tags)
526 def verify(self): argument
528 Check the pre/post/failsafe scripts, user and Test. Omit the Test from
531 files = [self.pre, self.pathname, self.post, self.failsafe]
532 users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
536 write_log("Warning: Test '%s' not added to this run because"
542 write_log("Not adding Test '%s' to this run.\n" %
543 self.pathname, LOG_ERR)
548 def run(self, options, dryrun=None, kmemleak=None): argument
551 script doesn't pass, skip this Test. Run the post script regardless.
552 If the Test is killed, also run the failsafe script.
554 odir = os.path.join(self.outputdir, os.path.basename(self.pre))
555 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
556 timeout=self.timeout, user=self.pre_user)
557 test = Cmd(self.pathname, identifier=self.identifier,
558 outputdir=self.outputdir, timeout=self.timeout,
559 user=self.user)
560 odir = os.path.join(self.outputdir, os.path.basename(self.failsafe))
561 failsafe = Cmd(self.failsafe, identifier=self.identifier,
562 outputdir=odir, timeout=self.timeout,
563 user=self.failsafe_user)
564 odir = os.path.join(self.outputdir, os.path.basename(self.post))
565 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
566 timeout=self.timeout, user=self.post_user)
575 test.run(options, kmemleak=kmemleak)
576 if test.result.result == 'KILLED' and len(failsafe.pathname):
580 test.skip()
582 test.log(options)
589 class TestGroup(Test):
590 props = Test.props + ['tests']
592 def __init__(self, pathname, tests=None, **kwargs): argument
593 super(TestGroup, self).__init__(pathname, **kwargs)
594 self.tests = tests or []
596 def __str__(self): argument
598 if len(self.pre_user):
599 pre_user = ' (as %s)' % (self.pre_user)
600 if len(self.post_user):
601 post_user = ' (as %s)' % (self.post_user)
602 if len(self.failsafe_user):
603 failsafe_user = ' (as %s)' % (self.failsafe_user)
615 ''' % (self.pathname, self.identifier, self.outputdir, self.tests,
616 self.timeout, self.user, self.pre, pre_user, self.post, post_user,
617 self.failsafe, failsafe_user, self.tags)
619 def filter(self, keeplist): argument
620 self.tests = [x for x in self.tests if x in keeplist]
622 def verify(self): argument
630 if len(self.pre) and not os.path.isabs(self.pre):
631 self.pre = os.path.join(self.pathname, self.pre)
632 if len(self.post) and not os.path.isabs(self.post):
633 self.post = os.path.join(self.pathname, self.post)
634 if len(self.failsafe) and not os.path.isabs(self.failsafe):
635 self.post = os.path.join(self.pathname, self.post)
637 auxfiles = [self.pre, self.post, self.failsafe]
638 users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
641 if f != self.failsafe and self.pathname != os.path.dirname(f):
644 "directory.\n" % (self.pathname, f), LOG_ERR)
650 (self.pathname, f), LOG_ERR)
656 self.pathname, LOG_ERR)
660 for test in self.tests:
661 if not verify_file(os.path.join(self.pathname, test)):
662 del self.tests[self.tests.index(test)]
663 write_log("Warning: Test '%s' removed from TestGroup '%s' "
665 (test, self.pathname), LOG_ERR)
667 return len(self.tests) != 0
669 def run(self, options, dryrun=None, kmemleak=None): argument
673 post script regardless. Run the failsafe script when a test is killed.
675 # tags assigned to this test group also include the test names
676 if options.tags and not set(self.tags).intersection(set(options.tags)):
679 odir = os.path.join(self.outputdir, os.path.basename(self.pre))
680 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
681 user=self.pre_user, identifier=self.identifier)
682 odir = os.path.join(self.outputdir, os.path.basename(self.post))
683 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
684 user=self.post_user, identifier=self.identifier)
692 for fname in self.tests:
693 odir = os.path.join(self.outputdir, fname)
694 test = Cmd(os.path.join(self.pathname, fname), outputdir=odir,
695 timeout=self.timeout, user=self.user,
696 identifier=self.identifier)
697 odir = os.path.join(odir, os.path.basename(self.failsafe))
698 failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
699 user=self.failsafe_user, identifier=self.identifier)
701 test.run(options, dryrun=dryrun, kmemleak=kmemleak)
702 if test.result.result == 'KILLED' and len(failsafe.pathname):
706 test.skip()
708 test.log(options)
718 def __init__(self, options): argument
719 self.tests = {}
720 self.testgroups = {}
721 self.starttime = time()
722 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
723 self.outputdir = os.path.join(options.outputdir, self.timestamp)
724 self.setup_logging(options)
725 self.defaults = [
740 def __str__(self): argument
741 s = 'TestRun:\n outputdir: %s\n' % self.outputdir
743 for key in sorted(self.tests.keys()):
744 s += '%s%s' % (self.tests[key].__str__(), '\n')
746 for key in sorted(self.testgroups.keys()):
747 s += '%s%s' % (self.testgroups[key].__str__(), '\n')
750 def addtest(self, pathname, options): argument
752 Create a new Test, and apply any properties that were passed in
756 test = Test(pathname)
757 for prop in Test.props:
758 setattr(test, prop, getattr(options, prop))
760 if test.verify():
761 self.tests[pathname] = test
763 def addtestgroup(self, dirname, filenames, options): argument
769 if dirname not in self.testgroups:
771 for prop in Test.props:
779 self.testgroups[dirname] = testgroup
780 self.testgroups[dirname].tests = sorted(filenames)
784 def filter(self, keeplist): argument
785 for group in list(self.testgroups.keys()):
787 del self.testgroups[group]
790 g = self.testgroups[group]
797 for test in list(self.tests.keys()):
798 directory, base = os.path.split(test)
800 del self.tests[test]
802 def read(self, options): argument
806 section, and apply the appropriate properties to the Test or
808 in the 'DEFAULT' section. If the Test or TestGroup passes
813 failed = options.runfiles - set(parsed)
822 setattr(self, opt, os.path.join(outputdir, self.timestamp))
824 setattr(self, opt, config.get('DEFAULT', opt))
854 setattr(self, opt,
856 self.timestamp))
865 self.testgroups[section] = testgroup
867 test = Test(section)
868 for prop in Test.props:
873 setattr(test, prop,
877 setattr(self, opt,
879 self.timestamp))
881 setattr(test, prop, config.get(sect, prop))
883 if test.verify():
884 self.tests[section] = test
886 def write(self, options): argument
897 self.defaults])
900 for test in sorted(self.tests.keys()):
901 config.add_section(test)
902 for prop in Test.props:
903 if prop not in self.props:
904 config.set(test, prop,
905 getattr(self.tests[test], prop))
907 for testgroup in sorted(self.testgroups.keys()):
909 config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
911 if prop not in self.props:
913 getattr(self.testgroups[testgroup], prop))
921 def complete_outputdirs(self): argument
926 the test and group names in structure. Tests will be able
928 TestGroups will be able to create one subdirectory per test in the
931 directory rooted at the outputdir of the Test or TestGroup in
933 rooted at the outputdir of each Test for their output.
936 alltests = dict(list(self.tests.items()) +
937 list(self.testgroups.items()))
938 base = os.path.join(self.outputdir, 'output')
957 uniq = path.split('/', cut)[-1]
960 def setup_logging(self, options): argument
973 os.makedirs(self.outputdir, mode=0o777)
975 filename = os.path.join(self.outputdir, 'log')
980 def run(self, options): argument
985 os.chdir(self.outputdir)
987 fail('Could not change to directory %s' % self.outputdir)
988 # make a symlink to the output for the currently running test
989 logsymlink = os.path.join(self.outputdir, '../current')
993 os.symlink(self.outputdir, logsymlink)
996 self.outputdir, LOG_ERR)
999 cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"'
1004 for test in sorted(self.tests.keys()):
1005 self.tests[test].run(options)
1006 for testgroup in sorted(self.testgroups.keys()):
1007 self.testgroups[testgroup].run(options)
1010 def summary(self): argument
1019 m, s = divmod(time() - self.starttime, 60)
1024 print('Log directory:\t%s' % self.outputdir)
1046 os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
1048 os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
1075 testcmd = [SUDO, '-n', '-u', user, TRUE]
1129 m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line)
1132 group, test = m.group(1, 2)
1134 failed[group].append(test)
1136 failed[group] = [test]
1175 parser.add_option('-c', action='callback', callback=options_cb,
1178 parser.add_option('-d', action='store_true', default=False, dest='dryrun',
1180 parser.add_option('-D', action='store_true', default=False, dest='debug',
1181 help='Write all test output to stdout as it arrives.')
1182 parser.add_option('-l', action='callback', callback=options_cb,
1185 help='Read logfile and re-run tests which failed.')
1186 parser.add_option('-g', action='store_true', default=False,
1188 parser.add_option('-o', action='callback', callback=options_cb,
1191 parser.add_option('-O', action='store_true', default=False,
1193 help='Dump debugging info to /dev/kmsg on test timeout')
1194 parser.add_option('-i', action='callback', callback=options_cb,
1196 metavar='testdir', help='Specify a test directory.')
1197 parser.add_option('-K', action='store_true', default=False, dest='kmsg',
1199 parser.add_option('-m', action='callback', callback=kmemleak_cb,
1202 parser.add_option('-p', action='callback', callback=options_cb,
1205 parser.add_option('-P', action='callback', callback=options_cb,
1208 parser.add_option('-q', action='store_true', default=False, dest='quiet',
1209 help='Silence on the console during a test run.')
1210 parser.add_option('-s', action='callback', callback=options_cb,
1213 parser.add_option('-S', action='callback', callback=options_cb,
1217 parser.add_option('-t', action='callback', callback=options_cb, default=60,
1219 help='Timeout (in seconds) for an individual test.')
1220 parser.add_option('-u', action='callback', callback=options_cb,
1223 parser.add_option('-w', action='callback', callback=options_cb,
1226 parser.add_option('-x', action='callback', callback=options_cb, default='',
1229 parser.add_option('-X', action='callback', callback=options_cb, default='',
1232 parser.add_option('-T', action='callback', callback=options_cb, default='',
1234 help='Specify tags to execute specific test groups.')
1235 parser.add_option('-I', action='callback', callback=options_cb, default=1,
1237 help='Number of times to run the test run.')
1271 fail("Uncaught exception in test runner:\n" + traceback.format_exc())