xref: /freebsd/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in (revision e92ffd9b626833ebdbf2742c8ffddc6cd94b963e)
1#!/usr/bin/env @PYTHON_SHEBANG@
2
3#
4# This file and its contents are supplied under the terms of the
5# Common Development and Distribution License ("CDDL"), version 1.0.
6# You may only use this file in accordance with the terms of version
7# 1.0 of the CDDL.
8#
9# A full copy of the text of the CDDL should have accompanied this
10# source.  A copy of the CDDL is also available via the Internet at
11# http://www.illumos.org/license/CDDL.
12#
13
14#
15# Copyright (c) 2012, 2018 by Delphix. All rights reserved.
16# Copyright (c) 2019 Datto Inc.
17#
18# This script must remain compatible with Python 3.6+.
19#
20
21import os
22import sys
23import ctypes
24import re
25import configparser
26
27from datetime import datetime
28from optparse import OptionParser
29from pwd import getpwnam
30from pwd import getpwuid
31from select import select
32from subprocess import PIPE
33from subprocess import Popen
34from threading import Timer
35from time import time, CLOCK_MONOTONIC
36
37BASEDIR = '/var/tmp/test_results'
38TESTDIR = '/usr/share/zfs/'
39KILL = 'kill'
40TRUE = 'true'
41SUDO = 'sudo'
42LOG_FILE = 'LOG_FILE'
43LOG_OUT = 'LOG_OUT'
44LOG_ERR = 'LOG_ERR'
45LOG_FILE_OBJ = None
46
47
48class timespec(ctypes.Structure):
49    _fields_ = [
50        ('tv_sec', ctypes.c_long),
51        ('tv_nsec', ctypes.c_long)
52    ]
53
54
55librt = ctypes.CDLL('librt.so.1', use_errno=True)
56clock_gettime = librt.clock_gettime
57clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)]
58
59
60def monotonic_time():
61    t = timespec()
62    if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
63        errno_ = ctypes.get_errno()
64        raise OSError(errno_, os.strerror(errno_))
65    return t.tv_sec + t.tv_nsec * 1e-9
66
67
68class Result(object):
69    total = 0
70    runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0}
71
72    def __init__(self):
73        self.starttime = None
74        self.returncode = None
75        self.runtime = ''
76        self.stdout = []
77        self.stderr = []
78        self.result = ''
79
80    def done(self, proc, killed, reran):
81        """
82        Finalize the results of this Cmd.
83        """
84        Result.total += 1
85        m, s = divmod(monotonic_time() - self.starttime, 60)
86        self.runtime = '%02d:%02d' % (m, s)
87        self.returncode = proc.returncode
88        if reran is True:
89            Result.runresults['RERAN'] += 1
90        if killed:
91            self.result = 'KILLED'
92            Result.runresults['KILLED'] += 1
93        elif self.returncode == 0:
94            self.result = 'PASS'
95            Result.runresults['PASS'] += 1
96        elif self.returncode == 4:
97            self.result = 'SKIP'
98            Result.runresults['SKIP'] += 1
99        elif self.returncode != 0:
100            self.result = 'FAIL'
101            Result.runresults['FAIL'] += 1
102
103
104class Output(object):
105    """
106    This class is a slightly modified version of the 'Stream' class found
107    here: http://goo.gl/aSGfv
108    """
109    def __init__(self, stream):
110        self.stream = stream
111        self._buf = b''
112        self.lines = []
113
114    def fileno(self):
115        return self.stream.fileno()
116
117    def read(self, drain=0):
118        """
119        Read from the file descriptor. If 'drain' set, read until EOF.
120        """
121        while self._read() is not None:
122            if not drain:
123                break
124
125    def _read(self):
126        """
127        Read up to 4k of data from this output stream. Collect the output
128        up to the last newline, and append it to any leftover data from a
129        previous call. The lines are stored as a (timestamp, data) tuple
130        for easy sorting/merging later.
131        """
132        fd = self.fileno()
133        buf = os.read(fd, 4096)
134        if not buf:
135            return None
136        if b'\n' not in buf:
137            self._buf += buf
138            return []
139
140        buf = self._buf + buf
141        tmp, rest = buf.rsplit(b'\n', 1)
142        self._buf = rest
143        now = datetime.now()
144        rows = tmp.split(b'\n')
145        self.lines += [(now, r) for r in rows]
146
147
148class Cmd(object):
149    verified_users = []
150
151    def __init__(self, pathname, identifier=None, outputdir=None,
152                 timeout=None, user=None, tags=None):
153        self.pathname = pathname
154        self.identifier = identifier
155        self.outputdir = outputdir or 'BASEDIR'
156        """
157        The timeout for tests is measured in wall-clock time
158        """
159        self.timeout = timeout
160        self.user = user or ''
161        self.killed = False
162        self.reran = None
163        self.result = Result()
164
165        if self.timeout is None:
166            self.timeout = 60
167
168    def __str__(self):
169        return '''\
170Pathname: %s
171Identifier: %s
172Outputdir: %s
173Timeout: %d
174User: %s
175''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
176
177    def kill_cmd(self, proc, keyboard_interrupt=False):
178        """
179        Kill a running command due to timeout, or ^C from the keyboard. If
180        sudo is required, this user was verified previously.
181        """
182        self.killed = True
183        do_sudo = len(self.user) != 0
184        signal = '-TERM'
185
186        cmd = [SUDO, KILL, signal, str(proc.pid)]
187        if not do_sudo:
188            del cmd[0]
189
190        try:
191            kp = Popen(cmd)
192            kp.wait()
193        except Exception:
194            pass
195
196        """
197        If this is not a user-initiated kill and the test has not been
198        reran before we consider if the test needs to be reran:
199        If the test has spent some time hibernating and didn't run the whole
200        length of time before being timed out we will rerun the test.
201        """
202        if keyboard_interrupt is False and self.reran is None:
203            runtime = monotonic_time() - self.result.starttime
204            if int(self.timeout) > runtime:
205                self.killed = False
206                self.reran = False
207                self.run(False)
208                self.reran = True
209
210    def update_cmd_privs(self, cmd, user):
211        """
212        If a user has been specified to run this Cmd and we're not already
213        running as that user, prepend the appropriate sudo command to run
214        as that user.
215        """
216        me = getpwuid(os.getuid())
217
218        if not user or user is me:
219            if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
220                cmd += '.ksh'
221            if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
222                cmd += '.sh'
223            return cmd
224
225        if not os.path.isfile(cmd):
226            if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
227                cmd += '.ksh'
228            if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
229                cmd += '.sh'
230
231        ret = '%s -E -u %s %s' % (SUDO, user, cmd)
232        return ret.split(' ')
233
234    def collect_output(self, proc):
235        """
236        Read from stdout/stderr as data becomes available, until the
237        process is no longer running. Return the lines from the stdout and
238        stderr Output objects.
239        """
240        out = Output(proc.stdout)
241        err = Output(proc.stderr)
242        res = []
243        while proc.returncode is None:
244            proc.poll()
245            res = select([out, err], [], [], .1)
246            for fd in res[0]:
247                fd.read()
248        for fd in res[0]:
249            fd.read(drain=1)
250
251        return out.lines, err.lines
252
253    def run(self, dryrun):
254        """
255        This is the main function that runs each individual test.
256        Determine whether or not the command requires sudo, and modify it
257        if needed. Run the command, and update the result object.
258        """
259        if dryrun is True:
260            print(self)
261            return
262
263        privcmd = self.update_cmd_privs(self.pathname, self.user)
264        try:
265            old = os.umask(0)
266            if not os.path.isdir(self.outputdir):
267                os.makedirs(self.outputdir, mode=0o777)
268            os.umask(old)
269        except OSError as e:
270            fail('%s' % e)
271
272        self.result.starttime = monotonic_time()
273        proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
274        # Allow a special timeout value of 0 to mean infinity
275        if int(self.timeout) == 0:
276            self.timeout = sys.maxsize
277        t = Timer(int(self.timeout), self.kill_cmd, [proc])
278
279        try:
280            t.start()
281            self.result.stdout, self.result.stderr = self.collect_output(proc)
282        except KeyboardInterrupt:
283            self.kill_cmd(proc, True)
284            fail('\nRun terminated at user request.')
285        finally:
286            t.cancel()
287
288        if self.reran is not False:
289            self.result.done(proc, self.killed, self.reran)
290
291    def skip(self):
292        """
293        Initialize enough of the test result that we can log a skipped
294        command.
295        """
296        Result.total += 1
297        Result.runresults['SKIP'] += 1
298        self.result.stdout = self.result.stderr = []
299        self.result.starttime = monotonic_time()
300        m, s = divmod(monotonic_time() - self.result.starttime, 60)
301        self.result.runtime = '%02d:%02d' % (m, s)
302        self.result.result = 'SKIP'
303
304    def log(self, options, suppress_console=False):
305        """
306        This function is responsible for writing all output. This includes
307        the console output, the logfile of all results (with timestamped
308        merged stdout and stderr), and for each test, the unmodified
309        stdout/stderr/merged in its own file.
310        """
311
312        logname = getpwuid(os.getuid()).pw_name
313        rer = ''
314        if self.reran is True:
315            rer = ' (RERAN)'
316        user = ' (run as %s)' % (self.user if len(self.user) else logname)
317        if self.identifier:
318            msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
319        else:
320            msga = 'Test: %s%s ' % (self.pathname, user)
321        msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer)
322        pad = ' ' * (80 - (len(msga) + len(msgb)))
323        result_line = msga + pad + msgb
324
325        # The result line is always written to the log file. If -q was
326        # specified only failures are written to the console, otherwise
327        # the result line is written to the console. The console output
328        # may be suppressed by calling log() with suppress_console=True.
329        write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
330        if not suppress_console:
331            if not options.quiet:
332                write_log(result_line, LOG_OUT)
333            elif options.quiet and self.result.result != 'PASS':
334                write_log(result_line, LOG_OUT)
335
336        lines = sorted(self.result.stdout + self.result.stderr,
337                       key=lambda x: x[0])
338
339        # Write timestamped output (stdout and stderr) to the logfile
340        for dt, line in lines:
341            timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
342                                  encoding='utf-8')
343            write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
344
345        # Write the separate stdout/stderr/merged files, if the data exists
346        if len(self.result.stdout):
347            with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
348                for _, line in self.result.stdout:
349                    os.write(out.fileno(), b'%s\n' % line)
350        if len(self.result.stderr):
351            with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
352                for _, line in self.result.stderr:
353                    os.write(err.fileno(), b'%s\n' % line)
354        if len(self.result.stdout) and len(self.result.stderr):
355            with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
356                for _, line in lines:
357                    os.write(merged.fileno(), b'%s\n' % line)
358
359
360class Test(Cmd):
361    props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
362             'post_user', 'failsafe', 'failsafe_user', 'tags']
363
364    def __init__(self, pathname,
365                 pre=None, pre_user=None, post=None, post_user=None,
366                 failsafe=None, failsafe_user=None, tags=None, **kwargs):
367        super(Test, self).__init__(pathname, **kwargs)
368        self.pre = pre or ''
369        self.pre_user = pre_user or ''
370        self.post = post or ''
371        self.post_user = post_user or ''
372        self.failsafe = failsafe or ''
373        self.failsafe_user = failsafe_user or ''
374        self.tags = tags or []
375
376    def __str__(self):
377        post_user = pre_user = failsafe_user = ''
378        if len(self.pre_user):
379            pre_user = ' (as %s)' % (self.pre_user)
380        if len(self.post_user):
381            post_user = ' (as %s)' % (self.post_user)
382        if len(self.failsafe_user):
383            failsafe_user = ' (as %s)' % (self.failsafe_user)
384        return '''\
385Pathname: %s
386Identifier: %s
387Outputdir: %s
388Timeout: %d
389User: %s
390Pre: %s%s
391Post: %s%s
392Failsafe: %s%s
393Tags: %s
394''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
395            self.pre, pre_user, self.post, post_user, self.failsafe,
396            failsafe_user, self.tags)
397
398    def verify(self):
399        """
400        Check the pre/post/failsafe scripts, user and Test. Omit the Test from
401        this run if there are any problems.
402        """
403        files = [self.pre, self.pathname, self.post, self.failsafe]
404        users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
405
406        for f in [f for f in files if len(f)]:
407            if not verify_file(f):
408                write_log("Warning: Test '%s' not added to this run because"
409                          " it failed verification.\n" % f, LOG_ERR)
410                return False
411
412        for user in [user for user in users if len(user)]:
413            if not verify_user(user):
414                write_log("Not adding Test '%s' to this run.\n" %
415                          self.pathname, LOG_ERR)
416                return False
417
418        return True
419
420    def run(self, options):
421        """
422        Create Cmd instances for the pre/post/failsafe scripts. If the pre
423        script doesn't pass, skip this Test. Run the post script regardless.
424        If the Test is killed, also run the failsafe script.
425        """
426        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
427        pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
428                      timeout=self.timeout, user=self.pre_user)
429        test = Cmd(self.pathname, identifier=self.identifier,
430                   outputdir=self.outputdir, timeout=self.timeout,
431                   user=self.user)
432        odir = os.path.join(self.outputdir, os.path.basename(self.failsafe))
433        failsafe = Cmd(self.failsafe, identifier=self.identifier,
434                       outputdir=odir, timeout=self.timeout,
435                       user=self.failsafe_user)
436        odir = os.path.join(self.outputdir, os.path.basename(self.post))
437        posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
438                       timeout=self.timeout, user=self.post_user)
439
440        cont = True
441        if len(pretest.pathname):
442            pretest.run(options.dryrun)
443            cont = pretest.result.result == 'PASS'
444            pretest.log(options)
445
446        if cont:
447            test.run(options.dryrun)
448            if test.result.result == 'KILLED' and len(failsafe.pathname):
449                failsafe.run(options.dryrun)
450                failsafe.log(options, suppress_console=True)
451        else:
452            test.skip()
453
454        test.log(options)
455
456        if len(posttest.pathname):
457            posttest.run(options.dryrun)
458            posttest.log(options)
459
460
461class TestGroup(Test):
462    props = Test.props + ['tests']
463
464    def __init__(self, pathname, tests=None, **kwargs):
465        super(TestGroup, self).__init__(pathname, **kwargs)
466        self.tests = tests or []
467
468    def __str__(self):
469        post_user = pre_user = failsafe_user = ''
470        if len(self.pre_user):
471            pre_user = ' (as %s)' % (self.pre_user)
472        if len(self.post_user):
473            post_user = ' (as %s)' % (self.post_user)
474        if len(self.failsafe_user):
475            failsafe_user = ' (as %s)' % (self.failsafe_user)
476        return '''\
477Pathname: %s
478Identifier: %s
479Outputdir: %s
480Tests: %s
481Timeout: %s
482User: %s
483Pre: %s%s
484Post: %s%s
485Failsafe: %s%s
486Tags: %s
487''' % (self.pathname, self.identifier, self.outputdir, self.tests,
488            self.timeout, self.user, self.pre, pre_user, self.post, post_user,
489            self.failsafe, failsafe_user, self.tags)
490
491    def filter(self, keeplist):
492        self.tests = [x for x in self.tests if x in keeplist]
493
494    def verify(self):
495        """
496        Check the pre/post/failsafe scripts, user and tests in this TestGroup.
497        Omit the TestGroup entirely, or simply delete the relevant tests in the
498        group, if that's all that's required.
499        """
500        # If the pre/post/failsafe scripts are relative pathnames, convert to
501        # absolute, so they stand a chance of passing verification.
502        if len(self.pre) and not os.path.isabs(self.pre):
503            self.pre = os.path.join(self.pathname, self.pre)
504        if len(self.post) and not os.path.isabs(self.post):
505            self.post = os.path.join(self.pathname, self.post)
506        if len(self.failsafe) and not os.path.isabs(self.failsafe):
507            self.post = os.path.join(self.pathname, self.post)
508
509        auxfiles = [self.pre, self.post, self.failsafe]
510        users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
511
512        for f in [f for f in auxfiles if len(f)]:
513            if f != self.failsafe and self.pathname != os.path.dirname(f):
514                write_log("Warning: TestGroup '%s' not added to this run. "
515                          "Auxiliary script '%s' exists in a different "
516                          "directory.\n" % (self.pathname, f), LOG_ERR)
517                return False
518
519            if not verify_file(f):
520                write_log("Warning: TestGroup '%s' not added to this run. "
521                          "Auxiliary script '%s' failed verification.\n" %
522                          (self.pathname, f), LOG_ERR)
523                return False
524
525        for user in [user for user in users if len(user)]:
526            if not verify_user(user):
527                write_log("Not adding TestGroup '%s' to this run.\n" %
528                          self.pathname, LOG_ERR)
529                return False
530
531        # If one of the tests is invalid, delete it, log it, and drive on.
532        for test in self.tests:
533            if not verify_file(os.path.join(self.pathname, test)):
534                del self.tests[self.tests.index(test)]
535                write_log("Warning: Test '%s' removed from TestGroup '%s' "
536                          "because it failed verification.\n" %
537                          (test, self.pathname), LOG_ERR)
538
539        return len(self.tests) != 0
540
541    def run(self, options):
542        """
543        Create Cmd instances for the pre/post/failsafe scripts. If the pre
544        script doesn't pass, skip all the tests in this TestGroup. Run the
545        post script regardless. Run the failsafe script when a test is killed.
546        """
547        # tags assigned to this test group also include the test names
548        if options.tags and not set(self.tags).intersection(set(options.tags)):
549            return
550
551        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
552        pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
553                      user=self.pre_user, identifier=self.identifier)
554        odir = os.path.join(self.outputdir, os.path.basename(self.post))
555        posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
556                       user=self.post_user, identifier=self.identifier)
557
558        cont = True
559        if len(pretest.pathname):
560            pretest.run(options.dryrun)
561            cont = pretest.result.result == 'PASS'
562            pretest.log(options)
563
564        for fname in self.tests:
565            odir = os.path.join(self.outputdir, fname)
566            test = Cmd(os.path.join(self.pathname, fname), outputdir=odir,
567                       timeout=self.timeout, user=self.user,
568                       identifier=self.identifier)
569            odir = os.path.join(odir, os.path.basename(self.failsafe))
570            failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
571                           user=self.failsafe_user, identifier=self.identifier)
572            if cont:
573                test.run(options.dryrun)
574                if test.result.result == 'KILLED' and len(failsafe.pathname):
575                    failsafe.run(options.dryrun)
576                    failsafe.log(options, suppress_console=True)
577            else:
578                test.skip()
579
580            test.log(options)
581
582        if len(posttest.pathname):
583            posttest.run(options.dryrun)
584            posttest.log(options)
585
586
587class TestRun(object):
588    props = ['quiet', 'outputdir']
589
590    def __init__(self, options):
591        self.tests = {}
592        self.testgroups = {}
593        self.starttime = time()
594        self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
595        self.outputdir = os.path.join(options.outputdir, self.timestamp)
596        self.setup_logging(options)
597        self.defaults = [
598            ('outputdir', BASEDIR),
599            ('quiet', False),
600            ('timeout', 60),
601            ('user', ''),
602            ('pre', ''),
603            ('pre_user', ''),
604            ('post', ''),
605            ('post_user', ''),
606            ('failsafe', ''),
607            ('failsafe_user', ''),
608            ('tags', [])
609        ]
610
611    def __str__(self):
612        s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
613        s += 'TESTS:\n'
614        for key in sorted(self.tests.keys()):
615            s += '%s%s' % (self.tests[key].__str__(), '\n')
616        s += 'TESTGROUPS:\n'
617        for key in sorted(self.testgroups.keys()):
618            s += '%s%s' % (self.testgroups[key].__str__(), '\n')
619        return s
620
621    def addtest(self, pathname, options):
622        """
623        Create a new Test, and apply any properties that were passed in
624        from the command line. If it passes verification, add it to the
625        TestRun.
626        """
627        test = Test(pathname)
628        for prop in Test.props:
629            setattr(test, prop, getattr(options, prop))
630
631        if test.verify():
632            self.tests[pathname] = test
633
634    def addtestgroup(self, dirname, filenames, options):
635        """
636        Create a new TestGroup, and apply any properties that were passed
637        in from the command line. If it passes verification, add it to the
638        TestRun.
639        """
640        if dirname not in self.testgroups:
641            testgroup = TestGroup(dirname)
642            for prop in Test.props:
643                setattr(testgroup, prop, getattr(options, prop))
644
645            # Prevent pre/post/failsafe scripts from running as regular tests
646            for f in [testgroup.pre, testgroup.post, testgroup.failsafe]:
647                if f in filenames:
648                    del filenames[filenames.index(f)]
649
650            self.testgroups[dirname] = testgroup
651            self.testgroups[dirname].tests = sorted(filenames)
652
653            testgroup.verify()
654
655    def filter(self, keeplist):
656        for group in list(self.testgroups.keys()):
657            if group not in keeplist:
658                del self.testgroups[group]
659                continue
660
661            g = self.testgroups[group]
662
663            if g.pre and os.path.basename(g.pre) in keeplist[group]:
664                continue
665
666            g.filter(keeplist[group])
667
668        for test in list(self.tests.keys()):
669            directory, base = os.path.split(test)
670            if directory not in keeplist or base not in keeplist[directory]:
671                del self.tests[test]
672
673    def read(self, options):
674        """
675        Read in the specified runfiles, and apply the TestRun properties
676        listed in the 'DEFAULT' section to our TestRun. Then read each
677        section, and apply the appropriate properties to the Test or
678        TestGroup. Properties from individual sections override those set
679        in the 'DEFAULT' section. If the Test or TestGroup passes
680        verification, add it to the TestRun.
681        """
682        config = configparser.RawConfigParser()
683        parsed = config.read(options.runfiles)
684        failed = options.runfiles - set(parsed)
685        if len(failed):
686            files = ' '.join(sorted(failed))
687            fail("Couldn't read config files: %s" % files)
688
689        for opt in TestRun.props:
690            if config.has_option('DEFAULT', opt):
691                setattr(self, opt, config.get('DEFAULT', opt))
692        self.outputdir = os.path.join(self.outputdir, self.timestamp)
693
694        testdir = options.testdir
695
696        for section in config.sections():
697            if 'tests' in config.options(section):
698                parts = section.split(':', 1)
699                sectiondir = parts[0]
700                identifier = parts[1] if len(parts) == 2 else None
701                if os.path.isdir(sectiondir):
702                    pathname = sectiondir
703                elif os.path.isdir(os.path.join(testdir, sectiondir)):
704                    pathname = os.path.join(testdir, sectiondir)
705                else:
706                    pathname = sectiondir
707
708                testgroup = TestGroup(os.path.abspath(pathname),
709                                      identifier=identifier)
710                for prop in TestGroup.props:
711                    for sect in ['DEFAULT', section]:
712                        if config.has_option(sect, prop):
713                            if prop == 'tags':
714                                setattr(testgroup, prop,
715                                        eval(config.get(sect, prop)))
716                            elif prop == 'failsafe':
717                                failsafe = config.get(sect, prop)
718                                setattr(testgroup, prop,
719                                        os.path.join(testdir, failsafe))
720                            else:
721                                setattr(testgroup, prop,
722                                        config.get(sect, prop))
723
724                # Repopulate tests using eval to convert the string to a list
725                testgroup.tests = eval(config.get(section, 'tests'))
726
727                if testgroup.verify():
728                    self.testgroups[section] = testgroup
729            else:
730                test = Test(section)
731                for prop in Test.props:
732                    for sect in ['DEFAULT', section]:
733                        if config.has_option(sect, prop):
734                            if prop == 'failsafe':
735                                failsafe = config.get(sect, prop)
736                                setattr(test, prop,
737                                        os.path.join(testdir, failsafe))
738                            else:
739                                setattr(test, prop, config.get(sect, prop))
740
741                if test.verify():
742                    self.tests[section] = test
743
744    def write(self, options):
745        """
746        Create a configuration file for editing and later use. The
747        'DEFAULT' section of the config file is created from the
748        properties that were specified on the command line. Tests are
749        simply added as sections that inherit everything from the
750        'DEFAULT' section. TestGroups are the same, except they get an
751        option including all the tests to run in that directory.
752        """
753
754        defaults = dict([(prop, getattr(options, prop)) for prop, _ in
755                         self.defaults])
756        config = configparser.RawConfigParser(defaults)
757
758        for test in sorted(self.tests.keys()):
759            config.add_section(test)
760            for prop in Test.props:
761                if prop not in self.props:
762                    config.set(test, prop,
763                               getattr(self.tests[test], prop))
764
765        for testgroup in sorted(self.testgroups.keys()):
766            config.add_section(testgroup)
767            config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
768            for prop in TestGroup.props:
769                if prop not in self.props:
770                    config.set(testgroup, prop,
771                               getattr(self.testgroups[testgroup], prop))
772
773        try:
774            with open(options.template, 'w') as f:
775                return config.write(f)
776        except IOError:
777            fail('Could not open \'%s\' for writing.' % options.template)
778
779    def complete_outputdirs(self):
780        """
781        Collect all the pathnames for Tests, and TestGroups. Work
782        backwards one pathname component at a time, to create a unique
783        directory name in which to deposit test output. Tests will be able
784        to write output files directly in the newly modified outputdir.
785        TestGroups will be able to create one subdirectory per test in the
786        outputdir, and are guaranteed uniqueness because a group can only
787        contain files in one directory. Pre and post tests will create a
788        directory rooted at the outputdir of the Test or TestGroup in
789        question for their output. Failsafe scripts will create a directory
790        rooted at the outputdir of each Test for their output.
791        """
792        done = False
793        components = 0
794        tmp_dict = dict(list(self.tests.items()) +
795                        list(self.testgroups.items()))
796        total = len(tmp_dict)
797        base = self.outputdir
798
799        while not done:
800            paths = []
801            components -= 1
802            for testfile in list(tmp_dict.keys()):
803                uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
804                if uniq not in paths:
805                    paths.append(uniq)
806                    tmp_dict[testfile].outputdir = os.path.join(base, uniq)
807                else:
808                    break
809            done = total == len(paths)
810
811    def setup_logging(self, options):
812        """
813        This function creates the output directory and gets a file object
814        for the logfile. This function must be called before write_log()
815        can be used.
816        """
817        if options.dryrun is True:
818            return
819
820        global LOG_FILE_OBJ
821        if not options.template:
822            try:
823                old = os.umask(0)
824                os.makedirs(self.outputdir, mode=0o777)
825                os.umask(old)
826                filename = os.path.join(self.outputdir, 'log')
827                LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
828            except OSError as e:
829                fail('%s' % e)
830
831    def run(self, options):
832        """
833        Walk through all the Tests and TestGroups, calling run().
834        """
835        try:
836            os.chdir(self.outputdir)
837        except OSError:
838            fail('Could not change to directory %s' % self.outputdir)
839        # make a symlink to the output for the currently running test
840        logsymlink = os.path.join(self.outputdir, '../current')
841        if os.path.islink(logsymlink):
842            os.unlink(logsymlink)
843        if not os.path.exists(logsymlink):
844            os.symlink(self.outputdir, logsymlink)
845        else:
846            write_log('Could not make a symlink to directory %s\n' %
847                      self.outputdir, LOG_ERR)
848        iteration = 0
849        while iteration < options.iterations:
850            for test in sorted(self.tests.keys()):
851                self.tests[test].run(options)
852            for testgroup in sorted(self.testgroups.keys()):
853                self.testgroups[testgroup].run(options)
854            iteration += 1
855
856    def summary(self):
857        if Result.total == 0:
858            return 2
859
860        print('\nResults Summary')
861        for key in list(Result.runresults.keys()):
862            if Result.runresults[key] != 0:
863                print('%s\t% 4d' % (key, Result.runresults[key]))
864
865        m, s = divmod(time() - self.starttime, 60)
866        h, m = divmod(m, 60)
867        print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
868        print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
869                                            float(Result.total)) * 100))
870        print('Log directory:\t%s' % self.outputdir)
871
872        if Result.runresults['FAIL'] > 0:
873            return 1
874
875        if Result.runresults['KILLED'] > 0:
876            return 1
877
878        if Result.runresults['RERAN'] > 0:
879            return 3
880
881        return 0
882
883
884def write_log(msg, target):
885    """
886    Write the provided message to standard out, standard error or
887    the logfile. If specifying LOG_FILE, then `msg` must be a bytes
888    like object. This way we can still handle output from tests that
889    may be in unexpected encodings.
890    """
891    if target == LOG_OUT:
892        os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
893    elif target == LOG_ERR:
894        os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
895    elif target == LOG_FILE:
896        os.write(LOG_FILE_OBJ.fileno(), msg)
897    else:
898        fail('log_msg called with unknown target "%s"' % target)
899
900
901def verify_file(pathname):
902    """
903    Verify that the supplied pathname is an executable regular file.
904    """
905    if os.path.isdir(pathname) or os.path.islink(pathname):
906        return False
907
908    for ext in '', '.ksh', '.sh':
909        script_path = pathname + ext
910        if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
911            return True
912
913    return False
914
915
916def verify_user(user):
917    """
918    Verify that the specified user exists on this system, and can execute
919    sudo without being prompted for a password.
920    """
921    testcmd = [SUDO, '-n', '-u', user, TRUE]
922
923    if user in Cmd.verified_users:
924        return True
925
926    try:
927        getpwnam(user)
928    except KeyError:
929        write_log("Warning: user '%s' does not exist.\n" % user,
930                  LOG_ERR)
931        return False
932
933    p = Popen(testcmd)
934    p.wait()
935    if p.returncode != 0:
936        write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
937                  LOG_ERR)
938        return False
939    else:
940        Cmd.verified_users.append(user)
941
942    return True
943
944
945def find_tests(testrun, options):
946    """
947    For the given list of pathnames, add files as Tests. For directories,
948    if do_groups is True, add the directory as a TestGroup. If False,
949    recursively search for executable files.
950    """
951
952    for p in sorted(options.pathnames):
953        if os.path.isdir(p):
954            for dirname, _, filenames in os.walk(p):
955                if options.do_groups:
956                    testrun.addtestgroup(dirname, filenames, options)
957                else:
958                    for f in sorted(filenames):
959                        testrun.addtest(os.path.join(dirname, f), options)
960        else:
961            testrun.addtest(p, options)
962
963
964def filter_tests(testrun, options):
965    try:
966        fh = open(options.logfile, "r")
967    except Exception as e:
968        fail('%s' % e)
969
970    failed = {}
971    while True:
972        line = fh.readline()
973        if not line:
974            break
975        m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line)
976        if not m:
977            continue
978        group, test = m.group(1, 2)
979        try:
980            failed[group].append(test)
981        except KeyError:
982            failed[group] = [test]
983    fh.close()
984
985    testrun.filter(failed)
986
987
988def fail(retstr, ret=1):
989    print('%s: %s' % (sys.argv[0], retstr))
990    exit(ret)
991
992
993def options_cb(option, opt_str, value, parser):
994    path_options = ['outputdir', 'template', 'testdir', 'logfile']
995
996    if opt_str in parser.rargs:
997        fail('%s may only be specified once.' % opt_str)
998
999    if option.dest == 'runfiles':
1000        parser.values.cmd = 'rdconfig'
1001        value = set(os.path.abspath(p) for p in value.split(','))
1002    if option.dest == 'tags':
1003        value = [x.strip() for x in value.split(',')]
1004
1005    if option.dest in path_options:
1006        setattr(parser.values, option.dest, os.path.abspath(value))
1007    else:
1008        setattr(parser.values, option.dest, value)
1009
1010
1011def parse_args():
1012    parser = OptionParser()
1013    parser.add_option('-c', action='callback', callback=options_cb,
1014                      type='string', dest='runfiles', metavar='runfiles',
1015                      help='Specify tests to run via config files.')
1016    parser.add_option('-d', action='store_true', default=False, dest='dryrun',
1017                      help='Dry run. Print tests, but take no other action.')
1018    parser.add_option('-l', action='callback', callback=options_cb,
1019                      default=None, dest='logfile', metavar='logfile',
1020                      type='string',
1021                      help='Read logfile and re-run tests which failed.')
1022    parser.add_option('-g', action='store_true', default=False,
1023                      dest='do_groups', help='Make directories TestGroups.')
1024    parser.add_option('-o', action='callback', callback=options_cb,
1025                      default=BASEDIR, dest='outputdir', type='string',
1026                      metavar='outputdir', help='Specify an output directory.')
1027    parser.add_option('-i', action='callback', callback=options_cb,
1028                      default=TESTDIR, dest='testdir', type='string',
1029                      metavar='testdir', help='Specify a test directory.')
1030    parser.add_option('-p', action='callback', callback=options_cb,
1031                      default='', dest='pre', metavar='script',
1032                      type='string', help='Specify a pre script.')
1033    parser.add_option('-P', action='callback', callback=options_cb,
1034                      default='', dest='post', metavar='script',
1035                      type='string', help='Specify a post script.')
1036    parser.add_option('-q', action='store_true', default=False, dest='quiet',
1037                      help='Silence on the console during a test run.')
1038    parser.add_option('-s', action='callback', callback=options_cb,
1039                      default='', dest='failsafe', metavar='script',
1040                      type='string', help='Specify a failsafe script.')
1041    parser.add_option('-S', action='callback', callback=options_cb,
1042                      default='', dest='failsafe_user',
1043                      metavar='failsafe_user', type='string',
1044                      help='Specify a user to execute the failsafe script.')
1045    parser.add_option('-t', action='callback', callback=options_cb, default=60,
1046                      dest='timeout', metavar='seconds', type='int',
1047                      help='Timeout (in seconds) for an individual test.')
1048    parser.add_option('-u', action='callback', callback=options_cb,
1049                      default='', dest='user', metavar='user', type='string',
1050                      help='Specify a different user name to run as.')
1051    parser.add_option('-w', action='callback', callback=options_cb,
1052                      default=None, dest='template', metavar='template',
1053                      type='string', help='Create a new config file.')
1054    parser.add_option('-x', action='callback', callback=options_cb, default='',
1055                      dest='pre_user', metavar='pre_user', type='string',
1056                      help='Specify a user to execute the pre script.')
1057    parser.add_option('-X', action='callback', callback=options_cb, default='',
1058                      dest='post_user', metavar='post_user', type='string',
1059                      help='Specify a user to execute the post script.')
1060    parser.add_option('-T', action='callback', callback=options_cb, default='',
1061                      dest='tags', metavar='tags', type='string',
1062                      help='Specify tags to execute specific test groups.')
1063    parser.add_option('-I', action='callback', callback=options_cb, default=1,
1064                      dest='iterations', metavar='iterations', type='int',
1065                      help='Number of times to run the test run.')
1066    (options, pathnames) = parser.parse_args()
1067
1068    if options.runfiles and len(pathnames):
1069        fail('Extraneous arguments.')
1070
1071    options.pathnames = [os.path.abspath(path) for path in pathnames]
1072
1073    return options
1074
1075
1076def main():
1077    options = parse_args()
1078
1079    testrun = TestRun(options)
1080
1081    if options.runfiles:
1082        testrun.read(options)
1083    else:
1084        find_tests(testrun, options)
1085
1086    if options.logfile:
1087        filter_tests(testrun, options)
1088
1089    if options.template:
1090        testrun.write(options)
1091        exit(0)
1092
1093    testrun.complete_outputdirs()
1094    testrun.run(options)
1095    exit(testrun.summary())
1096
1097
1098if __name__ == '__main__':
1099    main()
1100