xref: /freebsd/sys/contrib/openzfs/tests/test-runner/bin/test-runner.py.in (revision a521f2116473fbd8c09db395518f060a27d02334)
1#!/usr/bin/env @PYTHON_SHEBANG@
2
3#
4# This file and its contents are supplied under the terms of the
5# Common Development and Distribution License ("CDDL"), version 1.0.
6# You may only use this file in accordance with the terms of version
7# 1.0 of the CDDL.
8#
9# A full copy of the text of the CDDL should have accompanied this
10# source.  A copy of the CDDL is also available via the Internet at
11# http://www.illumos.org/license/CDDL.
12#
13
14#
15# Copyright (c) 2012, 2018 by Delphix. All rights reserved.
16# Copyright (c) 2019 Datto Inc.
17#
18# This script must remain compatible with Python 2.6+ and Python 3.4+.
19#
20
21# some python 2.7 system don't have a configparser shim
22try:
23    import configparser
24except ImportError:
25    import ConfigParser as configparser
26
27import os
28import sys
29import ctypes
30
31from datetime import datetime
32from optparse import OptionParser
33from pwd import getpwnam
34from pwd import getpwuid
35from select import select
36from subprocess import PIPE
37from subprocess import Popen
38from threading import Timer
39from time import time
40
41BASEDIR = '/var/tmp/test_results'
42TESTDIR = '/usr/share/zfs/'
43KILL = 'kill'
44TRUE = 'true'
45SUDO = 'sudo'
46LOG_FILE = 'LOG_FILE'
47LOG_OUT = 'LOG_OUT'
48LOG_ERR = 'LOG_ERR'
49LOG_FILE_OBJ = None
50
51# some python 2.7 system don't have a concept of monotonic time
52CLOCK_MONOTONIC_RAW = 4  # see <linux/time.h>
53
54
55class timespec(ctypes.Structure):
56    _fields_ = [
57        ('tv_sec', ctypes.c_long),
58        ('tv_nsec', ctypes.c_long)
59    ]
60
61
62librt = ctypes.CDLL('librt.so.1', use_errno=True)
63clock_gettime = librt.clock_gettime
64clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)]
65
66
67def monotonic_time():
68    t = timespec()
69    if clock_gettime(CLOCK_MONOTONIC_RAW, ctypes.pointer(t)) != 0:
70        errno_ = ctypes.get_errno()
71        raise OSError(errno_, os.strerror(errno_))
72    return t.tv_sec + t.tv_nsec * 1e-9
73
74
75class Result(object):
76    total = 0
77    runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0}
78
79    def __init__(self):
80        self.starttime = None
81        self.returncode = None
82        self.runtime = ''
83        self.stdout = []
84        self.stderr = []
85        self.result = ''
86
87    def done(self, proc, killed, reran):
88        """
89        Finalize the results of this Cmd.
90        """
91        Result.total += 1
92        m, s = divmod(monotonic_time() - self.starttime, 60)
93        self.runtime = '%02d:%02d' % (m, s)
94        self.returncode = proc.returncode
95        if reran is True:
96            Result.runresults['RERAN'] += 1
97        if killed:
98            self.result = 'KILLED'
99            Result.runresults['KILLED'] += 1
100        elif self.returncode == 0:
101            self.result = 'PASS'
102            Result.runresults['PASS'] += 1
103        elif self.returncode == 4:
104            self.result = 'SKIP'
105            Result.runresults['SKIP'] += 1
106        elif self.returncode != 0:
107            self.result = 'FAIL'
108            Result.runresults['FAIL'] += 1
109
110
111class Output(object):
112    """
113    This class is a slightly modified version of the 'Stream' class found
114    here: http://goo.gl/aSGfv
115    """
116    def __init__(self, stream):
117        self.stream = stream
118        self._buf = b''
119        self.lines = []
120
121    def fileno(self):
122        return self.stream.fileno()
123
124    def read(self, drain=0):
125        """
126        Read from the file descriptor. If 'drain' set, read until EOF.
127        """
128        while self._read() is not None:
129            if not drain:
130                break
131
132    def _read(self):
133        """
134        Read up to 4k of data from this output stream. Collect the output
135        up to the last newline, and append it to any leftover data from a
136        previous call. The lines are stored as a (timestamp, data) tuple
137        for easy sorting/merging later.
138        """
139        fd = self.fileno()
140        buf = os.read(fd, 4096)
141        if not buf:
142            return None
143        if b'\n' not in buf:
144            self._buf += buf
145            return []
146
147        buf = self._buf + buf
148        tmp, rest = buf.rsplit(b'\n', 1)
149        self._buf = rest
150        now = datetime.now()
151        rows = tmp.split(b'\n')
152        self.lines += [(now, r) for r in rows]
153
154
155class Cmd(object):
156    verified_users = []
157
158    def __init__(self, pathname, identifier=None, outputdir=None,
159                 timeout=None, user=None, tags=None):
160        self.pathname = pathname
161        self.identifier = identifier
162        self.outputdir = outputdir or 'BASEDIR'
163        """
164        The timeout for tests is measured in wall-clock time
165        """
166        self.timeout = timeout
167        self.user = user or ''
168        self.killed = False
169        self.reran = None
170        self.result = Result()
171
172        if self.timeout is None:
173            self.timeout = 60
174
175    def __str__(self):
176        return '''\
177Pathname: %s
178Identifier: %s
179Outputdir: %s
180Timeout: %d
181User: %s
182''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
183
184    def kill_cmd(self, proc, keyboard_interrupt=False):
185        """
186        Kill a running command due to timeout, or ^C from the keyboard. If
187        sudo is required, this user was verified previously.
188        """
189        self.killed = True
190        do_sudo = len(self.user) != 0
191        signal = '-TERM'
192
193        cmd = [SUDO, KILL, signal, str(proc.pid)]
194        if not do_sudo:
195            del cmd[0]
196
197        try:
198            kp = Popen(cmd)
199            kp.wait()
200        except Exception:
201            pass
202
203        """
204        If this is not a user-initiated kill and the test has not been
205        reran before we consider if the test needs to be reran:
206        If the test has spent some time hibernating and didn't run the whole
207        length of time before being timed out we will rerun the test.
208        """
209        if keyboard_interrupt is False and self.reran is None:
210            runtime = monotonic_time() - self.result.starttime
211            if int(self.timeout) > runtime:
212                self.killed = False
213                self.reran = False
214                self.run(False)
215                self.reran = True
216
217    def update_cmd_privs(self, cmd, user):
218        """
219        If a user has been specified to run this Cmd and we're not already
220        running as that user, prepend the appropriate sudo command to run
221        as that user.
222        """
223        me = getpwuid(os.getuid())
224
225        if not user or user is me:
226            if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
227                cmd += '.ksh'
228            if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
229                cmd += '.sh'
230            return cmd
231
232        if not os.path.isfile(cmd):
233            if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
234                cmd += '.ksh'
235            if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
236                cmd += '.sh'
237
238        ret = '%s -E -u %s %s' % (SUDO, user, cmd)
239        return ret.split(' ')
240
241    def collect_output(self, proc):
242        """
243        Read from stdout/stderr as data becomes available, until the
244        process is no longer running. Return the lines from the stdout and
245        stderr Output objects.
246        """
247        out = Output(proc.stdout)
248        err = Output(proc.stderr)
249        res = []
250        while proc.returncode is None:
251            proc.poll()
252            res = select([out, err], [], [], .1)
253            for fd in res[0]:
254                fd.read()
255        for fd in res[0]:
256            fd.read(drain=1)
257
258        return out.lines, err.lines
259
260    def run(self, dryrun):
261        """
262        This is the main function that runs each individual test.
263        Determine whether or not the command requires sudo, and modify it
264        if needed. Run the command, and update the result object.
265        """
266        if dryrun is True:
267            print(self)
268            return
269
270        privcmd = self.update_cmd_privs(self.pathname, self.user)
271        try:
272            old = os.umask(0)
273            if not os.path.isdir(self.outputdir):
274                os.makedirs(self.outputdir, mode=0o777)
275            os.umask(old)
276        except OSError as e:
277            fail('%s' % e)
278
279        self.result.starttime = monotonic_time()
280        proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
281        # Allow a special timeout value of 0 to mean infinity
282        if int(self.timeout) == 0:
283            self.timeout = sys.maxsize
284        t = Timer(int(self.timeout), self.kill_cmd, [proc])
285
286        try:
287            t.start()
288            self.result.stdout, self.result.stderr = self.collect_output(proc)
289        except KeyboardInterrupt:
290            self.kill_cmd(proc, True)
291            fail('\nRun terminated at user request.')
292        finally:
293            t.cancel()
294
295        if self.reran is not False:
296            self.result.done(proc, self.killed, self.reran)
297
298    def skip(self):
299        """
300        Initialize enough of the test result that we can log a skipped
301        command.
302        """
303        Result.total += 1
304        Result.runresults['SKIP'] += 1
305        self.result.stdout = self.result.stderr = []
306        self.result.starttime = monotonic_time()
307        m, s = divmod(monotonic_time() - self.result.starttime, 60)
308        self.result.runtime = '%02d:%02d' % (m, s)
309        self.result.result = 'SKIP'
310
311    def log(self, options, suppress_console=False):
312        """
313        This function is responsible for writing all output. This includes
314        the console output, the logfile of all results (with timestamped
315        merged stdout and stderr), and for each test, the unmodified
316        stdout/stderr/merged in its own file.
317        """
318
319        logname = getpwuid(os.getuid()).pw_name
320        rer = ''
321        if self.reran is True:
322            rer = ' (RERAN)'
323        user = ' (run as %s)' % (self.user if len(self.user) else logname)
324        if self.identifier:
325            msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
326        else:
327            msga = 'Test: %s%s ' % (self.pathname, user)
328        msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer)
329        pad = ' ' * (80 - (len(msga) + len(msgb)))
330        result_line = msga + pad + msgb
331
332        # The result line is always written to the log file. If -q was
333        # specified only failures are written to the console, otherwise
334        # the result line is written to the console. The console output
335        # may be suppressed by calling log() with suppress_console=True.
336        write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
337        if not suppress_console:
338            if not options.quiet:
339                write_log(result_line, LOG_OUT)
340            elif options.quiet and self.result.result != 'PASS':
341                write_log(result_line, LOG_OUT)
342
343        lines = sorted(self.result.stdout + self.result.stderr,
344                       key=lambda x: x[0])
345
346        # Write timestamped output (stdout and stderr) to the logfile
347        for dt, line in lines:
348            timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
349                                  encoding='utf-8')
350            write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
351
352        # Write the separate stdout/stderr/merged files, if the data exists
353        if len(self.result.stdout):
354            with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
355                for _, line in self.result.stdout:
356                    os.write(out.fileno(), b'%s\n' % line)
357        if len(self.result.stderr):
358            with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
359                for _, line in self.result.stderr:
360                    os.write(err.fileno(), b'%s\n' % line)
361        if len(self.result.stdout) and len(self.result.stderr):
362            with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
363                for _, line in lines:
364                    os.write(merged.fileno(), b'%s\n' % line)
365
366
367class Test(Cmd):
368    props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
369             'post_user', 'failsafe', 'failsafe_user', 'tags']
370
371    def __init__(self, pathname,
372                 pre=None, pre_user=None, post=None, post_user=None,
373                 failsafe=None, failsafe_user=None, tags=None, **kwargs):
374        super(Test, self).__init__(pathname, **kwargs)
375        self.pre = pre or ''
376        self.pre_user = pre_user or ''
377        self.post = post or ''
378        self.post_user = post_user or ''
379        self.failsafe = failsafe or ''
380        self.failsafe_user = failsafe_user or ''
381        self.tags = tags or []
382
383    def __str__(self):
384        post_user = pre_user = failsafe_user = ''
385        if len(self.pre_user):
386            pre_user = ' (as %s)' % (self.pre_user)
387        if len(self.post_user):
388            post_user = ' (as %s)' % (self.post_user)
389        if len(self.failsafe_user):
390            failsafe_user = ' (as %s)' % (self.failsafe_user)
391        return '''\
392Pathname: %s
393Identifier: %s
394Outputdir: %s
395Timeout: %d
396User: %s
397Pre: %s%s
398Post: %s%s
399Failsafe: %s%s
400Tags: %s
401''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
402            self.pre, pre_user, self.post, post_user, self.failsafe,
403            failsafe_user, self.tags)
404
405    def verify(self):
406        """
407        Check the pre/post/failsafe scripts, user and Test. Omit the Test from
408        this run if there are any problems.
409        """
410        files = [self.pre, self.pathname, self.post, self.failsafe]
411        users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
412
413        for f in [f for f in files if len(f)]:
414            if not verify_file(f):
415                write_log("Warning: Test '%s' not added to this run because"
416                          " it failed verification.\n" % f, LOG_ERR)
417                return False
418
419        for user in [user for user in users if len(user)]:
420            if not verify_user(user):
421                write_log("Not adding Test '%s' to this run.\n" %
422                          self.pathname, LOG_ERR)
423                return False
424
425        return True
426
427    def run(self, options):
428        """
429        Create Cmd instances for the pre/post/failsafe scripts. If the pre
430        script doesn't pass, skip this Test. Run the post script regardless.
431        If the Test is killed, also run the failsafe script.
432        """
433        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
434        pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
435                      timeout=self.timeout, user=self.pre_user)
436        test = Cmd(self.pathname, identifier=self.identifier,
437                   outputdir=self.outputdir, timeout=self.timeout,
438                   user=self.user)
439        odir = os.path.join(self.outputdir, os.path.basename(self.failsafe))
440        failsafe = Cmd(self.failsafe, identifier=self.identifier,
441                       outputdir=odir, timeout=self.timeout,
442                       user=self.failsafe_user)
443        odir = os.path.join(self.outputdir, os.path.basename(self.post))
444        posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
445                       timeout=self.timeout, user=self.post_user)
446
447        cont = True
448        if len(pretest.pathname):
449            pretest.run(options.dryrun)
450            cont = pretest.result.result == 'PASS'
451            pretest.log(options)
452
453        if cont:
454            test.run(options.dryrun)
455            if test.result.result == 'KILLED' and len(failsafe.pathname):
456                failsafe.run(options.dryrun)
457                failsafe.log(options, suppress_console=True)
458        else:
459            test.skip()
460
461        test.log(options)
462
463        if len(posttest.pathname):
464            posttest.run(options.dryrun)
465            posttest.log(options)
466
467
468class TestGroup(Test):
469    props = Test.props + ['tests']
470
471    def __init__(self, pathname, tests=None, **kwargs):
472        super(TestGroup, self).__init__(pathname, **kwargs)
473        self.tests = tests or []
474
475    def __str__(self):
476        post_user = pre_user = failsafe_user = ''
477        if len(self.pre_user):
478            pre_user = ' (as %s)' % (self.pre_user)
479        if len(self.post_user):
480            post_user = ' (as %s)' % (self.post_user)
481        if len(self.failsafe_user):
482            failsafe_user = ' (as %s)' % (self.failsafe_user)
483        return '''\
484Pathname: %s
485Identifier: %s
486Outputdir: %s
487Tests: %s
488Timeout: %s
489User: %s
490Pre: %s%s
491Post: %s%s
492Failsafe: %s%s
493Tags: %s
494''' % (self.pathname, self.identifier, self.outputdir, self.tests,
495            self.timeout, self.user, self.pre, pre_user, self.post, post_user,
496            self.failsafe, failsafe_user, self.tags)
497
498    def verify(self):
499        """
500        Check the pre/post/failsafe scripts, user and tests in this TestGroup.
501        Omit the TestGroup entirely, or simply delete the relevant tests in the
502        group, if that's all that's required.
503        """
504        # If the pre/post/failsafe scripts are relative pathnames, convert to
505        # absolute, so they stand a chance of passing verification.
506        if len(self.pre) and not os.path.isabs(self.pre):
507            self.pre = os.path.join(self.pathname, self.pre)
508        if len(self.post) and not os.path.isabs(self.post):
509            self.post = os.path.join(self.pathname, self.post)
510        if len(self.failsafe) and not os.path.isabs(self.failsafe):
511            self.post = os.path.join(self.pathname, self.post)
512
513        auxfiles = [self.pre, self.post, self.failsafe]
514        users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
515
516        for f in [f for f in auxfiles if len(f)]:
517            if f != self.failsafe and self.pathname != os.path.dirname(f):
518                write_log("Warning: TestGroup '%s' not added to this run. "
519                          "Auxiliary script '%s' exists in a different "
520                          "directory.\n" % (self.pathname, f), LOG_ERR)
521                return False
522
523            if not verify_file(f):
524                write_log("Warning: TestGroup '%s' not added to this run. "
525                          "Auxiliary script '%s' failed verification.\n" %
526                          (self.pathname, f), LOG_ERR)
527                return False
528
529        for user in [user for user in users if len(user)]:
530            if not verify_user(user):
531                write_log("Not adding TestGroup '%s' to this run.\n" %
532                          self.pathname, LOG_ERR)
533                return False
534
535        # If one of the tests is invalid, delete it, log it, and drive on.
536        for test in self.tests:
537            if not verify_file(os.path.join(self.pathname, test)):
538                del self.tests[self.tests.index(test)]
539                write_log("Warning: Test '%s' removed from TestGroup '%s' "
540                          "because it failed verification.\n" %
541                          (test, self.pathname), LOG_ERR)
542
543        return len(self.tests) != 0
544
545    def run(self, options):
546        """
547        Create Cmd instances for the pre/post/failsafe scripts. If the pre
548        script doesn't pass, skip all the tests in this TestGroup. Run the
549        post script regardless. Run the failsafe script when a test is killed.
550        """
551        # tags assigned to this test group also include the test names
552        if options.tags and not set(self.tags).intersection(set(options.tags)):
553            return
554
555        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
556        pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
557                      user=self.pre_user, identifier=self.identifier)
558        odir = os.path.join(self.outputdir, os.path.basename(self.post))
559        posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
560                       user=self.post_user, identifier=self.identifier)
561
562        cont = True
563        if len(pretest.pathname):
564            pretest.run(options.dryrun)
565            cont = pretest.result.result == 'PASS'
566            pretest.log(options)
567
568        for fname in self.tests:
569            odir = os.path.join(self.outputdir, fname)
570            test = Cmd(os.path.join(self.pathname, fname), outputdir=odir,
571                       timeout=self.timeout, user=self.user,
572                       identifier=self.identifier)
573            odir = os.path.join(odir, os.path.basename(self.failsafe))
574            failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
575                           user=self.failsafe_user, identifier=self.identifier)
576            if cont:
577                test.run(options.dryrun)
578                if test.result.result == 'KILLED' and len(failsafe.pathname):
579                    failsafe.run(options.dryrun)
580                    failsafe.log(options, suppress_console=True)
581            else:
582                test.skip()
583
584            test.log(options)
585
586        if len(posttest.pathname):
587            posttest.run(options.dryrun)
588            posttest.log(options)
589
590
591class TestRun(object):
592    props = ['quiet', 'outputdir']
593
594    def __init__(self, options):
595        self.tests = {}
596        self.testgroups = {}
597        self.starttime = time()
598        self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
599        self.outputdir = os.path.join(options.outputdir, self.timestamp)
600        self.setup_logging(options)
601        self.defaults = [
602            ('outputdir', BASEDIR),
603            ('quiet', False),
604            ('timeout', 60),
605            ('user', ''),
606            ('pre', ''),
607            ('pre_user', ''),
608            ('post', ''),
609            ('post_user', ''),
610            ('failsafe', ''),
611            ('failsafe_user', ''),
612            ('tags', [])
613        ]
614
615    def __str__(self):
616        s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
617        s += 'TESTS:\n'
618        for key in sorted(self.tests.keys()):
619            s += '%s%s' % (self.tests[key].__str__(), '\n')
620        s += 'TESTGROUPS:\n'
621        for key in sorted(self.testgroups.keys()):
622            s += '%s%s' % (self.testgroups[key].__str__(), '\n')
623        return s
624
625    def addtest(self, pathname, options):
626        """
627        Create a new Test, and apply any properties that were passed in
628        from the command line. If it passes verification, add it to the
629        TestRun.
630        """
631        test = Test(pathname)
632        for prop in Test.props:
633            setattr(test, prop, getattr(options, prop))
634
635        if test.verify():
636            self.tests[pathname] = test
637
638    def addtestgroup(self, dirname, filenames, options):
639        """
640        Create a new TestGroup, and apply any properties that were passed
641        in from the command line. If it passes verification, add it to the
642        TestRun.
643        """
644        if dirname not in self.testgroups:
645            testgroup = TestGroup(dirname)
646            for prop in Test.props:
647                setattr(testgroup, prop, getattr(options, prop))
648
649            # Prevent pre/post/failsafe scripts from running as regular tests
650            for f in [testgroup.pre, testgroup.post, testgroup.failsafe]:
651                if f in filenames:
652                    del filenames[filenames.index(f)]
653
654            self.testgroups[dirname] = testgroup
655            self.testgroups[dirname].tests = sorted(filenames)
656
657            testgroup.verify()
658
659    def read(self, options):
660        """
661        Read in the specified runfiles, and apply the TestRun properties
662        listed in the 'DEFAULT' section to our TestRun. Then read each
663        section, and apply the appropriate properties to the Test or
664        TestGroup. Properties from individual sections override those set
665        in the 'DEFAULT' section. If the Test or TestGroup passes
666        verification, add it to the TestRun.
667        """
668        config = configparser.RawConfigParser()
669        parsed = config.read(options.runfiles)
670        failed = options.runfiles - set(parsed)
671        if len(failed):
672            files = ' '.join(sorted(failed))
673            fail("Couldn't read config files: %s" % files)
674
675        for opt in TestRun.props:
676            if config.has_option('DEFAULT', opt):
677                setattr(self, opt, config.get('DEFAULT', opt))
678        self.outputdir = os.path.join(self.outputdir, self.timestamp)
679
680        testdir = options.testdir
681
682        for section in config.sections():
683            if 'tests' in config.options(section):
684                parts = section.split(':', 1)
685                sectiondir = parts[0]
686                identifier = parts[1] if len(parts) == 2 else None
687                if os.path.isdir(sectiondir):
688                    pathname = sectiondir
689                elif os.path.isdir(os.path.join(testdir, sectiondir)):
690                    pathname = os.path.join(testdir, sectiondir)
691                else:
692                    pathname = sectiondir
693
694                testgroup = TestGroup(os.path.abspath(pathname),
695                                      identifier=identifier)
696                for prop in TestGroup.props:
697                    for sect in ['DEFAULT', section]:
698                        if config.has_option(sect, prop):
699                            if prop == 'tags':
700                                setattr(testgroup, prop,
701                                        eval(config.get(sect, prop)))
702                            elif prop == 'failsafe':
703                                failsafe = config.get(sect, prop)
704                                setattr(testgroup, prop,
705                                        os.path.join(testdir, failsafe))
706                            else:
707                                setattr(testgroup, prop,
708                                        config.get(sect, prop))
709
710                # Repopulate tests using eval to convert the string to a list
711                testgroup.tests = eval(config.get(section, 'tests'))
712
713                if testgroup.verify():
714                    self.testgroups[section] = testgroup
715            else:
716                test = Test(section)
717                for prop in Test.props:
718                    for sect in ['DEFAULT', section]:
719                        if config.has_option(sect, prop):
720                            if prop == 'failsafe':
721                                failsafe = config.get(sect, prop)
722                                setattr(test, prop,
723                                        os.path.join(testdir, failsafe))
724                            else:
725                                setattr(test, prop, config.get(sect, prop))
726
727                if test.verify():
728                    self.tests[section] = test
729
730    def write(self, options):
731        """
732        Create a configuration file for editing and later use. The
733        'DEFAULT' section of the config file is created from the
734        properties that were specified on the command line. Tests are
735        simply added as sections that inherit everything from the
736        'DEFAULT' section. TestGroups are the same, except they get an
737        option including all the tests to run in that directory.
738        """
739
740        defaults = dict([(prop, getattr(options, prop)) for prop, _ in
741                         self.defaults])
742        config = configparser.RawConfigParser(defaults)
743
744        for test in sorted(self.tests.keys()):
745            config.add_section(test)
746
747        for testgroup in sorted(self.testgroups.keys()):
748            config.add_section(testgroup)
749            config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
750
751        try:
752            with open(options.template, 'w') as f:
753                return config.write(f)
754        except IOError:
755            fail('Could not open \'%s\' for writing.' % options.template)
756
757    def complete_outputdirs(self):
758        """
759        Collect all the pathnames for Tests, and TestGroups. Work
760        backwards one pathname component at a time, to create a unique
761        directory name in which to deposit test output. Tests will be able
762        to write output files directly in the newly modified outputdir.
763        TestGroups will be able to create one subdirectory per test in the
764        outputdir, and are guaranteed uniqueness because a group can only
765        contain files in one directory. Pre and post tests will create a
766        directory rooted at the outputdir of the Test or TestGroup in
767        question for their output. Failsafe scripts will create a directory
768        rooted at the outputdir of each Test for their output.
769        """
770        done = False
771        components = 0
772        tmp_dict = dict(list(self.tests.items()) +
773                        list(self.testgroups.items()))
774        total = len(tmp_dict)
775        base = self.outputdir
776
777        while not done:
778            paths = []
779            components -= 1
780            for testfile in list(tmp_dict.keys()):
781                uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
782                if uniq not in paths:
783                    paths.append(uniq)
784                    tmp_dict[testfile].outputdir = os.path.join(base, uniq)
785                else:
786                    break
787            done = total == len(paths)
788
789    def setup_logging(self, options):
790        """
791        This function creates the output directory and gets a file object
792        for the logfile. This function must be called before write_log()
793        can be used.
794        """
795        if options.dryrun is True:
796            return
797
798        global LOG_FILE_OBJ
799        if options.cmd != 'wrconfig':
800            try:
801                old = os.umask(0)
802                os.makedirs(self.outputdir, mode=0o777)
803                os.umask(old)
804                filename = os.path.join(self.outputdir, 'log')
805                LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
806            except OSError as e:
807                fail('%s' % e)
808
809    def run(self, options):
810        """
811        Walk through all the Tests and TestGroups, calling run().
812        """
813        try:
814            os.chdir(self.outputdir)
815        except OSError:
816            fail('Could not change to directory %s' % self.outputdir)
817        # make a symlink to the output for the currently running test
818        logsymlink = os.path.join(self.outputdir, '../current')
819        if os.path.islink(logsymlink):
820            os.unlink(logsymlink)
821        if not os.path.exists(logsymlink):
822            os.symlink(self.outputdir, logsymlink)
823        else:
824            write_log('Could not make a symlink to directory %s\n' %
825                      self.outputdir, LOG_ERR)
826        iteration = 0
827        while iteration < options.iterations:
828            for test in sorted(self.tests.keys()):
829                self.tests[test].run(options)
830            for testgroup in sorted(self.testgroups.keys()):
831                self.testgroups[testgroup].run(options)
832            iteration += 1
833
834    def summary(self):
835        if Result.total == 0:
836            return 2
837
838        print('\nResults Summary')
839        for key in list(Result.runresults.keys()):
840            if Result.runresults[key] != 0:
841                print('%s\t% 4d' % (key, Result.runresults[key]))
842
843        m, s = divmod(time() - self.starttime, 60)
844        h, m = divmod(m, 60)
845        print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
846        print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
847                                            float(Result.total)) * 100))
848        print('Log directory:\t%s' % self.outputdir)
849
850        if Result.runresults['FAIL'] > 0:
851            return 1
852
853        if Result.runresults['KILLED'] > 0:
854            return 1
855
856        if Result.runresults['RERAN'] > 0:
857            return 3
858
859        return 0
860
861
862def write_log(msg, target):
863    """
864    Write the provided message to standard out, standard error or
865    the logfile. If specifying LOG_FILE, then `msg` must be a bytes
866    like object. This way we can still handle output from tests that
867    may be in unexpected encodings.
868    """
869    if target == LOG_OUT:
870        os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
871    elif target == LOG_ERR:
872        os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
873    elif target == LOG_FILE:
874        os.write(LOG_FILE_OBJ.fileno(), msg)
875    else:
876        fail('log_msg called with unknown target "%s"' % target)
877
878
879def verify_file(pathname):
880    """
881    Verify that the supplied pathname is an executable regular file.
882    """
883    if os.path.isdir(pathname) or os.path.islink(pathname):
884        return False
885
886    for ext in '', '.ksh', '.sh':
887        script_path = pathname + ext
888        if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
889            return True
890
891    return False
892
893
894def verify_user(user):
895    """
896    Verify that the specified user exists on this system, and can execute
897    sudo without being prompted for a password.
898    """
899    testcmd = [SUDO, '-n', '-u', user, TRUE]
900
901    if user in Cmd.verified_users:
902        return True
903
904    try:
905        getpwnam(user)
906    except KeyError:
907        write_log("Warning: user '%s' does not exist.\n" % user,
908                  LOG_ERR)
909        return False
910
911    p = Popen(testcmd)
912    p.wait()
913    if p.returncode != 0:
914        write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
915                  LOG_ERR)
916        return False
917    else:
918        Cmd.verified_users.append(user)
919
920    return True
921
922
923def find_tests(testrun, options):
924    """
925    For the given list of pathnames, add files as Tests. For directories,
926    if do_groups is True, add the directory as a TestGroup. If False,
927    recursively search for executable files.
928    """
929
930    for p in sorted(options.pathnames):
931        if os.path.isdir(p):
932            for dirname, _, filenames in os.walk(p):
933                if options.do_groups:
934                    testrun.addtestgroup(dirname, filenames, options)
935                else:
936                    for f in sorted(filenames):
937                        testrun.addtest(os.path.join(dirname, f), options)
938        else:
939            testrun.addtest(p, options)
940
941
942def fail(retstr, ret=1):
943    print('%s: %s' % (sys.argv[0], retstr))
944    exit(ret)
945
946
947def options_cb(option, opt_str, value, parser):
948    path_options = ['outputdir', 'template', 'testdir']
949
950    if option.dest == 'runfiles' and '-w' in parser.rargs or \
951            option.dest == 'template' and '-c' in parser.rargs:
952        fail('-c and -w are mutually exclusive.')
953
954    if opt_str in parser.rargs:
955        fail('%s may only be specified once.' % opt_str)
956
957    if option.dest == 'runfiles':
958        parser.values.cmd = 'rdconfig'
959        value = set(os.path.abspath(p) for p in value.split(','))
960    if option.dest == 'template':
961        parser.values.cmd = 'wrconfig'
962    if option.dest == 'tags':
963        value = [x.strip() for x in value.split(',')]
964
965    if option.dest in path_options:
966        setattr(parser.values, option.dest, os.path.abspath(value))
967    else:
968        setattr(parser.values, option.dest, value)
969
970
971def parse_args():
972    parser = OptionParser()
973    parser.add_option('-c', action='callback', callback=options_cb,
974                      type='string', dest='runfiles', metavar='runfiles',
975                      help='Specify tests to run via config files.')
976    parser.add_option('-d', action='store_true', default=False, dest='dryrun',
977                      help='Dry run. Print tests, but take no other action.')
978    parser.add_option('-g', action='store_true', default=False,
979                      dest='do_groups', help='Make directories TestGroups.')
980    parser.add_option('-o', action='callback', callback=options_cb,
981                      default=BASEDIR, dest='outputdir', type='string',
982                      metavar='outputdir', help='Specify an output directory.')
983    parser.add_option('-i', action='callback', callback=options_cb,
984                      default=TESTDIR, dest='testdir', type='string',
985                      metavar='testdir', help='Specify a test directory.')
986    parser.add_option('-p', action='callback', callback=options_cb,
987                      default='', dest='pre', metavar='script',
988                      type='string', help='Specify a pre script.')
989    parser.add_option('-P', action='callback', callback=options_cb,
990                      default='', dest='post', metavar='script',
991                      type='string', help='Specify a post script.')
992    parser.add_option('-q', action='store_true', default=False, dest='quiet',
993                      help='Silence on the console during a test run.')
994    parser.add_option('-s', action='callback', callback=options_cb,
995                      default='', dest='failsafe', metavar='script',
996                      type='string', help='Specify a failsafe script.')
997    parser.add_option('-S', action='callback', callback=options_cb,
998                      default='', dest='failsafe_user',
999                      metavar='failsafe_user', type='string',
1000                      help='Specify a user to execute the failsafe script.')
1001    parser.add_option('-t', action='callback', callback=options_cb, default=60,
1002                      dest='timeout', metavar='seconds', type='int',
1003                      help='Timeout (in seconds) for an individual test.')
1004    parser.add_option('-u', action='callback', callback=options_cb,
1005                      default='', dest='user', metavar='user', type='string',
1006                      help='Specify a different user name to run as.')
1007    parser.add_option('-w', action='callback', callback=options_cb,
1008                      default=None, dest='template', metavar='template',
1009                      type='string', help='Create a new config file.')
1010    parser.add_option('-x', action='callback', callback=options_cb, default='',
1011                      dest='pre_user', metavar='pre_user', type='string',
1012                      help='Specify a user to execute the pre script.')
1013    parser.add_option('-X', action='callback', callback=options_cb, default='',
1014                      dest='post_user', metavar='post_user', type='string',
1015                      help='Specify a user to execute the post script.')
1016    parser.add_option('-T', action='callback', callback=options_cb, default='',
1017                      dest='tags', metavar='tags', type='string',
1018                      help='Specify tags to execute specific test groups.')
1019    parser.add_option('-I', action='callback', callback=options_cb, default=1,
1020                      dest='iterations', metavar='iterations', type='int',
1021                      help='Number of times to run the test run.')
1022    (options, pathnames) = parser.parse_args()
1023
1024    if not options.runfiles and not options.template:
1025        options.cmd = 'runtests'
1026
1027    if options.runfiles and len(pathnames):
1028        fail('Extraneous arguments.')
1029
1030    options.pathnames = [os.path.abspath(path) for path in pathnames]
1031
1032    return options
1033
1034
1035def main():
1036    options = parse_args()
1037    testrun = TestRun(options)
1038
1039    if options.cmd == 'runtests':
1040        find_tests(testrun, options)
1041    elif options.cmd == 'rdconfig':
1042        testrun.read(options)
1043    elif options.cmd == 'wrconfig':
1044        find_tests(testrun, options)
1045        testrun.write(options)
1046        exit(0)
1047    else:
1048        fail('Unknown command specified')
1049
1050    testrun.complete_outputdirs()
1051    testrun.run(options)
1052    exit(testrun.summary())
1053
1054
1055if __name__ == '__main__':
1056    main()
1057