xref: /illumos-gate/usr/src/test/test-runner/cmd/run (revision d9c3e05c2d8261e3f133b5e96a300b4fa6c0f1b7)
1#!@PYTHON@
2
3#
4# This file and its contents are supplied under the terms of the
5# Common Development and Distribution License ("CDDL"), version 1.0.
6# You may only use this file in accordance with the terms of version
7# 1.0 of the CDDL.
8#
9# A full copy of the text of the CDDL should have accompanied this
10# source.  A copy of the CDDL is also available via the Internet at
11# http://www.illumos.org/license/CDDL.
12#
13
14#
15# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
16# Copyright (c) 2017, Chris Fraire <cfraire@me.com>.
17# Copyright 2019 Joyent, Inc.
18# Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
19#
20
21from __future__ import print_function
22import sys
23PY3 = sys.version_info[0] == 3
24
25if PY3:
26    import configparser
27else:
28    import ConfigParser as configparser
29
30import io
31import os
32import logging
33import platform
34import re
35from logging.handlers import WatchedFileHandler
36from datetime import datetime
37from optparse import OptionParser
38from pwd import getpwnam
39from pwd import getpwuid
40from select import select
41from subprocess import PIPE
42from subprocess import Popen
43from sys import argv
44from sys import exit
45from sys import maxsize
46from threading import Timer
47from time import time
48
49BASEDIR = '/var/tmp/test_results'
50KILL = '/usr/bin/kill'
51TRUE = '/usr/bin/true'
52SUDO = '/usr/bin/sudo'
53
54retcode = 0
55
56# Custom class to reopen the log file in case it is forcibly closed by a test.
57class WatchedFileHandlerClosed(WatchedFileHandler):
58    """Watch files, including closed files.
59    Similar to (and inherits from) logging.handler.WatchedFileHandler,
60    except that IOErrors are handled by reopening the stream and retrying.
61    This will be retried up to a configurable number of times before
62    giving up, default 5.
63    """
64
65    def __init__(self, filename, mode='a', encoding='utf-8', delay=0, max_tries=5):
66        self.max_tries = max_tries
67        self.tries = 0
68        WatchedFileHandler.__init__(self, filename, mode, encoding, delay)
69
70    def emit(self, record):
71        while True:
72            try:
73                WatchedFileHandler.emit(self, record)
74                self.tries = 0
75                return
76            except IOError as err:
77                if self.tries == self.max_tries:
78                    raise
79                self.stream.close()
80                self.stream = self._open()
81                self.tries += 1
82
83class Result(object):
84    total = 0
85    runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
86
87    def __init__(self):
88        self.starttime = None
89        self.returncode = None
90        self.runtime = ''
91        self.stdout = []
92        self.stderr = []
93        self.result = ''
94
95    def done(self, proc, killed):
96        """
97        Finalize the results of this Cmd.
98        Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED)
99        as defined in ../stf/include/stf.shlib
100        """
101        global retcode
102
103        Result.total += 1
104        m, s = divmod(time() - self.starttime, 60)
105        self.runtime = '%02d:%02d' % (m, s)
106        self.returncode = proc.returncode
107        if killed:
108            self.result = 'KILLED'
109            Result.runresults['KILLED'] += 1
110            retcode = 2;
111        elif self.returncode == 0:
112            self.result = 'PASS'
113            Result.runresults['PASS'] += 1
114        elif self.returncode == 3 or self.returncode == 4:
115            self.result = 'SKIP'
116            Result.runresults['SKIP'] += 1
117        elif self.returncode != 0:
118            self.result = 'FAIL'
119            Result.runresults['FAIL'] += 1
120            retcode = 1;
121
122
123class Output(object):
124    """
125    This class is a slightly modified version of the 'Stream' class found
126    here: http://goo.gl/aSGfv
127    """
128    def __init__(self, stream):
129        self.stream = stream
130        self._buf = ''
131        self.lines = []
132
133    def fileno(self):
134        return self.stream.fileno()
135
136    def read(self, drain=0):
137        """
138        Read from the file descriptor. If 'drain' set, read until EOF.
139        """
140        while self._read() is not None:
141            if not drain:
142                break
143
144    def _read(self):
145        """
146        Read up to 4k of data from this output stream. Collect the output
147        up to the last newline, and append it to any leftover data from a
148        previous call. The lines are stored as a (timestamp, data) tuple
149        for easy sorting/merging later.
150        """
151        fd = self.fileno()
152        buf = os.read(fd, 4096).decode('utf-8', errors='ignore')
153        if not buf:
154            return None
155        if '\n' not in buf:
156            self._buf += buf
157            return []
158
159        buf = self._buf + buf
160        tmp, rest = buf.rsplit('\n', 1)
161        self._buf = rest
162        now = datetime.now()
163        rows = tmp.split('\n')
164        self.lines += [(now, r) for r in rows]
165
166
167class Cmd(object):
168    verified_users = []
169
170    def __init__(self, pathname, outputdir=None, timeout=None, user=None):
171        self.pathname = pathname
172        self.outputdir = outputdir or 'BASEDIR'
173        self.timeout = timeout
174        self.user = user or ''
175        self.killed = False
176        self.result = Result()
177
178        if self.timeout is None:
179            self.timeout = 60
180
181    def __str__(self):
182        return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nUser: %s\n" % \
183            (self.pathname, self.outputdir, self.timeout, self.user)
184
185    def kill_cmd(self, proc):
186        """
187        Kill a running command due to timeout, or ^C from the keyboard. If
188        sudo is required, this user was verified previously.
189        """
190        self.killed = True
191        do_sudo = len(self.user) != 0
192        signal = '-TERM'
193
194        cmd = [SUDO, KILL, signal, str(proc.pid)]
195        if not do_sudo:
196            del cmd[0]
197
198        try:
199            kp = Popen(cmd)
200            kp.wait()
201        except:
202            pass
203
204    def update_cmd_privs(self, cmd, user):
205        """
206        If a user has been specified to run this Cmd and we're not already
207        running as that user, prepend the appropriate sudo command to run
208        as that user.
209        """
210        me = getpwuid(os.getuid())
211
212        if not user or user == me:
213            return cmd
214
215        ret = '%s -E -u %s %s' % (SUDO, user, cmd)
216        return ret.split(' ')
217
218    def collect_output(self, proc):
219        """
220        Read from stdout/stderr as data becomes available, until the
221        process is no longer running. Return the lines from the stdout and
222        stderr Output objects.
223        """
224        out = Output(proc.stdout)
225        err = Output(proc.stderr)
226        res = []
227        while proc.returncode is None:
228            proc.poll()
229            res = select([out, err], [], [], .1)
230            for fd in res[0]:
231                fd.read()
232        for fd in res[0]:
233            fd.read(drain=1)
234
235        return out.lines, err.lines
236
237    def run(self, options):
238        """
239        This is the main function that runs each individual test.
240        Determine whether or not the command requires sudo, and modify it
241        if needed. Run the command, and update the result object.
242        """
243        if options.dryrun is True:
244            print(self)
245            return
246
247        privcmd = self.update_cmd_privs(self.pathname, self.user)
248        try:
249            old = os.umask(0)
250            if not os.path.isdir(self.outputdir):
251                os.makedirs(self.outputdir, mode=0o777)
252            os.umask(old)
253        except OSError as e:
254            fail('%s' % e)
255
256        try:
257            self.result.starttime = time()
258            proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE,
259                         universal_newlines=True)
260            proc.stdin.close()
261
262            # Allow a special timeout value of 0 to mean infinity
263            if int(self.timeout) == 0:
264                self.timeout = maxsize
265            t = Timer(int(self.timeout), self.kill_cmd, [proc])
266            t.start()
267            self.result.stdout, self.result.stderr = self.collect_output(proc)
268        except KeyboardInterrupt:
269            self.kill_cmd(proc)
270            fail('\nRun terminated at user request.')
271        finally:
272            t.cancel()
273
274        self.result.done(proc, self.killed)
275
276    def skip(self):
277        """
278        Initialize enough of the test result that we can log a skipped
279        command.
280        """
281        Result.total += 1
282        Result.runresults['SKIP'] += 1
283        self.result.stdout = self.result.stderr = []
284        self.result.starttime = time()
285        m, s = divmod(time() - self.result.starttime, 60)
286        self.result.runtime = '%02d:%02d' % (m, s)
287        self.result.result = 'SKIP'
288
289    def log(self, logger, options):
290        """
291        This function is responsible for writing all output. This includes
292        the console output, the logfile of all results (with timestamped
293        merged stdout and stderr), and for each test, the unmodified
294        stdout/stderr/merged in it's own file.
295        """
296        if logger is None:
297            return
298
299        logname = getpwuid(os.getuid()).pw_name
300        user = ' (run as %s)' % (self.user if len(self.user) else logname)
301        msga = 'Test: %s%s ' % (self.pathname, user)
302        msgb = '[%s] [%s]' % (self.result.runtime, self.result.result)
303        pad = ' ' * (80 - (len(msga) + len(msgb)))
304
305        # If -q is specified, only print a line for tests that didn't pass.
306        # This means passing tests need to be logged as DEBUG, or the one
307        # line summary will only be printed in the logfile for failures.
308        if not options.quiet:
309            logger.info('%s%s%s' % (msga, pad, msgb))
310        elif self.result.result != 'PASS':
311            logger.info('%s%s%s' % (msga, pad, msgb))
312        else:
313            logger.debug('%s%s%s' % (msga, pad, msgb))
314
315        lines = sorted(self.result.stdout + self.result.stderr,
316                       key=lambda x: x[0])
317
318        for dt, line in lines:
319            logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line))
320
321        if len(self.result.stdout):
322            with io.open(os.path.join(self.outputdir, 'stdout'),
323                                   encoding='utf-8',
324                                   errors='surrogateescape',
325                                   mode='w') as out:
326                for _, line in self.result.stdout:
327                    out.write('%s\n' % line)
328        if len(self.result.stderr):
329            with io.open(os.path.join(self.outputdir, 'stderr'),
330                                   encoding='utf-8',
331                                   errors='surrogateescape',
332                                   mode='w') as err:
333                for _, line in self.result.stderr:
334                    err.write('%s\n' % line)
335        if len(self.result.stdout) and len(self.result.stderr):
336            with io.open(os.path.join(self.outputdir, 'merged'),
337                                   encoding='utf-8',
338                                   errors='surrogateescape',
339                                   mode='w') as merged:
340                for _, line in lines:
341                    merged.write('%s\n' % line)
342
343
344class Test(Cmd):
345    props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
346             'post_user']
347
348    def __init__(self, pathname, outputdir=None, timeout=None, user=None,
349                 pre=None, pre_user=None, post=None, post_user=None):
350        super(Test, self).__init__(pathname, outputdir, timeout, user)
351        self.pre = pre or ''
352        self.pre_user = pre_user or ''
353        self.post = post or ''
354        self.post_user = post_user or ''
355
356    def __str__(self):
357        post_user = pre_user = ''
358        if len(self.pre_user):
359            pre_user = ' (as %s)' % (self.pre_user)
360        if len(self.post_user):
361            post_user = ' (as %s)' % (self.post_user)
362        return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
363               "%s%s\nUser: %s\n" % \
364               (self.pathname, self.outputdir, self.timeout, self.pre,
365                pre_user, self.post, post_user, self.user)
366
367    def verify(self, logger):
368        """
369        Check the pre/post scripts, user and Test. Omit the Test from this
370        run if there are any problems.
371        """
372        files = [self.pre, self.pathname, self.post]
373        users = [self.pre_user, self.user, self.post_user]
374
375        for f in [f for f in files if len(f)]:
376            if not verify_file(f):
377                logger.info("Warning: Test '%s' not added to this run because"
378                            " it failed verification." % f)
379                return False
380
381        for user in [user for user in users if len(user)]:
382            if not verify_user(user, logger):
383                logger.info("Not adding Test '%s' to this run." %
384                            self.pathname)
385                return False
386
387        return True
388
389    def run(self, logger, options):
390        """
391        Create Cmd instances for the pre/post scripts. If the pre script
392        doesn't pass, skip this Test. Run the post script regardless.
393        """
394        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
395        pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
396                      user=self.pre_user)
397        test = Cmd(self.pathname, outputdir=self.outputdir,
398                   timeout=self.timeout, user=self.user)
399        odir = os.path.join(self.outputdir, os.path.basename(self.post))
400        posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
401                       user=self.post_user)
402
403        cont = True
404        if len(pretest.pathname):
405            pretest.run(options)
406            cont = pretest.result.result == 'PASS'
407            pretest.log(logger, options)
408
409        if cont:
410            test.run(options)
411        else:
412            test.skip()
413
414        test.log(logger, options)
415
416        if len(posttest.pathname):
417            posttest.run(options)
418            posttest.log(logger, options)
419
420
421class TestGroup(Test):
422    props = Test.props + ['tests']
423
424    def __init__(self, pathname, outputdir=None, timeout=None, user=None,
425                 pre=None, pre_user=None, post=None, post_user=None,
426                 tests=None):
427        super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
428                                        pre, pre_user, post, post_user)
429        self.tests = tests or []
430
431    def __str__(self):
432        post_user = pre_user = ''
433        if len(self.pre_user):
434            pre_user = ' (as %s)' % (self.pre_user)
435        if len(self.post_user):
436            post_user = ' (as %s)' % (self.post_user)
437        return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \
438               "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \
439               (self.pathname, self.outputdir, self.tests, self.timeout,
440                self.pre, pre_user, self.post, post_user, self.user)
441
442    def filter(self, keeplist):
443        self.tests = [ x for x in self.tests if x in keeplist ]
444
445    def verify(self, logger):
446        """
447        Check the pre/post scripts, user and tests in this TestGroup. Omit
448        the TestGroup entirely, or simply delete the relevant tests in the
449        group, if that's all that's required.
450        """
451        # If the pre or post scripts are relative pathnames, convert to
452        # absolute, so they stand a chance of passing verification.
453        if len(self.pre) and not os.path.isabs(self.pre):
454            self.pre = os.path.join(self.pathname, self.pre)
455        if len(self.post) and not os.path.isabs(self.post):
456            self.post = os.path.join(self.pathname, self.post)
457
458        auxfiles = [self.pre, self.post]
459        users = [self.pre_user, self.user, self.post_user]
460
461        for f in [f for f in auxfiles if len(f)]:
462            if self.pathname != os.path.dirname(f):
463                logger.info("Warning: TestGroup '%s' not added to this run. "
464                            "Auxiliary script '%s' exists in a different "
465                            "directory." % (self.pathname, f))
466                return False
467
468            if not verify_file(f):
469                logger.info("Warning: TestGroup '%s' not added to this run. "
470                            "Auxiliary script '%s' failed verification." %
471                            (self.pathname, f))
472                return False
473
474        for user in [user for user in users if len(user)]:
475            if not verify_user(user, logger):
476                logger.info("Not adding TestGroup '%s' to this run." %
477                            self.pathname)
478                return False
479
480        # If one of the tests is invalid, delete it, log it, and drive on.
481        self.tests[:] = [f for f in self.tests if
482          verify_file(os.path.join(self.pathname, f))]
483
484        return len(self.tests) != 0
485
486    def run(self, logger, options):
487        """
488        Create Cmd instances for the pre/post scripts. If the pre script
489        doesn't pass, skip all the tests in this TestGroup. Run the post
490        script regardless.
491        """
492        odir = os.path.join(self.outputdir, os.path.basename(self.pre))
493        pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
494                      user=self.pre_user)
495        odir = os.path.join(self.outputdir, os.path.basename(self.post))
496        posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
497                       user=self.post_user)
498
499        cont = True
500        if len(pretest.pathname):
501            pretest.run(options)
502            cont = pretest.result.result == 'PASS'
503            pretest.log(logger, options)
504
505        for fname in self.tests:
506            test = Cmd(os.path.join(self.pathname, fname),
507                       outputdir=os.path.join(self.outputdir, fname),
508                       timeout=self.timeout, user=self.user)
509            if cont:
510                test.run(options)
511            else:
512                test.skip()
513
514            test.log(logger, options)
515
516        if len(posttest.pathname):
517            posttest.run(options)
518            posttest.log(logger, options)
519
520
521class TestRun(object):
522    props = ['quiet', 'outputdir']
523
524    def __init__(self, options):
525        self.tests = {}
526        self.testgroups = {}
527        self.starttime = time()
528        self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
529        self.outputdir = os.path.join(options.outputdir, self.timestamp)
530        self.logger = self.setup_logging(options)
531        self.defaults = [
532            ('outputdir', BASEDIR),
533            ('quiet', False),
534            ('timeout', 60),
535            ('user', ''),
536            ('pre', ''),
537            ('pre_user', ''),
538            ('post', ''),
539            ('post_user', '')
540        ]
541
542    def __str__(self):
543        s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
544        s += 'TESTS:\n'
545        for key in sorted(self.tests.keys()):
546            s += '%s%s' % (self.tests[key].__str__(), '\n')
547        s += 'TESTGROUPS:\n'
548        for key in sorted(self.testgroups.keys()):
549            s += '%s%s' % (self.testgroups[key].__str__(), '\n')
550        return s
551
552    def addtest(self, pathname, options):
553        """
554        Create a new Test, and apply any properties that were passed in
555        from the command line. If it passes verification, add it to the
556        TestRun.
557        """
558        test = Test(pathname)
559        for prop in Test.props:
560            setattr(test, prop, getattr(options, prop))
561
562        if test.verify(self.logger):
563            self.tests[pathname] = test
564
565    def addtestgroup(self, dirname, filenames, options):
566        """
567        Create a new TestGroup, and apply any properties that were passed
568        in from the command line. If it passes verification, add it to the
569        TestRun.
570        """
571        if dirname not in self.testgroups:
572            testgroup = TestGroup(dirname)
573            for prop in Test.props:
574                setattr(testgroup, prop, getattr(options, prop))
575
576            # Prevent pre/post scripts from running as regular tests
577            for f in [testgroup.pre, testgroup.post]:
578                if f in filenames:
579                    del filenames[filenames.index(f)]
580
581            self.testgroups[dirname] = testgroup
582            self.testgroups[dirname].tests = sorted(filenames)
583
584            testgroup.verify(self.logger)
585
586    def filter(self, keeplist):
587        for group in list(self.testgroups.keys()):
588            if group not in keeplist:
589                del self.testgroups[group]
590                continue
591
592            g = self.testgroups[group]
593
594            if g.pre and os.path.basename(g.pre) in keeplist[group]:
595                continue
596
597            g.filter(keeplist[group])
598
599        for test in list(self.tests.keys()):
600            directory, base = os.path.split(test)
601            if directory not in keeplist or base not in keeplist[directory]:
602                del self.tests[test]
603
604    def read(self, logger, options):
605        """
606        Read in the specified runfile, and apply the TestRun properties
607        listed in the 'DEFAULT' section to our TestRun. Then read each
608        section, and apply the appropriate properties to the Test or
609        TestGroup. Properties from individual sections override those set
610        in the 'DEFAULT' section. If the Test or TestGroup passes
611        verification, add it to the TestRun.
612        """
613        config = configparser.RawConfigParser()
614        if not len(config.read(options.runfile)):
615            fail("Couldn't read config file %s" % options.runfile)
616
617        for opt in TestRun.props:
618            if config.has_option('DEFAULT', opt):
619                setattr(self, opt, config.get('DEFAULT', opt))
620        self.outputdir = os.path.join(self.outputdir, self.timestamp)
621
622        for section in config.sections():
623            if ('arch' in config.options(section) and
624                platform.machine() != config.get(section, 'arch')):
625                continue
626
627            if 'tests' in config.options(section):
628                testgroup = TestGroup(section)
629                for prop in TestGroup.props:
630                    for sect in ['DEFAULT', section]:
631                        if config.has_option(sect, prop):
632                            setattr(testgroup, prop, config.get(sect, prop))
633
634                # Repopulate tests using eval to convert the string to a list
635                testgroup.tests = eval(config.get(section, 'tests'))
636
637                if testgroup.verify(logger):
638                    self.testgroups[section] = testgroup
639
640            elif 'autotests' in config.options(section):
641                testgroup = TestGroup(section)
642                for prop in TestGroup.props:
643                    for sect in ['DEFAULT', section]:
644                        if config.has_option(sect, prop):
645                            setattr(testgroup, prop, config.get(sect, prop))
646
647                filenames = os.listdir(section)
648                # only files starting with "tst." are considered tests
649                filenames = [f for f in filenames if f.startswith("tst.")]
650                testgroup.tests = sorted(filenames)
651
652                if testgroup.verify(logger):
653                    self.testgroups[section] = testgroup
654
655            else:
656                test = Test(section)
657                for prop in Test.props:
658                    for sect in ['DEFAULT', section]:
659                        if config.has_option(sect, prop):
660                            setattr(test, prop, config.get(sect, prop))
661
662                if test.verify(logger):
663                    self.tests[section] = test
664
665    def write(self, options):
666        """
667        Create a configuration file for editing and later use. The
668        'DEFAULT' section of the config file is created from the
669        properties that were specified on the command line. Tests are
670        simply added as sections that inherit everything from the
671        'DEFAULT' section. TestGroups are the same, except they get an
672        option including all the tests to run in that directory.
673        """
674
675        defaults = dict([(prop, getattr(options, prop)) for prop, _ in
676                         self.defaults])
677        config = configparser.RawConfigParser(defaults)
678
679        for test in sorted(self.tests.keys()):
680            config.add_section(test)
681            for prop in Test.props:
682                if prop not in self.props:
683                    config.set(testgroup, prop,
684                        getattr(self.testgroups[testgroup], prop))
685
686        for testgroup in sorted(self.testgroups.keys()):
687            config.add_section(testgroup)
688            config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
689            for prop in TestGroup.props:
690                if prop not in self.props:
691                    config.set(testgroup, prop,
692                        getattr(self.testgroups[testgroup], prop))
693
694        try:
695            with open(options.template, 'w') as f:
696                return config.write(f)
697        except IOError:
698            fail('Could not open \'%s\' for writing.' % options.template)
699
700    def complete_outputdirs(self):
701        """
702        Collect all the pathnames for Tests, and TestGroups. Work
703        backwards one pathname component at a time, to create a unique
704        directory name in which to deposit test output. Tests will be able
705        to write output files directly in the newly modified outputdir.
706        TestGroups will be able to create one subdirectory per test in the
707        outputdir, and are guaranteed uniqueness because a group can only
708        contain files in one directory. Pre and post tests will create a
709        directory rooted at the outputdir of the Test or TestGroup in
710        question for their output.
711        """
712        done = False
713        components = 0
714        tmp_dict = dict(list(self.tests.items()) + list(self.testgroups.items()))
715        total = len(tmp_dict)
716        base = self.outputdir
717
718        while not done:
719            l = []
720            components -= 1
721            for testfile in list(tmp_dict.keys()):
722                uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
723                if uniq not in l:
724                    l.append(uniq)
725                    tmp_dict[testfile].outputdir = os.path.join(base, uniq)
726                else:
727                    break
728            done = total == len(l)
729
730    def setup_logging(self, options):
731        """
732        Two loggers are set up here. The first is for the logfile which
733        will contain one line summarizing the test, including the test
734        name, result, and running time. This logger will also capture the
735        timestamped combined stdout and stderr of each run. The second
736        logger is optional console output, which will contain only the one
737        line summary. The loggers are initialized at two different levels
738        to facilitate segregating the output.
739        """
740        if options.dryrun is True:
741            return
742
743        testlogger = logging.getLogger(__name__)
744        testlogger.setLevel(logging.DEBUG)
745
746        if not options.template:
747            try:
748                old = os.umask(0)
749                os.makedirs(self.outputdir, mode=0o777)
750                os.umask(old)
751            except OSError as e:
752                fail('%s' % e)
753            filename = os.path.join(self.outputdir, 'log')
754
755            logfile = WatchedFileHandlerClosed(filename)
756            logfile.setLevel(logging.DEBUG)
757            logfilefmt = logging.Formatter('%(message)s')
758            logfile.setFormatter(logfilefmt)
759            testlogger.addHandler(logfile)
760
761        cons = logging.StreamHandler()
762        cons.setLevel(logging.INFO)
763        consfmt = logging.Formatter('%(message)s')
764        cons.setFormatter(consfmt)
765        testlogger.addHandler(cons)
766
767        return testlogger
768
769    def run(self, options):
770        """
771        Walk through all the Tests and TestGroups, calling run().
772        """
773        if not options.dryrun:
774            try:
775                os.chdir(self.outputdir)
776            except OSError:
777                fail('Could not change to directory %s' % self.outputdir)
778        for test in sorted(self.tests.keys()):
779            self.tests[test].run(self.logger, options)
780        for testgroup in sorted(self.testgroups.keys()):
781            self.testgroups[testgroup].run(self.logger, options)
782
783    def summary(self):
784        if Result.total == 0:
785            return
786
787        print('\nResults Summary')
788        for key in list(Result.runresults.keys()):
789            if Result.runresults[key] != 0:
790                print('%s\t% 4d' % (key, Result.runresults[key]))
791
792        m, s = divmod(time() - self.starttime, 60)
793        h, m = divmod(m, 60)
794        print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
795        print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
796                                            float(Result.total)) * 100))
797        print('Log directory:\t%s' % self.outputdir)
798
799
800def verify_file(pathname):
801    """
802    Verify that the supplied pathname is an executable regular file.
803    """
804    if os.path.isdir(pathname) or os.path.islink(pathname):
805        return False
806
807    if os.path.isfile(pathname) and os.access(pathname, os.X_OK):
808        return True
809
810    return False
811
812
813def verify_user(user, logger):
814    """
815    Verify that the specified user exists on this system, and can execute
816    sudo without being prompted for a password.
817    """
818    testcmd = [SUDO, '-n', '-u', user, TRUE]
819
820    if user in Cmd.verified_users:
821        return True
822
823    try:
824        _ = getpwnam(user)
825    except KeyError:
826        logger.info("Warning: user '%s' does not exist.", user)
827        return False
828
829    p = Popen(testcmd)
830    p.wait()
831    if p.returncode != 0:
832        logger.info("Warning: user '%s' cannot use passwordless sudo.", user)
833        return False
834    else:
835        Cmd.verified_users.append(user)
836
837    return True
838
839
840def find_tests(testrun, options):
841    """
842    For the given list of pathnames, add files as Tests. For directories,
843    if do_groups is True, add the directory as a TestGroup. If False,
844    recursively search for executable files.
845    """
846
847    for p in sorted(options.pathnames):
848        if os.path.isdir(p):
849            for dirname, _, filenames in os.walk(p):
850                if options.do_groups:
851                    testrun.addtestgroup(dirname, filenames, options)
852                else:
853                    for f in sorted(filenames):
854                        testrun.addtest(os.path.join(dirname, f), options)
855        else:
856            testrun.addtest(p, options)
857
858
859def filter_tests(testrun, options):
860    try:
861        fh = open(options.logfile, "r")
862    except Exception as e:
863        fail('%s' % e)
864
865    failed = {}
866    while True:
867            line = fh.readline()
868            if not line:
869                 break
870            m = re.match(r'Test: (.*)/(\S+).*\[FAIL\]', line)
871            if not m:
872                continue
873            group, test = m.group(1, 2)
874            try:
875                failed[group].append(test)
876            except KeyError:
877                failed[group] = [ test ]
878    fh.close()
879
880    testrun.filter(failed)
881
882
883def fail(retstr, ret=1):
884    print('%s: %s' % (argv[0], retstr))
885    exit(ret)
886
887
888def options_cb(option, opt_str, value, parser):
889    path_options = ['runfile', 'outputdir', 'template', 'logfile']
890
891    if opt_str in parser.rargs:
892        fail('%s may only be specified once.' % opt_str)
893
894    if option.dest in path_options:
895        setattr(parser.values, option.dest, os.path.abspath(value))
896    else:
897        setattr(parser.values, option.dest, value)
898
899
900def parse_args():
901    parser = OptionParser()
902    parser.add_option('-c', action='callback', callback=options_cb,
903                      type='string', dest='runfile', metavar='runfile',
904                      help='Specify tests to run via config file.')
905    parser.add_option('-d', action='store_true', default=False, dest='dryrun',
906                      help='Dry run. Print tests, but take no other action.')
907    parser.add_option('-l', action='callback', callback=options_cb,
908                      default=None, dest='logfile', metavar='logfile',
909                      type='string',
910                      help='Read logfile and re-run tests which failed.')
911    parser.add_option('-g', action='store_true', default=False,
912                      dest='do_groups', help='Make directories TestGroups.')
913    parser.add_option('-o', action='callback', callback=options_cb,
914                      default=BASEDIR, dest='outputdir', type='string',
915                      metavar='outputdir', help='Specify an output directory.')
916    parser.add_option('-p', action='callback', callback=options_cb,
917                      default='', dest='pre', metavar='script',
918                      type='string', help='Specify a pre script.')
919    parser.add_option('-P', action='callback', callback=options_cb,
920                      default='', dest='post', metavar='script',
921                      type='string', help='Specify a post script.')
922    parser.add_option('-q', action='store_true', default=False, dest='quiet',
923                      help='Silence on the console during a test run.')
924    parser.add_option('-t', action='callback', callback=options_cb, default=60,
925                      dest='timeout', metavar='seconds', type='int',
926                      help='Timeout (in seconds) for an individual test.')
927    parser.add_option('-u', action='callback', callback=options_cb,
928                      default='', dest='user', metavar='user', type='string',
929                      help='Specify a different user name to run as.')
930    parser.add_option('-w', action='callback', callback=options_cb,
931                      default=None, dest='template', metavar='template',
932                      type='string', help='Create a new config file.')
933    parser.add_option('-x', action='callback', callback=options_cb, default='',
934                      dest='pre_user', metavar='pre_user', type='string',
935                      help='Specify a user to execute the pre script.')
936    parser.add_option('-X', action='callback', callback=options_cb, default='',
937                      dest='post_user', metavar='post_user', type='string',
938                      help='Specify a user to execute the post script.')
939    (options, pathnames) = parser.parse_args()
940
941    if options.runfile and len(pathnames):
942        fail('Extraneous arguments.')
943
944    options.pathnames = [os.path.abspath(path) for path in pathnames]
945
946    return options
947
948
949def main():
950    options = parse_args()
951
952    testrun = TestRun(options)
953
954    if options.runfile:
955        testrun.read(testrun.logger, options)
956    else:
957        find_tests(testrun, options)
958
959    if options.logfile:
960        filter_tests(testrun, options)
961
962    if options.template:
963        testrun.write(options)
964        exit(0)
965
966    testrun.complete_outputdirs()
967    testrun.run(options)
968    testrun.summary()
969    exit(retcode)
970
971
972if __name__ == '__main__':
973    main()
974