xref: /illumos-gate/usr/src/test/smbsrv-tests/cmd/run_smbtorture.py (revision 1b38ec66a3ea040587c1b346f8c010a3b37c8080) 
1#!@PYTHON@
2#
3# This file and its contents are supplied under the terms of the
4# Common Development and Distribution License ("CDDL"), version 1.0.
5# You may only use this file in accordance with the terms of version
6# 1.0 of the CDDL.
7#
8# A full copy of the text of the CDDL should have accompanied this
9# source.  A copy of the CDDL is also available via the Internet at
10# http://www.illumos.org/license/CDDL.
11#
12
13#
14# Copyright 2022 Tintri by DDN, Inc. All rights reserved.
15#
16
17#
18# Run tests provided by smbtorture.
19#
20
21import subprocess
22import argparse
23import re
24import fnmatch
25
26from enum import Enum
27from datetime import datetime
28from tempfile import TemporaryFile
29
30def stripped_file(f):
31    """Strips trailing whitespace from lines in f"""
32
33    for line in f:
34        yield line.strip()
35
36def parse_tests(f):
37    """Returns test names from f, skipping commented lines"""
38
39    yield from (line for line in f
40        if line and not line.startswith('#'))
41
42def matched_suites(m):
43    """Gets all smbtorture tests that match pattern m"""
44
45    with TemporaryFile('w+') as tmp:
46        subprocess.run(['smbtorture', '--list'], stdout=tmp,
47            universal_newlines=True)
48        tmp.seek(0)
49        yield from (line for line in stripped_file(tmp)
50                    if not line.startswith('smbtorture') and
51                    m.match(line))
52
53class TestResult(Enum):
54    PASS = 0
55    FAIL = 1
56    UNKNOWN = 2
57    SKIP = 3
58    KILLED = 4
59    TEST_ERR = 5
60
61    def __str__(self):
62        return self.name
63
64    def __len__(self):
65        return len(self.name)
66
67class TestCase:
68    """A particular instance of an smbtorture test"""
69
70    __slots__ = 'name', 'result'
71
72    def __init__(self, name):
73        self.name = name
74        self.result = TestResult.UNKNOWN
75
76    def __str__(self):
77        return f'{self.name} | {self.result}'
78
79    def run(self, rfd, wfd, timeout, cmd):
80        """Run cmd, setting the last element to the test name, and setting result
81        based on rfd. Output is sent to wfd, and the test is killed based on timeout."""
82
83        def finish(self, start, wfd):
84            timediff = datetime.now() - start
85            wfd.write(f'END   | {self} | {timediff}\n')
86            return self.result
87
88        starttime = datetime.now()
89        wfd.write(f'START | {self.name} | {starttime.time()}\n')
90        if self.result == TestResult.SKIP:
91            return finish(self, starttime, wfd)
92
93        cmd[-1] = self.name
94        try:
95            subprocess.run(cmd, universal_newlines=True, stdout=wfd,
96                stderr=subprocess.STDOUT, timeout=timeout)
97            for line in stripped_file(rfd):
98                if self.result != TestResult.UNKNOWN:
99                    continue
100                elif line.startswith('failure:') or line.startswith('error:'):
101                    self.result = TestResult.FAIL
102                elif line.startswith('success:'):
103                    self.result = TestResult.PASS
104                elif line.startswith('skip:'):
105                    self.result = TestResult.SKIP
106                elif line.startswith('INTERNAL ERROR:'):
107                    self.result = TestResult.TEST_ERR
108        except subprocess.TimeoutExpired:
109            self.result = TestResult.KILLED
110            wfd.write('\nKilled due to timeout\n')
111            rfd.read()
112
113        return finish(self, starttime, wfd)
114
115class TestSet:
116    """Class to track state associated with the entire test set"""
117
118    __slots__ = 'excluded', 'tests'
119
120    def __init__(self, tests, skip_pat, verbose):
121        self.excluded = 0
122
123        def should_skip(self, test, pattern, verbose):
124            """Returns whether test matches pattern, indicating it should be
125            skipped."""
126
127            if not pattern or not pattern.match(test):
128                return False
129
130            if verbose:
131                print(f'{test} matches exception pattern; marking as skipped')
132
133            self.excluded += 1
134            return True
135
136        self.tests = [TestCase(line) for line in tests
137            if not should_skip(self, line, skip_pat, verbose)]
138
139
140    def __iter__(self):
141        return iter(self.tests)
142
143    def __len__(self):
144        return len(self.tests)
145
146def fnm2regex(fnm_pat):
147    """Maps an fnmatch(7) pattern to a regex pattern that will match against
148    any suite that encapsulates the test name"""
149
150    rpat = fnmatch.translate(fnm_pat)
151
152    #
153    # If the pattern doesn't end with '*', we also need it to match against
154    # any sub-module; '*test' needs to also match 'smb2.test.first', but
155    # not 'smb2.test-other.second'.
156    #
157    if not fnm_pat.endswith('*'):
158        rpat += '|' + fnmatch.translate(fnm_pat + '.*')
159    return rpat
160
161def verbose_fnm2regex(fnm_pat):
162    """fnm2regex(), but prints the input and output patterns"""
163    ret_pat = fnm2regex(fnm_pat)
164    print(f'fnmatch: {fnm_pat} regex: {ret_pat}')
165    return ret_pat
166
167def combine_patterns(iterable, verbose):
168    """Combines patterns in an iterable into a single REGEX"""
169
170    if verbose > 1:
171        func = verbose_fnm2regex
172    else:
173        func = fnm2regex
174
175    fnmatch_pat = '|'.join(map(func, iterable))
176
177    if not fnmatch_pat:
178        pat = None;
179    else:
180        pat = re.compile(fnmatch_pat, flags=re.DEBUG if verbose > 2 else 0)
181
182    if verbose > 1:
183        print(f'final pattern: {pat.pattern if pat else "<None>"}')
184    return pat
185
186class ArgumentFile(argparse.FileType):
187    """argparse.FileType, but wrapped in stripped_file()"""
188
189    def __call__(self, *args, **kwargs):
190        return stripped_file(argparse.FileType.__call__(self, *args, **kwargs))
191
192
193def main():
194    parser = argparse.ArgumentParser(description=
195        'Run a set of smbtorture tests, parsing the results.')
196
197    parser.add_argument('server', help='The target server')
198    parser.add_argument('share', help='The target share')
199    parser.add_argument('user', help='Username for smbtorture')
200    parser.add_argument('password', help='Password for user')
201
202    parser.add_argument('--except', '-e',
203        type=ArgumentFile('r'), metavar='EXCEPTIONS_FILE', dest='skip_list',
204        help='A file containing fnmatch(7) patterns of tests to skip')
205    parser.add_argument('--list', '-l',
206        type=ArgumentFile('r'), metavar='LIST_FILE',
207        help='A file containing the list of tests to run')
208    parser.add_argument('--match', '-m',
209        action='append', metavar='FNMATCH',
210        help='An fnmatch(7) pattern to select tests from smbtorture --list')
211    parser.add_argument('--output', '-o',
212        default='/tmp/lastrun.log', metavar='LOG_FILE',
213        help='Location to store full smbtorture output')
214    parser.add_argument('--seed', '-s',
215        type=int,
216        help='Seed passed to smbtorture')
217    parser.add_argument('--timeout', '-t',
218        default=120, type=float,
219        help='Timeout after which test is killed')
220    parser.add_argument('--verbose', '-v',
221        action='count', default=0,
222        help='Verbose output')
223
224    args = parser.parse_args()
225
226    if (args.match == None) == (args.list == None):
227        print('Must provide one of -l and -m')
228        return
229
230    server = args.server
231    share = args.share
232    user = args.user
233    pswd = args.password
234    fout = args.output
235
236    if args.match != None:
237        if args.verbose > 1:
238            print('Patterns to match:')
239            print(*args.match)
240
241        testgen = matched_suites(combine_patterns(args.match, args.verbose))
242    else:
243        testgen = args.list
244
245    if args.skip_list != None:
246        skip_pat = combine_patterns(parse_tests(args.skip_list), args.verbose)
247        if args.verbose > 1:
248            exc_pat = skip_pat.pattern if skip_pat else '<NONE>'
249            print(f'Exceptions pattern (in REGEX): {exc_pat}')
250    else:
251        skip_pat = None
252
253    tests = TestSet(parse_tests(testgen), skip_pat, args.verbose)
254
255    if args.verbose:
256        print('Tests to run:')
257        for test in tests:
258            print(test.name)
259
260    outw = open(fout, 'w', buffering=1)
261    outr = open(fout, 'r')
262
263    cmd = f'smbtorture //{server}/{share} -U{user}%{pswd}'.split()
264    if args.seed != None:
265        cmd.append(f'--seed={args.seed}')
266    cmd.append('TEST_HERE')
267
268    if args.verbose:
269        print('Command to run:')
270        print(*cmd)
271
272    results = {res: 0 for res in TestResult}
273    for test in tests:
274        print(test.name, end=': ', flush=True)
275        res = test.run(outr, outw, args.timeout, cmd)
276        results[res] += 1
277        print(res, flush=True)
278
279    print('\n\nRESULTS:')
280    print('=' * 22)
281    for res in TestResult:
282        print(f'{res}: {results[res]:>{20 - len(res)}}')
283    print('=' * 22)
284    print(f'Total: {len(tests):>15}')
285    print(f'Excluded: {tests.excluded:>12}')
286
287if __name__ == '__main__':
288    try:
289        main()
290    except KeyboardInterrupt:
291        print('Terminated by KeyboardInterrupt.')
292