xref: /freebsd/contrib/lib9p/pytest/client.py (revision af23369a6deaaeb612ab266eb88b8bb8d560c322) 
1#! /usr/bin/env python
2
3"""
4Run various tests, as a client.
5"""
6
7from __future__ import print_function
8
9import argparse
10try:
11    import ConfigParser as configparser
12except ImportError:
13    import configparser
14import functools
15import logging
16import os
17import socket
18import struct
19import sys
20import time
21import traceback
22
23import p9conn
24import protocol
25
26LocalError = p9conn.LocalError
27RemoteError = p9conn.RemoteError
28TEError = p9conn.TEError
29
30class TestState(object):
31    def __init__(self):
32        self.config = None
33        self.logger = None
34        self.successes = 0
35        self.skips = 0
36        self.failures = 0
37        self.exceptions = 0
38        self.clnt_tab = {}
39        self.mkclient = None
40        self.stop = False
41        self.gid = 0
42
43    def ccc(self, cid=None):
44        """
45        Connect or reconnect as client (ccc = check and connect client).
46
47        If caller provides a cid (client ID) we check that specific
48        client.  Otherwise the default ID ('base') is used.
49        In any case we return the now-connected client, plus the
50        attachment (session info) if any.
51        """
52        if cid is None:
53            cid = 'base'
54        pair = self.clnt_tab.get(cid)
55        if pair is None:
56            clnt = self.mkclient()
57            pair = [clnt, None]
58            self.clnt_tab[cid] = pair
59        else:
60            clnt = pair[0]
61        if not clnt.is_connected():
62            clnt.connect()
63        return pair
64
65    def dcc(self, cid=None):
66        """
67        Disconnect client (disconnect checked client).  If no specific
68        client ID is provided, this disconnects ALL checked clients!
69        """
70        if cid is None:
71            for cid in list(self.clnt_tab.keys()):
72                self.dcc(cid)
73        pair = self.clnt_tab.get(cid)
74        if pair is not None:
75            clnt = pair[0]
76            if clnt.is_connected():
77                clnt.shutdown()
78            del self.clnt_tab[cid]
79
80    def ccs(self, cid=None):
81        """
82        Like ccc, but establish a session as well, by setting up
83        the uname/n_uname.
84
85        Return the client instance (only).
86        """
87        pair = self.ccc(cid)
88        clnt = pair[0]
89        if pair[1] is None:
90            # No session yet - establish one.  Note, this may fail.
91            section = None if cid is None else ('client-' + cid)
92            aname = getconf(self.config, section, 'aname', '')
93            uname = getconf(self.config, section, 'uname', '')
94            if clnt.proto > protocol.plain:
95                n_uname = getint(self.config, section, 'n_uname', 1001)
96            else:
97                n_uname = None
98            clnt.attach(afid=None, aname=aname, uname=uname, n_uname=n_uname)
99            pair[1] = (aname, uname, n_uname)
100        return clnt
101
102def getconf(conf, section, name, default=None, rtype=str):
103    """
104    Get configuration item for given section, or for "client" if
105    there is no entry for that particular section (or if section
106    is None).
107
108    This lets us get specific values for specific tests or
109    groups ([foo] name=value), falling back to general values
110    ([client] name=value).
111
112    The type of the returned value <rtype> can be str, int, bool,
113    or float.  The default is str (and see getconfint, getconfbool,
114    getconffloat below).
115
116    A default value may be supplied; if it is, that's the default
117    return value (this default should have the right type).  If
118    no default is supplied, a missing value is an error.
119    """
120    try:
121        # note: conf.get(None, 'foo') raises NoSectionError
122        where = section
123        result = conf.get(where, name)
124    except (configparser.NoSectionError, configparser.NoOptionError):
125        try:
126            where = 'client'
127            result = conf.get(where, name)
128        except configparser.NoSectionError:
129            sys.exit('no [{0}] section in configuration!'.format(where))
130        except configparser.NoOptionError:
131            if default is not None:
132                return default
133            if section is not None:
134                where = '[{0}] or [{1}]'.format(section, where)
135            else:
136                where = '[{0}]'.format(where)
137            raise LocalError('need {0}=value in {1}'.format(name, where))
138    where = '[{0}]'.format(where)
139    if rtype is str:
140        return result
141    if rtype is int:
142        return int(result)
143    if rtype is float:
144        return float(result)
145    if rtype is bool:
146        if result.lower() in ('1', 't', 'true', 'y', 'yes'):
147            return True
148        if result.lower() in ('0', 'f', 'false', 'n', 'no'):
149            return False
150        raise ValueError('{0} {1}={2}: invalid boolean'.format(where, name,
151                                                              result))
152    raise ValueError('{0} {1}={2}: internal error: bad result type '
153                     '{3!r}'.format(where, name, result, rtype))
154
155def getint(conf, section, name, default=None):
156    "get integer config item"
157    return getconf(conf, section, name, default, int)
158
159def getfloat(conf, section, name, default=None):
160    "get float config item"
161    return getconf(conf, section, name, default, float)
162
163def getbool(conf, section, name, default=None):
164    "get boolean config item"
165    return getconf(conf, section, name, default, bool)
166
167def pluralize(n, singular, plural):
168    "return singular or plural based on value of n"
169    return plural if n != 1 else singular
170
171class TCDone(Exception):
172    "used in succ/fail/skip - skips rest of testcase with"
173    pass
174
175class TestCase(object):
176    """
177    Start a test case.  Most callers must then do a ccs() to connect.
178
179    A failed test will generally disconnect from the server; a
180    new ccs() will reconnect, if the server is still alive.
181    """
182    def __init__(self, name, tstate):
183        self.name = name
184        self.status = None
185        self.detail = None
186        self.tstate = tstate
187        self._shutdown = None
188        self._autoclunk = None
189        self._acconn = None
190
191    def auto_disconnect(self, conn):
192        self._shutdown = conn
193
194    def succ(self, detail=None):
195        "set success status"
196        self.status = 'SUCC'
197        self.detail = detail
198        raise TCDone()
199
200    def fail(self, detail):
201        "set failure status"
202        self.status = 'FAIL'
203        self.detail = detail
204        raise TCDone()
205
206    def skip(self, detail=None):
207        "set skip status"
208        self.status = 'SKIP'
209        self.detail = detail
210        raise TCDone()
211
212    def autoclunk(self, fid):
213        "mark fid to be closed/clunked on test exit"
214        if self._acconn is None:
215            raise ValueError('autoclunk: no _acconn')
216        self._autoclunk.append(fid)
217
218    def trace(self, msg, *args, **kwargs):
219        "add tracing info to log-file output"
220        level = kwargs.pop('level', logging.INFO)
221        self.tstate.logger.log(level, '      ' + msg, *args, **kwargs)
222
223    def ccs(self):
224        "call tstate ccs, turn socket.error connect failure into test fail"
225        try:
226            self.detail = 'connecting'
227            ret = self.tstate.ccs()
228            self.detail = None
229            self._acconn = ret
230            return ret
231        except socket.error as err:
232            self.fail(str(err))
233
234    def __enter__(self):
235        self.tstate.logger.log(logging.DEBUG, 'ENTER: %s', self.name)
236        self._autoclunk = []
237        return self
238
239    def __exit__(self, exc_type, exc_val, exc_tb):
240        tstate = self.tstate
241        eat_exc = False
242        tb_detail = None
243        if exc_type is TCDone:
244            # we exited with succ, fail, or skip
245            eat_exc = True
246            exc_type = None
247        if exc_type is not None:
248            if self.status is None:
249                self.status = 'EXCP'
250            else:
251                self.status += ' EXC'
252            if exc_type == TEError:
253                # timeout/eof - best guess is that we crashed the server!
254                eat_exc = True
255                tb_detail = ['timeout or EOF']
256            elif exc_type in (socket.error, RemoteError, LocalError):
257                eat_exc = True
258                tb_detail = traceback.format_exception(exc_type, exc_val,
259                                                       exc_tb)
260            level = logging.ERROR
261            tstate.failures += 1
262            tstate.exceptions += 1
263        else:
264            if self.status is None:
265                self.status = 'SUCC'
266            if self.status == 'SUCC':
267                level = logging.INFO
268                tstate.successes += 1
269            elif self.status == 'SKIP':
270                level = logging.INFO
271                tstate.skips += 1
272            else:
273                level = logging.ERROR
274                tstate.failures += 1
275        tstate.logger.log(level, '%s: %s', self.status, self.name)
276        if self.detail:
277            tstate.logger.log(level, '      detail: %s', self.detail)
278        if tb_detail:
279            for line in tb_detail:
280                tstate.logger.log(level, '      %s', line.rstrip())
281        for fid in self._autoclunk:
282            self._acconn.clunk(fid, ignore_error=True)
283        if self._shutdown:
284            self._shutdown.shutdown()
285        return eat_exc
286
287def main():
288    "the usual main"
289    parser = argparse.ArgumentParser(description='run tests against a server')
290
291    parser.add_argument('-c', '--config',
292        action='append',
293        help='specify additional file(s) to read (beyond testconf.ini)')
294
295    args = parser.parse_args()
296    config = configparser.SafeConfigParser()
297    # use case sensitive keys
298    config.optionxform = str
299
300    try:
301        with open('testconf.ini', 'r') as stream:
302            config.readfp(stream)
303    except (OSError, IOError) as err:
304        sys.exit(str(err))
305    if args.config:
306        ok = config.read(args.config)
307        failed = set(ok) - set(args.config)
308        if len(failed):
309            nfailed = len(failed)
310            word = 'files' if nfailed > 1 else 'file'
311            failed = ', '.join(failed)
312            print('failed to read {0} {1}: {2}'.format(nfailed, word, failed))
313            sys.exit(1)
314
315    logging.basicConfig(level=config.get('client', 'loglevel').upper())
316    logger = logging.getLogger(__name__)
317    tstate = TestState()
318    tstate.logger = logger
319    tstate.config = config
320
321    server = config.get('client', 'server')
322    port = config.getint('client', 'port')
323    proto = config.get('client', 'protocol')
324    may_downgrade = config.getboolean('client', 'may_downgrade')
325    timeout = config.getfloat('client', 'timeout')
326
327    tstate.stop = True # unless overwritten below
328    with TestCase('send bad packet', tstate) as tc:
329        tc.detail = 'connecting to {0}:{1}'.format(server, port)
330        try:
331            conn = p9conn.P9SockIO(logger, server=server, port=port)
332        except socket.error as err:
333            tc.fail('cannot connect at all (server down?)')
334        tc.auto_disconnect(conn)
335        tc.detail = None
336        pkt = struct.pack('<I', 256);
337        conn.write(pkt)
338        # ignore reply if any, we're just trying to trip the server
339        tstate.stop = False
340        tc.succ()
341
342    if not tstate.stop:
343        tstate.mkclient = functools.partial(p9conn.P9Client, logger,
344                                           timeout, proto, may_downgrade,
345                                           server=server, port=port)
346        tstate.stop = True
347        with TestCase('send bad Tversion', tstate) as tc:
348            try:
349                clnt = tstate.mkclient()
350            except socket.error as err:
351                tc.fail('can no longer connect, did bad pkt crash server?')
352            tc.auto_disconnect(clnt)
353            clnt.set_monkey('version', b'wrongo, fishbreath!')
354            tc.detail = 'connecting'
355            try:
356                clnt.connect()
357            except RemoteError as err:
358                tstate.stop = False
359                tc.succ(err.args[0])
360            tc.fail('server accepted a bad Tversion')
361
362    if not tstate.stop:
363        # All NUL characters in strings are invalid.
364        with TestCase('send illegal NUL in Tversion', tstate) as tc:
365            clnt = tstate.mkclient()
366            tc.auto_disconnect(clnt)
367            clnt.set_monkey('version', b'9P2000\0')
368            # Forcibly allow downgrade so that Tversion
369            # succeeds if they ignore the \0.
370            clnt.may_downgrade = True
371            tc.detail = 'connecting'
372            try:
373                clnt.connect()
374            except (TEError, RemoteError) as err:
375                tc.succ(err.args[0])
376            tc.fail('server accepted NUL in Tversion')
377
378    if not tstate.stop:
379        with TestCase('connect normally', tstate) as tc:
380            tc.detail = 'connecting'
381            try:
382                tstate.ccc()
383            except RemoteError as err:
384                # can't test any further, but this might be success
385                tstate.stop = True
386                if 'they only support version' in err.args[0]:
387                    tc.succ(err.args[0])
388                tc.fail(err.args[0])
389            tc.succ()
390
391    if not tstate.stop:
392        with TestCase('attach with bad afid', tstate) as tc:
393            clnt = tstate.ccc()[0]
394            section = 'attach-with-bad-afid'
395            aname = getconf(tstate.config, section, 'aname', '')
396            uname = getconf(tstate.config, section, 'uname', '')
397            if clnt.proto > protocol.plain:
398                n_uname = getint(tstate.config, section, 'n_uname', 1001)
399            else:
400                n_uname = None
401            try:
402                clnt.attach(afid=42, aname=aname, uname=uname, n_uname=n_uname)
403            except RemoteError as err:
404                tc.succ(err.args[0])
405            tc.dcc()
406            tc.fail('bad attach afid not rejected')
407
408    try:
409        if not tstate.stop:
410            # Various Linux tests need gids.  Just get them for everyone.
411            tstate.gid = getint(tstate.config, 'client', 'gid', 0)
412            more_test_cases(tstate)
413    finally:
414        tstate.dcc()
415
416    n_tests = tstate.successes + tstate.failures
417    print('summary:')
418    if tstate.successes:
419        print('{0}/{1} tests succeeded'.format(tstate.successes, n_tests))
420    if tstate.failures:
421        print('{0}/{1} tests failed'.format(tstate.failures, n_tests))
422    if tstate.skips:
423        print('{0} {1} skipped'.format(tstate.skips,
424                                       pluralize(tstate.skips,
425                                                 'test', 'tests')))
426    if tstate.exceptions:
427        print('{0} {1} occurred'.format(tstate.exceptions,
428                                       pluralize(tstate.exceptions,
429                                                 'exception', 'exceptions')))
430    if tstate.stop:
431        print('tests stopped early')
432    return 1 if tstate.stop or tstate.exceptions or tstate.failures else 0
433
434def more_test_cases(tstate):
435    "run cases that can only proceed if connecting works at all"
436    with TestCase('attach normally', tstate) as tc:
437        tc.ccs()
438        tc.succ()
439    if tstate.stop:
440        return
441
442    # Empty string is not technically illegal.  It's not clear
443    # whether it should be accepted or rejected.  However, it
444    # used to crash the server entirely, so it's a desirable
445    # test case.
446    with TestCase('empty string in Twalk request', tstate) as tc:
447        clnt = tc.ccs()
448        try:
449            fid, qid = clnt.lookup(clnt.rootfid, [b''])
450        except RemoteError as err:
451            tc.succ(err.args[0])
452        clnt.clunk(fid)
453        tc.succ('note: empty Twalk component name not rejected')
454
455    # Name components may not contain /
456    with TestCase('embedded / in lookup component name', tstate) as tc:
457        clnt = tc.ccs()
458        try:
459            fid, qid = clnt.lookup(clnt.rootfid, [b'/'])
460            tc.autoclunk(fid)
461        except RemoteError as err:
462            tc.succ(err.args[0])
463        tc.fail('/ in lookup component name not rejected')
464
465    # Proceed from a clean tree.  As a side effect, this also tests
466    # either the old style readdir (read() on a directory fid) or
467    # the dot-L readdir().
468    #
469    # The test case will fail if we don't have permission to remove
470    # some file(s).
471    with TestCase('clean up tree (readdir+remove)', tstate) as tc:
472        clnt = tc.ccs()
473        fset = clnt.uxreaddir(b'/')
474        fset = [i for i in fset if i != '.' and i != '..']
475        tc.trace("what's there initially: {0!r}".format(fset))
476        try:
477            clnt.uxremove(b'/', force=False, recurse=True)
478        except RemoteError as err:
479            tc.trace('failed to read or clean up tree', level=logging.ERROR)
480            tc.trace('this might be a permissions error', level=logging.ERROR)
481            tstate.stop = True
482            tc.fail(str(err))
483        fset = clnt.uxreaddir(b'/')
484        fset = [i for i in fset if i != '.' and i != '..']
485        tc.trace("what's left after removing everything: {0!r}".format(fset))
486        if fset:
487            tstate.stop = True
488            tc.trace('note: could be a permissions error', level=logging.ERROR)
489            tc.fail('/ not empty after removing all: {0!r}'.format(fset))
490        tc.succ()
491    if tstate.stop:
492        return
493
494    # Name supplied to create, mkdir, etc, may not contain /.
495    # Note that this test may fail for the wrong reason if /dir
496    # itself does not already exist, so first let's make /dir.
497    only_dotl = getbool(tstate.config, 'client', 'only_dotl', False)
498    with TestCase('mkdir', tstate) as tc:
499        clnt = tc.ccs()
500        if only_dotl and not clnt.supports(protocol.td.Tmkdir):
501            tc.skip('cannot test dot-L mkdir on {0}'.format(clnt.proto))
502        try:
503            fid, qid = clnt.uxlookup(b'/dir', None)
504            tc.autoclunk(fid)
505            tstate.stop = True
506            tc.fail('found existing /dir after cleaning tree')
507        except RemoteError as err:
508            # we'll just assume it's "no such file or directory"
509            pass
510        if only_dotl:
511            qid = clnt.mkdir(clnt.rootfid, b'dir', 0o777, tstate.gid)
512        else:
513            qid, _ = clnt.create(clnt.rootfid, b'dir',
514                                 protocol.td.DMDIR | 0o777,
515                                 protocol.td.OREAD)
516        if qid.type != protocol.td.QTDIR:
517            tstate.stop = True
518            tc.fail('creating /dir: result is not a directory')
519        tc.trace('now attempting to create /dir/sub the wrong way')
520        try:
521            if only_dotl:
522                qid = clnt.mkdir(clnt.rootfid, b'dir/sub', 0o777, tstate.gid)
523            else:
524                qid, _ = clnt.create(clnt.rootfid, b'dir/sub',
525                                     protocol.td.DMDIR | 0o777,
526                                     protocol.td.OREAD)
527            # it's not clear what happened on the server at this point!
528            tc.trace("creating dir/sub (with embedded '/') should have "
529                     'failed but did not')
530            tstate.stop = True
531            fset = clnt.uxreaddir(b'/dir')
532            if 'sub' in fset:
533                tc.trace('(found our dir/sub detritus)')
534                clnt.uxremove(b'dir/sub', force=True)
535                fset = clnt.uxreaddir(b'/dir')
536                if 'sub' not in fset:
537                    tc.trace('(successfully removed our dir/sub detritus)')
538                    tstate.stop = False
539            tc.fail('created dir/sub as single directory with embedded slash')
540        except RemoteError as err:
541            # we'll just assume it's the right kind of error
542            tc.trace('invalid path dir/sub failed with: %s', str(err))
543            tc.succ('embedded slash in mkdir correctly refused')
544    if tstate.stop:
545        return
546
547    with TestCase('getattr/setattr', tstate) as tc:
548        # This test is not really thorough enough, need to test
549        # all combinations of settings.  Should also test that
550        # old values are restored on failure, although it is not
551        # clear how to trigger failures.
552        clnt = tc.ccs()
553        if not clnt.supports(protocol.td.Tgetattr):
554            tc.skip('%s does not support Tgetattr', clnt)
555        fid, _, _, _ = clnt.uxopen(b'/dir/file', os.O_CREAT | os.O_RDWR, 0o666,
556            gid=tstate.gid)
557        tc.autoclunk(fid)
558        written = clnt.write(fid, 0, 'bytes\n')
559        if written != 6:
560            tc.trace('expected to write 6 bytes, actually wrote %d', written,
561                     level=logging.WARN)
562        attrs = clnt.Tgetattr(fid)
563        #tc.trace('getattr: after write, before setattr: got %s', attrs)
564        if attrs.size != written:
565            tc.fail('getattr: expected size=%d, got size=%d',
566                    written, attrs.size)
567        # now truncate, set mtime to (3,14), and check result
568        set_time_to = p9conn.Timespec(sec=0, nsec=140000000)
569        clnt.Tsetattr(fid, size=0, mtime=set_time_to)
570        attrs = clnt.Tgetattr(fid)
571        #tc.trace('getattr: after setattr: got %s', attrs)
572        if attrs.mtime.sec != set_time_to.sec or attrs.size != 0:
573            tc.fail('setattr: expected to get back mtime.sec={0}, size=0; '
574                    'got mtime.sec={1}, size='
575                    '{1}'.format(set_time_to.sec, attrs.mtime.sec, attrs.size))
576        # nsec is not as stable but let's check
577        if attrs.mtime.nsec != set_time_to.nsec:
578            tc.trace('setattr: expected to get back mtime_nsec=%d; '
579                     'got %d', set_time_to.nsec, mtime_nsec)
580        tc.succ('able to set and see size and mtime')
581
582    # this test should be much later, but we know the current
583    # server is broken...
584    with TestCase('rename adjusts other fids', tstate) as tc:
585        clnt = tc.ccs()
586        dirfid, _ = clnt.uxlookup(b'/dir')
587        tc.autoclunk(dirfid)
588        clnt.uxmkdir(b'd1', 0o777, tstate.gid, startdir=dirfid)
589        clnt.uxmkdir(b'd1/sub', 0o777, tstate.gid, startdir=dirfid)
590        d1fid, _ = clnt.uxlookup(b'd1', dirfid)
591        tc.autoclunk(d1fid)
592        subfid, _ = clnt.uxlookup(b'sub', d1fid)
593        tc.autoclunk(subfid)
594        fid, _, _, _ = clnt.uxopen(b'file', os.O_CREAT | os.O_RDWR,
595                                   0o666, startdir=subfid, gid=tstate.gid)
596        tc.autoclunk(fid)
597        written = clnt.write(fid, 0, 'filedata\n')
598        if written != 9:
599            tc.trace('expected to write 9 bytes, actually wrote %d', written,
600                     level=logging.WARN)
601        # Now if we rename /dir/d1 to /dir/d2, the fids for both
602        # sub/file and sub itself should still be usable.  This
603        # holds for both Trename (Linux only) and Twstat based
604        # rename ops.
605        #
606        # Note that some servers may cache some number of files and/or
607        # diretories held open, so we should open many fids to wipe
608        # out the cache (XXX notyet).
609        if clnt.supports(protocol.td.Trename):
610            clnt.rename(d1fid, dirfid, name=b'd2')
611        else:
612            clnt.wstat(d1fid, name=b'd2')
613        try:
614            rofid, _, _, _ = clnt.uxopen(b'file', os.O_RDONLY, startdir=subfid)
615            clnt.clunk(rofid)
616        except RemoteError as err:
617            tc.fail('open file in renamed dir/d2/sub: {0}'.format(err))
618        tc.succ()
619
620    # Even if xattrwalk is supported by the protocol, it's optional
621    # on the server.
622    with TestCase('xattrwalk', tstate) as tc:
623        clnt = tc.ccs()
624        if not clnt.supports(protocol.td.Txattrwalk):
625            tc.skip('{0} does not support Txattrwalk'.format(clnt))
626        dirfid, _ = clnt.uxlookup(b'/dir')
627        tc.autoclunk(dirfid)
628        try:
629            # need better tests...
630            attrfid, size = clnt.xattrwalk(dirfid)
631            tc.autoclunk(attrfid)
632            data = clnt.read(attrfid, 0, size)
633            tc.trace('xattrwalk with no name: data=%r', data)
634            tc.succ('xattrwalk size={0} datalen={1}'.format(size, len(data)))
635        except RemoteError as err:
636            tc.trace('xattrwalk on /dir: {0}'.format(err))
637        tc.succ('xattrwalk apparently not implemented')
638
639if __name__ == '__main__':
640    try:
641        sys.exit(main())
642    except KeyboardInterrupt:
643        sys.exit('\nInterrupted')
644