xref: /freebsd/contrib/bmake/mk/meta2deps.py (revision a98ff317388a00b992f1bf8404dee596f9383f5e)
1#!/usr/bin/env python
2
3"""
4This script parses each "meta" file and extracts the
5information needed to deduce build and src dependencies.
6
7It works much the same as the original shell script, but is
8*much* more efficient.
9
10The parsing work is handled by the class MetaFile.
11We only pay attention to a subset of the information in the
12"meta" files.  Specifically:
13
14'CWD'	to initialize our notion.
15
16'C'	to track chdir(2) on a per process basis
17
18'R'	files read are what we really care about.
19	directories read, provide a clue to resolving
20	subsequent relative paths.  That is if we cannot find
21	them relative to 'cwd', we check relative to the last
22	dir read.
23
24'W'	files opened for write or read-write,
25	for filemon V3 and earlier.
26
27'E'	files executed.
28
29'L'	files linked
30
31'V'	the filemon version, this record is used as a clue
32	that we have reached the interesting bit.
33
34"""
35
36"""
37RCSid:
38	$Id: meta2deps.py,v 1.13 2013/05/11 05:16:26 sjg Exp $
39
40	Copyright (c) 2011-2013, Juniper Networks, Inc.
41	All rights reserved.
42
43	Redistribution and use in source and binary forms, with or without
44	modification, are permitted provided that the following conditions
45	are met:
46	1. Redistributions of source code must retain the above copyright
47	   notice, this list of conditions and the following disclaimer.
48	2. Redistributions in binary form must reproduce the above copyright
49	   notice, this list of conditions and the following disclaimer in the
50	   documentation and/or other materials provided with the distribution.
51
52	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
53	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
54	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
55	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
56	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
57	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
58	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
59	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
60	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
61	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
62	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
63
64"""
65
66import os, re, sys
67
68def getv(dict, key, d=None):
69    """Lookup key in dict and return value or the supplied default."""
70    if key in dict:
71        return dict[key]
72    return d
73
74def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr):
75    """
76    Return an absolute path, resolving via cwd or last_dir if needed.
77    """
78    if path.endswith('/.'):
79        path = path[0:-2]
80    if path[0] == '/':
81        return path
82    if path == '.':
83        return cwd
84    if path.startswith('./'):
85        return cwd + path[1:]
86    if last_dir == cwd:
87        last_dir = None
88    for d in [last_dir, cwd]:
89        if not d:
90            continue
91        p = '/'.join([d,path])
92        if debug > 2:
93            print >> debug_out, "looking for:", p,
94        if not os.path.exists(p):
95            if debug > 2:
96                print >> debug_out, "nope"
97            p = None
98            continue
99        if debug > 2:
100            print >> debug_out, "found:", p
101        return p
102    return None
103
104def abspath(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr):
105    """
106    Return an absolute path, resolving via cwd or last_dir if needed.
107    this gets called a lot, so we try to avoid calling realpath
108    until we know we have something.
109    """
110    path = resolve(path, cwd, last_dir, debug, debug_out)
111    if path and (path.find('./') > 0 or
112                 path.endswith('/..') or
113                 os.path.islink(path)):
114        return os.path.realpath(path)
115    return path
116
117def sort_unique(list, cmp=None, key=None, reverse=False):
118    list.sort(cmp, key, reverse)
119    nl = []
120    le = None
121    for e in list:
122        if e == le:
123            continue
124        nl.append(e)
125    return nl
126
127def add_trims(x):
128    return ['/' + x + '/',
129            '/' + x,
130            x + '/',
131            x]
132
133class MetaFile:
134    """class to parse meta files generated by bmake."""
135
136    conf = None
137    dirdep_re = None
138    host_target = None
139    srctops = []
140    objroots = []
141
142    seen = {}
143    obj_deps = []
144    src_deps = []
145    file_deps = []
146
147    def __init__(self, name, conf={}):
148        """if name is set we will parse it now.
149        conf can have the follwing keys:
150
151        SRCTOPS	list of tops of the src tree(s).
152
153        CURDIR	the src directory 'bmake' was run from.
154
155        RELDIR	the relative path from SRCTOP to CURDIR
156
157        MACHINE	the machine we built for.
158        	set to 'none' if we are not cross-building.
159		More specifically if machine cannot be deduced from objdirs.
160
161        TARGET_SPEC
162        	Sometimes MACHINE isn't enough.
163
164        HOST_TARGET
165		when we build for the psuedo machine 'host'
166		the object tree uses HOST_TARGET rather than MACHINE.
167
168        OBJROOTS a list of the common prefix for all obj dirs it might
169		end in '/' or '-'.
170
171        DPDEPS	names an optional file to which per file dependencies
172		will be appended.
173		For example if 'some/path/foo.h' is read from SRCTOP
174		then 'DPDEPS_some/path/foo.h +=' "RELDIR" is output.
175		This can allow 'bmake' to learn all the dirs within
176 		the tree that depend on 'foo.h'
177
178        debug	desired debug level
179
180        debug_out open file to send debug output to (sys.stderr)
181
182        """
183
184        self.name = name
185        self.debug = getv(conf, 'debug', 0)
186        self.debug_out = getv(conf, 'debug_out', sys.stderr)
187
188        self.machine = getv(conf, 'MACHINE', '')
189        self.machine_arch = getv(conf, 'MACHINE_ARCH', '')
190        self.target_spec = getv(conf, 'TARGET_SPEC', '')
191        self.curdir = getv(conf, 'CURDIR')
192        self.reldir = getv(conf, 'RELDIR')
193        self.dpdeps = getv(conf, 'DPDEPS')
194
195        if not self.conf:
196            # some of the steps below we want to do only once
197            self.conf = conf
198            self.host_target = getv(conf, 'HOST_TARGET')
199            for srctop in getv(conf, 'SRCTOPS', []):
200                if srctop[-1] != '/':
201                    srctop += '/'
202                if not srctop in self.srctops:
203                    self.srctops.append(srctop)
204                _srctop = os.path.realpath(srctop)
205                if _srctop[-1] != '/':
206                    _srctop += '/'
207                if not _srctop in self.srctops:
208                    self.srctops.append(_srctop)
209
210            trim_list = add_trims(self.machine)
211            if self.machine == 'host':
212                trim_list += add_trims(self.host_target)
213            if self.target_spec:
214                trim_list += add_trims(self.target_spec)
215
216            for objroot in getv(conf, 'OBJROOTS', []):
217                for e in trim_list:
218                    if objroot.endswith(e):
219                        # this is not what we want - fix it
220                        objroot = objroot[0:-len(e)]
221                        if e.endswith('/'):
222                            objroot += '/'
223                if not objroot in self.objroots:
224                    self.objroots.append(objroot)
225                    _objroot = os.path.realpath(objroot)
226                    if objroot[-1] == '/':
227                        _objroot += '/'
228                    if not _objroot in self.objroots:
229                        self.objroots.append(_objroot)
230
231            # we want the longest match
232            self.srctops.sort(reverse=True)
233            self.objroots.sort(reverse=True)
234
235            if self.debug:
236                print >> self.debug_out, "host_target=", self.host_target
237                print >> self.debug_out, "srctops=", self.srctops
238                print >> self.debug_out, "objroots=", self.objroots
239
240            self.dirdep_re = re.compile(r'([^/]+)/(.+)')
241
242        if self.dpdeps and not self.reldir:
243            if self.debug:
244                print >> self.debug_out, "need reldir:",
245            if self.curdir:
246                srctop = self.find_top(self.curdir, self.srctops)
247                if srctop:
248                    self.reldir = self.curdir.replace(srctop,'')
249                    if self.debug:
250                        print >> self.debug_out, self.reldir
251            if not self.reldir:
252                self.dpdeps = None      # we cannot do it?
253
254        self.cwd = os.getcwd()          # make sure this is initialized
255
256        if name:
257            self.parse()
258
259    def reset(self):
260        """reset state if we are being passed meta files from multiple directories."""
261        self.seen = {}
262        self.obj_deps = []
263        self.src_deps = []
264        self.file_deps = []
265
266    def dirdeps(self, sep='\n'):
267        """return DIRDEPS"""
268        return sep.strip() + sep.join(self.obj_deps)
269
270    def src_dirdeps(self, sep='\n'):
271        """return SRC_DIRDEPS"""
272        return sep.strip() + sep.join(self.src_deps)
273
274    def file_depends(self, out=None):
275        """Append DPDEPS_${file} += ${RELDIR}
276        for each file we saw, to the output file."""
277        if not self.reldir:
278            return None
279        for f in sort_unique(self.file_deps):
280            print >> out, 'DPDEPS_%s += %s' % (f, self.reldir)
281
282    def seenit(self, dir):
283        """rememer that we have seen dir."""
284        self.seen[dir] = 1
285
286    def add(self, list, data, clue=''):
287        """add data to list if it isn't already there."""
288        if data not in list:
289            list.append(data)
290            if self.debug:
291                print >> self.debug_out, "%s: %sAdd: %s" % (self.name, clue, data)
292
293    def find_top(self, path, list):
294        """the logical tree may be split accross multiple trees"""
295        for top in list:
296            if path.startswith(top):
297                if self.debug > 2:
298                    print >> self.debug_out, "found in", top
299                return top
300        return None
301
302    def find_obj(self, objroot, dir, path, input):
303        """return path within objroot, taking care of .dirdep files"""
304        ddep = None
305        for ddepf in [path + '.dirdep', dir + '/.dirdep']:
306            if not ddep and os.path.exists(ddepf):
307                ddep = open(ddepf, 'rb').readline().strip('# \n')
308                if self.debug > 1:
309                    print >> self.debug_out, "found %s: %s\n" % (ddepf, ddep)
310                if ddep.endswith(self.machine):
311                    ddep = ddep[0:-(1+len(self.machine))]
312                elif self.target_spec and ddep.endswith(self.target_spec):
313                    ddep = ddep[0:-(1+len(self.target_spec))]
314
315        if not ddep:
316            # no .dirdeps, so remember that we've seen the raw input
317            self.seenit(input)
318            self.seenit(dir)
319            if self.machine == 'none':
320                if dir.startswith(objroot):
321                    return dir.replace(objroot,'')
322                return None
323            m = self.dirdep_re.match(dir.replace(objroot,''))
324            if m:
325                ddep = m.group(2)
326                dmachine = m.group(1)
327                if dmachine != self.machine:
328                    if not (self.machine == 'host' and
329                            dmachine == self.host_target):
330                        if self.debug > 2:
331                            print >> self.debug_out, "adding .%s to %s" % (dmachine, ddep)
332                        ddep += '.' + dmachine
333
334        return ddep
335
336    def parse(self, name=None, file=None):
337        """A meta file looks like:
338
339	# Meta data file "path"
340	CMD "command-line"
341	CWD "cwd"
342	TARGET "target"
343	-- command output --
344	-- filemon acquired metadata --
345	# buildmon version 3
346	V 3
347	C "pid" "cwd"
348	E "pid" "path"
349        F "pid" "child"
350	R "pid" "path"
351	W "pid" "path"
352	X "pid" "status"
353        D "pid" "path"
354        L "pid" "src" "target"
355        M "pid" "old" "new"
356        S "pid" "path"
357        # Bye bye
358
359        We go to some effort to avoid processing a dependency more than once.
360        Of the above record types only C,E,F,L,R,V and W are of interest.
361        """
362
363        version = 0                     # unknown
364        if name:
365            self.name = name;
366        if file:
367            f = file
368            cwd = last_dir = self.cwd
369        else:
370            f = open(self.name, 'rb')
371        skip = True
372        pid_cwd = {}
373        pid_last_dir = {}
374        last_pid = 0
375
376        if self.curdir:
377            self.seenit(self.curdir)    # we ignore this
378
379        interesting = 'CEFLRV'
380        for line in f:
381            # ignore anything we don't care about
382            if not line[0] in interesting:
383                continue
384            if self.debug > 2:
385                print >> self.debug_out, "input:", line,
386            w = line.split()
387
388            if skip:
389                if w[0] == 'V':
390                    skip = False
391                    version = int(w[1])
392                    """
393                    if version < 4:
394                        # we cannot ignore 'W' records
395                        # as they may be 'rw'
396                        interesting += 'W'
397                    """
398                elif w[0] == 'CWD':
399                    self.cwd = cwd = last_dir = w[1]
400                    self.seenit(cwd)    # ignore this
401                    if self.debug:
402                        print >> self.debug_out, "%s: CWD=%s" % (self.name, cwd)
403                continue
404
405            pid = int(w[1])
406            if pid != last_pid:
407                if last_pid:
408                    pid_cwd[last_pid] = cwd
409                    pid_last_dir[last_pid] = last_dir
410                cwd = getv(pid_cwd, pid, self.cwd)
411                last_dir = getv(pid_last_dir, pid, self.cwd)
412                last_pid = pid
413
414            # process operations
415            if w[0] == 'F':
416                npid = int(w[2])
417                pid_cwd[npid] = cwd
418                pid_last_dir[npid] = cwd
419                last_pid = npid
420                continue
421            elif w[0] == 'C':
422                cwd = abspath(w[2], cwd, None, self.debug, self.debug_out)
423                if cwd.endswith('/.'):
424                    cwd = cwd[0:-2]
425                last_dir = cwd
426                if self.debug > 1:
427                    print >> self.debug_out, "cwd=", cwd
428                continue
429
430            if w[2] in self.seen:
431                if self.debug > 2:
432                    print >> self.debug_out, "seen:", w[2]
433                continue
434            # file operations
435            if w[0] in 'ML':
436                path = w[2].strip("'")
437            else:
438                path = w[2]
439            # we are never interested in .dirdep files as dependencies
440            if path.endswith('.dirdep'):
441                continue
442            # we don't want to resolve the last component if it is
443            # a symlink
444            path = resolve(path, cwd, last_dir, self.debug, self.debug_out)
445            if not path:
446                continue
447            dir,base = os.path.split(path)
448            if dir in self.seen:
449                if self.debug > 2:
450                    print >> self.debug_out, "seen:", dir
451                continue
452            # we can have a path in an objdir which is a link
453            # to the src dir, we may need to add dependencies for each
454            rdir = dir
455            dir = abspath(dir, cwd, last_dir, self.debug, self.debug_out)
456            if rdir == dir or rdir.find('./') > 0:
457                rdir = None
458            # now put path back together
459            path = '/'.join([dir,base])
460            if self.debug > 1:
461                print >> self.debug_out, "raw=%s rdir=%s dir=%s path=%s" % (w[2], rdir, dir, path)
462            if w[0] in 'SRWL':
463                if w[0] == 'W' and path.endswith('.dirdep'):
464                    continue
465                if path in [last_dir, cwd, self.cwd, self.curdir]:
466                    if self.debug > 1:
467                        print >> self.debug_out, "skipping:", path
468                    continue
469                if os.path.isdir(path):
470                    if w[0] in 'RW':
471                        last_dir = path;
472                    if self.debug > 1:
473                        print >> self.debug_out, "ldir=", last_dir
474                    continue
475
476            if w[0] in 'REWML':
477                # finally, we get down to it
478                if dir == self.cwd or dir == self.curdir:
479                    continue
480                srctop = self.find_top(path, self.srctops)
481                if srctop:
482                    if self.dpdeps:
483                        self.add(self.file_deps, path.replace(srctop,''), 'file')
484                    self.add(self.src_deps, dir.replace(srctop,''), 'src')
485                    self.seenit(w[2])
486                    self.seenit(dir)
487                    if rdir and not rdir.startswith(srctop):
488                        dir = rdir      # for below
489                        rdir = None
490                    else:
491                        continue
492
493                objroot = None
494                for dir in [dir,rdir]:
495                    if not dir:
496                        continue
497                    objroot = self.find_top(dir, self.objroots)
498                    if objroot:
499                        break
500                if objroot:
501                    ddep = self.find_obj(objroot, dir, path, w[2])
502                    if ddep:
503                        self.add(self.obj_deps, ddep, 'obj')
504                else:
505                    # don't waste time looking again
506                    self.seenit(w[2])
507                    self.seenit(dir)
508        if not file:
509            f.close()
510
511
512def main(argv, klass=MetaFile, xopts='', xoptf=None):
513    """Simple driver for class MetaFile.
514
515    Usage:
516    	script [options] [key=value ...] "meta" ...
517
518    Options and key=value pairs contribute to the
519    dictionary passed to MetaFile.
520
521    -S "SRCTOP"
522		add "SRCTOP" to the "SRCTOPS" list.
523
524    -C "CURDIR"
525
526    -O "OBJROOT"
527    		add "OBJROOT" to the "OBJROOTS" list.
528
529    -m "MACHINE"
530
531    -a "MACHINE_ARCH"
532
533    -H "HOST_TARGET"
534
535    -D "DPDEPS"
536
537    -d	bumps debug level
538
539    """
540    import getopt
541
542    # import Psyco if we can
543    # it can speed things up quite a bit
544    have_psyco = 0
545    try:
546        import psyco
547        psyco.full()
548        have_psyco = 1
549    except:
550        pass
551
552    conf = {
553        'SRCTOPS': [],
554        'OBJROOTS': [],
555        }
556
557    try:
558        machine = os.environ['MACHINE']
559        if machine:
560            conf['MACHINE'] = machine
561        machine_arch = os.environ['MACHINE_ARCH']
562        if machine_arch:
563            conf['MACHINE_ARCH'] = machine_arch
564        srctop = os.environ['SB_SRC']
565        if srctop:
566            conf['SRCTOPS'].append(srctop)
567        objroot = os.environ['SB_OBJROOT']
568        if objroot:
569            conf['OBJROOTS'].append(objroot)
570    except:
571        pass
572
573    debug = 0
574    output = True
575
576    opts, args = getopt.getopt(argv[1:], 'a:dS:C:O:R:m:D:H:qT:' + xopts)
577    for o, a in opts:
578        if o == '-a':
579            conf['MACHINE_ARCH'] = a
580        elif o == '-d':
581            debug += 1
582        elif o == '-q':
583            output = False
584        elif o == '-H':
585            conf['HOST_TARGET'] = a
586        elif o == '-S':
587            if a not in conf['SRCTOPS']:
588                conf['SRCTOPS'].append(a)
589        elif o == '-C':
590            conf['CURDIR'] = a
591        elif o == '-O':
592            if a not in conf['OBJROOTS']:
593                conf['OBJROOTS'].append(a)
594        elif o == '-R':
595            conf['RELDIR'] = a
596        elif o == '-D':
597            conf['DPDEPS'] = a
598        elif o == '-m':
599            conf['MACHINE'] = a
600        elif o == '-T':
601            conf['TARGET_SPEC'] = a
602        elif xoptf:
603            xoptf(o, a, conf)
604
605    conf['debug'] = debug
606
607    # get any var=val assignments
608    eaten = []
609    for a in args:
610        if a.find('=') > 0:
611            k,v = a.split('=')
612            if k in ['SRCTOP','OBJROOT','SRCTOPS','OBJROOTS']:
613                if k == 'SRCTOP':
614                    k = 'SRCTOPS'
615                elif k == 'OBJROOT':
616                    k = 'OBJROOTS'
617                if v not in conf[k]:
618                    conf[k].append(v)
619            else:
620                conf[k] = v
621            eaten.append(a)
622            continue
623        break
624
625    for a in eaten:
626        args.remove(a)
627
628    debug_out = getv(conf, 'debug_out', sys.stderr)
629
630    if debug:
631        print >> debug_out, "config:"
632        print >> debug_out, "psyco=", have_psyco
633        for k,v in conf.items():
634            print >> debug_out, "%s=%s" % (k,v)
635
636    for a in args:
637        m = klass(a, conf)
638
639    if output:
640        print m.dirdeps()
641
642        print m.src_dirdeps('\nsrc:')
643
644        dpdeps = getv(conf, 'DPDEPS')
645        if dpdeps:
646            m.file_depends(open(dpdeps, 'wb'))
647
648    return m
649
650if __name__ == '__main__':
651    try:
652        main(sys.argv)
653    except:
654        # yes, this goes to stdout
655        print "ERROR: ", sys.exc_info()[1]
656        raise
657
658