xref: /freebsd/share/mk/meta2deps.sh (revision 924226fba12cc9a228c73b956e1b7fa24c60b055) 
1#!/bin/sh
2
3# NAME:
4#	meta2deps.sh - extract useful info from .meta files
5#
6# SYNOPSIS:
7#	meta2deps.sh SB="SB" "meta" ...
8#
9# DESCRIPTION:
10#	This script looks each "meta" file and extracts the
11#	information needed to deduce build and src dependencies.
12#
13#	To do this, we extract the 'CWD' record as well as all the
14#	syscall traces which describe 'R'ead, 'C'hdir and 'E'xec
15#	syscalls.
16#
17#	The typical meta file looks like::
18#.nf
19#
20#	# Meta data file "path"
21#	CMD "command-line"
22#	CWD "cwd"
23#	TARGET "target"
24#	-- command output --
25#	-- filemon acquired metadata --
26#	# buildmon version 2
27#	V 2
28#	E "pid" "path"
29#	R "pid" "path"
30#	C "pid" "cwd"
31#	R "pid" "path"
32#	X "pid" "status"
33#.fi
34#
35#	The fact that all the syscall entry lines start with a single
36#	character make these files quite easy to process using sed(1).
37#
38#	To simplify the logic the 'CWD' line is made to look like a
39#	normal 'C'hdir entry, and "cwd" is remembered so that it can
40#	be prefixed to any "path" which is not absolute.
41#
42#	If the "path" being read ends in '.srcrel' it is the content
43#	of (actually the first line of) that file that we are
44#	interested in.
45#
46#	Any "path" which lies outside of the sandbox "SB" is generally
47#	not of interest and is ignored.
48#
49#	The output, is a set of absolute paths with "SB" like:
50#.nf
51#
52#	$SB/obj-i386/bsd/include
53#	$SB/obj-i386/bsd/lib/csu/i386
54#	$SB/obj-i386/bsd/lib/libc
55#	$SB/src/bsd/include
56#	$SB/src/bsd/sys/i386/include
57#	$SB/src/bsd/sys/sys
58#	$SB/src/pan-release/rtsock
59#	$SB/src/pfe-shared/include/jnx
60#.fi
61#
62#	Which can then be further processed by 'gendirdeps.mk'
63#
64#	If we are passed 'DPDEPS='"dpdeps", then for each src file
65#	outside of "CURDIR" we read, we output a line like:
66#.nf
67#
68#	DPDEPS_$path += $RELDIR
69#.fi
70#
71#	with "$path" geting turned into reldir's, so that we can end
72#	up with a list of all the directories which depend on each src
73#	file in another directory.  This can allow for efficient yet
74#	complete testing of changes.
75
76
77# RCSid:
78#	$FreeBSD$
79#	$Id: meta2deps.sh,v 1.14 2020/10/02 03:11:17 sjg Exp $
80
81# Copyright (c) 2010-2013, Juniper Networks, Inc.
82# All rights reserved.
83#
84# Redistribution and use in source and binary forms, with or without
85# modification, are permitted provided that the following conditions
86# are met:
87# 1. Redistributions of source code must retain the above copyright
88#    notice, this list of conditions and the following disclaimer.
89# 2. Redistributions in binary form must reproduce the above copyright
90#    notice, this list of conditions and the following disclaimer in the
91#    documentation and/or other materials provided with the distribution.
92#
93# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
94# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
95# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
96# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
97# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
98# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
99# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
100# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
101# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
102# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
103# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
104
105meta2src() {
106    cat /dev/null "$@" |
107    sed -n '/^R .*\.[chyl]$/s,^..[0-9]* ,,p' |
108    sort -u
109}
110
111meta2dirs() {
112    cat /dev/null "$@" |
113    sed -n '/^R .*\/.*\.[a-z0-9][^\/]*$/s,^..[0-9]* \(.*\)/[^/]*$,\1,p' |
114    sort -u
115}
116
117add_list() {
118    sep=' '
119    suffix=
120    while :
121    do
122	case "$1" in
123	"|") sep="$1"; shift;;
124	-s) suffix="$2"; shift 2;;
125	*) break;;
126	esac
127    done
128    name=$1
129    shift
130    eval list="\$$name"
131    for top in "$@"
132    do
133	case "$sep$list$sep" in
134	*"$sep$top$suffix$sep"*) continue;;
135	esac
136	list="${list:+$list$sep}$top$suffix"
137    done
138    eval "$name=\"$list\""
139}
140
141_excludes_f() {
142    egrep -v "$EXCLUDES"
143}
144
145error() {
146    echo "ERROR: $@" >&2
147    exit 1
148}
149
150meta2deps() {
151    DPDEPS=
152    SRCTOPS=$SRCTOP
153    OBJROOTS=
154    EXCLUDES=
155    while :
156    do
157	case "$1" in
158	*=*) eval export "$1"; shift;;
159	-a) MACHINE_ARCH=$2; shift 2;;
160	-m) MACHINE=$2; shift 2;;
161	-C) CURDIR=$2; shift 2;;
162	-H) HOST_TARGET=$2; shift 2;;
163	-S) add_list SRCTOPS $2; shift 2;;
164	-O) add_list OBJROOTS $2; shift 2;;
165	-X) add_list EXCLUDES '|' $2; shift 2;;
166	-R) RELDIR=$2; shift 2;;
167	-T) TARGET_SPEC=$2; shift 2;;
168	*) break;;
169	esac
170    done
171
172    _th= _o=
173    case "$MACHINE" in
174    host) _ht=$HOST_TARGET;;
175    esac
176
177    for o in $OBJROOTS
178    do
179	case "$MACHINE,/$o/" in
180	host,*$HOST_TARGET*) ;;
181	*$MACHINE*|*${TARGET_SPEC:-$MACHINE}*) ;;
182	*) add_list _o $o; continue;;
183	esac
184	for x in $_ht $TARGET_SPEC $MACHINE
185	do
186	    case "$o" in
187	    "") continue;;
188	    */$x/) add_list _o ${o%$x/}; o=;;
189	    */$x) add_list _o ${o%$x}; o=;;
190	    *$x/) add_list _o ${o%$x/}; o=;;
191	    *$x) add_list _o ${o%$x}; o=;;
192	    esac
193	done
194    done
195    OBJROOTS="$_o"
196
197    case "$OBJTOP" in
198    "")
199	for o in $OBJROOTS
200	do
201	    OBJTOP=$o${TARGET_SPEC:-$MACHINE}
202	    break
203	done
204	;;
205    esac
206    src_re=
207    obj_re=
208    add_list '|' -s '/*' src_re $SRCTOPS
209    add_list '|' -s '*' obj_re $OBJROOTS
210
211    [ -z "$RELDIR" ] && unset DPDEPS
212    tf=/tmp/m2d$$-$USER
213    rm -f $tf.*
214    trap 'rm -f $tf.*; trap 0' 0
215
216    > $tf.dirdep
217    > $tf.qual
218    > $tf.srcdep
219    > $tf.srcrel
220    > $tf.dpdeps
221
222    seenit=
223    seensrc=
224    lpid=
225    case "$EXCLUDES" in
226    "") _excludes=cat;;
227    *) _excludes=_excludes_f;;
228    esac
229    # handle @list files
230    case "$@" in
231    *@[!.]*)
232	for f in "$@"
233	do
234	    case "$f" in
235	    *.meta) cat $f;;
236	    @*) xargs cat < ${f#@};;
237	    *) cat $f;;
238	    esac
239	done
240	;;
241    *) cat /dev/null "$@";;
242    esac 2> /dev/null |
243    sed -e 's,^CWD,C C,;/^[CREFLMV] /!d' -e "s,',,g" |
244    $_excludes | ( version=no
245    while read op pid path junk
246    do
247	: op=$op pid=$pid path=$path
248	# we track cwd and ldir (of interest) per pid
249	# CWD is bmake's cwd
250	case "$lpid,$pid" in
251	,C) CWD=$path cwd=$path ldir=$path
252	    if [ -z "$SB" ]; then
253		SB=`echo $CWD | sed 's,/obj.*,,'`
254	    fi
255	    SRCTOP=${SRCTOP:-$SB/src}
256	    case "$verion" in
257	    no) ;;		# ignore
258	    0) error "no filemon data";;
259	    *) ;;
260	    esac
261	    version=0
262	    continue
263	    ;;
264	$pid,$pid) ;;
265	*)
266	    case "$lpid" in
267	    "") ;;
268	    *) eval ldir_$lpid=$ldir;;
269	    esac
270	    eval ldir=\${ldir_$pid:-$CWD} cwd=\${cwd_$pid:-$CWD}
271	    lpid=$pid
272	    ;;
273	esac
274
275	case "$op,$path" in
276	V,*) version=$path; continue;;
277	W,*srcrel|*.dirdep) continue;;
278	C,*)
279	    case "$path" in
280	    /*) cwd=$path;;
281	    *) cwd=`cd $cwd/$path 2> /dev/null && /bin/pwd`;;
282	    esac
283	    # watch out for temp dirs that no longer exist
284	    test -d ${cwd:-/dev/null/no/such} || cwd=$CWD
285	    eval cwd_$pid=$cwd
286	    continue
287	    ;;
288	F,*) # $path is new pid
289	    eval cwd_$path=$cwd ldir_$path=$ldir
290	    continue
291	    ;;
292	*)  dir=${path%/*}
293	    case "$path" in
294	    $src_re|$obj_re) ;;
295	    /*/stage/*) ;;
296	    /*) continue;;
297	    *)	for path in $ldir/$path $cwd/$path
298		do
299			test -e $path && break
300		done
301		dir=${path%/*}
302		;;
303	    esac
304	    ;;
305	esac
306	# avoid repeating ourselves...
307	case "$DPDEPS,$seensrc," in
308	,*)
309	    case ",$seenit," in
310	    *,$dir,*) continue;;
311	    esac
312	    ;;
313	*,$path,*) continue;;
314	esac
315	# canonicalize if needed
316	case "/$dir/" in
317	*/../*|*/./*)
318	    rdir=$dir
319	    dir=`cd $dir 2> /dev/null && /bin/pwd`
320	    seen="$rdir,$dir"
321	    ;;
322	*)  seen=$dir;;
323	esac
324	case "$dir" in
325	${CURDIR:-.}|"") continue;;
326	$src_re)
327	    # avoid repeating ourselves...
328	    case "$DPDEPS,$seensrc," in
329	    ,*)
330		case ",$seenit," in
331		*,$dir,*) continue;;
332		esac
333		;;
334	    esac
335	    ;;
336	*)
337	    case ",$seenit," in
338	    *,$dir,*) continue;;
339	    esac
340	    ;;
341	esac
342	if [ -d $path ]; then
343	    case "$path" in
344	    */..) ldir=${dir%/*};;
345	    *) ldir=$path;;
346	    esac
347	    continue
348	fi
349	[ -f $path ] || continue
350	case "$dir" in
351	$CWD) continue;;		# ignore
352	$src_re)
353	    seenit="$seenit,$seen"
354	    echo $dir >> $tf.srcdep
355	    case "$DPDEPS,$reldir,$seensrc," in
356	    ,*) ;;
357	    *)	seensrc="$seensrc,$path"
358		echo "DPDEPS_$dir/${path##*/} += $RELDIR" >> $tf.dpdeps
359		;;
360	    esac
361	    continue
362	    ;;
363	esac
364	# if there is a .dirdep we cannot skip
365	# just because we've seen the dir before.
366	if [ -s $path.dirdep ]; then
367	    # this file contains:
368	    # '# ${RELDIR}.<machine>'
369	    echo $path.dirdep >> $tf.qual
370	    continue
371	elif [ -s $dir.dirdep ]; then
372	    echo $dir.dirdep >> $tf.qual
373	    seenit="$seenit,$seen"
374	    continue
375	fi
376	seenit="$seenit,$seen"
377	case "$dir" in
378	$obj_re)
379	    echo $dir;;
380	esac
381    done > $tf.dirdep
382    case "$version" in
383    0) error "no filemon data";;
384    esac ) || exit 1
385    _nl=echo
386    for f in $tf.dirdep $tf.qual $tf.srcdep
387    do
388	[ -s $f ] || continue
389	case $f in
390	*qual) # a list of .dirdep files
391	    # we can prefix everything with $OBJTOP to
392	    # tell gendirdeps.mk that these are
393	    # DIRDEP entries, since they are already
394	    # qualified with .<machine> as needed.
395	    # We strip .$MACHINE though
396	    xargs cat < $f | sort -u |
397	    sed "s,^# ,,;s,^,$OBJTOP/,;s,\.${TARGET_SPEC:-$MACHINE}\$,,;s,\.$MACHINE\$,,"
398	    ;;
399	*)  sort -u $f;;
400	esac
401	_nl=:
402    done
403    if [ -s $tf.dpdeps ]; then
404	case "$DPDEPS" in
405	*/*) ;;
406	*) echo > $DPDEPS;;		# the echo is needed!
407	esac
408	sort -u $tf.dpdeps |
409	sed "s,${SRCTOP}/,,;s,${SB_BACKING_SB:-$SB}/src/,," >> $DPDEPS
410    fi
411    # ensure we produce _something_ else egrep -v gets upset
412    $_nl
413}
414
415case /$0 in
416*/meta2dep*) meta2deps "$@";;
417*/meta2dirs*) meta2dirs "$@";;
418*/meta2src*) meta2src "$@";;
419esac
420