xref: /freebsd/usr.sbin/freebsd-update/freebsd-update.sh (revision 39beb93c3f8bdbf72a61fda42300b5ebed7390c8)
1#!/bin/sh
2
3#-
4# Copyright 2004-2007 Colin Percival
5# All rights reserved
6#
7# Redistribution and use in source and binary forms, with or without
8# modification, are permitted providing that the following conditions
9# are met:
10# 1. Redistributions of source code must retain the above copyright
11#    notice, this list of conditions and the following disclaimer.
12# 2. Redistributions in binary form must reproduce the above copyright
13#    notice, this list of conditions and the following disclaimer in the
14#    documentation and/or other materials provided with the distribution.
15#
16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
20# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26# POSSIBILITY OF SUCH DAMAGE.
27
28# $FreeBSD$
29
30#### Usage function -- called from command-line handling code.
31
32# Usage instructions.  Options not listed:
33# --debug	-- don't filter output from utilities
34# --no-stats	-- don't show progress statistics while fetching files
35usage () {
36	cat <<EOF
37usage: `basename $0` [options] command ... [path]
38
39Options:
40  -b basedir   -- Operate on a system mounted at basedir
41                  (default: /)
42  -d workdir   -- Store working files in workdir
43                  (default: /var/db/freebsd-update/)
44  -f conffile  -- Read configuration options from conffile
45                  (default: /etc/freebsd-update.conf)
46  -k KEY       -- Trust an RSA key with SHA256 hash of KEY
47  -r release   -- Target for upgrade (e.g., 6.2-RELEASE)
48  -s server    -- Server from which to fetch updates
49                  (default: update.FreeBSD.org)
50  -t address   -- Mail output of cron command, if any, to address
51                  (default: root)
52Commands:
53  fetch        -- Fetch updates from server
54  cron         -- Sleep rand(3600) seconds, fetch updates, and send an
55                  email if updates were found
56  upgrade      -- Fetch upgrades to FreeBSD version specified via -r option
57  install      -- Install downloaded updates or upgrades
58  rollback     -- Uninstall most recently installed updates
59  IDS          -- Compare the system against an index of "known good" files.
60EOF
61	exit 0
62}
63
64#### Configuration processing functions
65
66#-
67# Configuration options are set in the following order of priority:
68# 1. Command line options
69# 2. Configuration file options
70# 3. Default options
71# In addition, certain options (e.g., IgnorePaths) can be specified multiple
72# times and (as long as these are all in the same place, e.g., inside the
73# configuration file) they will accumulate.  Finally, because the path to the
74# configuration file can be specified at the command line, the entire command
75# line must be processed before we start reading the configuration file.
76#
77# Sound like a mess?  It is.  Here's how we handle this:
78# 1. Initialize CONFFILE and all the options to "".
79# 2. Process the command line.  Throw an error if a non-accumulating option
80#    is specified twice.
81# 3. If CONFFILE is "", set CONFFILE to /etc/freebsd-update.conf .
82# 4. For all the configuration options X, set X_saved to X.
83# 5. Initialize all the options to "".
84# 6. Read CONFFILE line by line, parsing options.
85# 7. For each configuration option X, set X to X_saved iff X_saved is not "".
86# 8. Repeat steps 4-7, except setting options to their default values at (6).
87
88CONFIGOPTIONS="KEYPRINT WORKDIR SERVERNAME MAILTO ALLOWADD ALLOWDELETE
89    KEEPMODIFIEDMETADATA COMPONENTS IGNOREPATHS UPDATEIFUNMODIFIED
90    BASEDIR VERBOSELEVEL TARGETRELEASE STRICTCOMPONENTS MERGECHANGES
91    IDSIGNOREPATHS"
92
93# Set all the configuration options to "".
94nullconfig () {
95	for X in ${CONFIGOPTIONS}; do
96		eval ${X}=""
97	done
98}
99
100# For each configuration option X, set X_saved to X.
101saveconfig () {
102	for X in ${CONFIGOPTIONS}; do
103		eval ${X}_saved=\$${X}
104	done
105}
106
107# For each configuration option X, set X to X_saved if X_saved is not "".
108mergeconfig () {
109	for X in ${CONFIGOPTIONS}; do
110		eval _=\$${X}_saved
111		if ! [ -z "${_}" ]; then
112			eval ${X}=\$${X}_saved
113		fi
114	done
115}
116
117# Set the trusted keyprint.
118config_KeyPrint () {
119	if [ -z ${KEYPRINT} ]; then
120		KEYPRINT=$1
121	else
122		return 1
123	fi
124}
125
126# Set the working directory.
127config_WorkDir () {
128	if [ -z ${WORKDIR} ]; then
129		WORKDIR=$1
130	else
131		return 1
132	fi
133}
134
135# Set the name of the server (pool) from which to fetch updates
136config_ServerName () {
137	if [ -z ${SERVERNAME} ]; then
138		SERVERNAME=$1
139	else
140		return 1
141	fi
142}
143
144# Set the address to which 'cron' output will be mailed.
145config_MailTo () {
146	if [ -z ${MAILTO} ]; then
147		MAILTO=$1
148	else
149		return 1
150	fi
151}
152
153# Set whether FreeBSD Update is allowed to add files (or directories, or
154# symlinks) which did not previously exist.
155config_AllowAdd () {
156	if [ -z ${ALLOWADD} ]; then
157		case $1 in
158		[Yy][Ee][Ss])
159			ALLOWADD=yes
160			;;
161		[Nn][Oo])
162			ALLOWADD=no
163			;;
164		*)
165			return 1
166			;;
167		esac
168	else
169		return 1
170	fi
171}
172
173# Set whether FreeBSD Update is allowed to remove files/directories/symlinks.
174config_AllowDelete () {
175	if [ -z ${ALLOWDELETE} ]; then
176		case $1 in
177		[Yy][Ee][Ss])
178			ALLOWDELETE=yes
179			;;
180		[Nn][Oo])
181			ALLOWDELETE=no
182			;;
183		*)
184			return 1
185			;;
186		esac
187	else
188		return 1
189	fi
190}
191
192# Set whether FreeBSD Update should keep existing inode ownership,
193# permissions, and flags, in the event that they have been modified locally
194# after the release.
195config_KeepModifiedMetadata () {
196	if [ -z ${KEEPMODIFIEDMETADATA} ]; then
197		case $1 in
198		[Yy][Ee][Ss])
199			KEEPMODIFIEDMETADATA=yes
200			;;
201		[Nn][Oo])
202			KEEPMODIFIEDMETADATA=no
203			;;
204		*)
205			return 1
206			;;
207		esac
208	else
209		return 1
210	fi
211}
212
213# Add to the list of components which should be kept updated.
214config_Components () {
215	for C in $@; do
216		COMPONENTS="${COMPONENTS} ${C}"
217	done
218}
219
220# Add to the list of paths under which updates will be ignored.
221config_IgnorePaths () {
222	for C in $@; do
223		IGNOREPATHS="${IGNOREPATHS} ${C}"
224	done
225}
226
227# Add to the list of paths which IDS should ignore.
228config_IDSIgnorePaths () {
229	for C in $@; do
230		IDSIGNOREPATHS="${IDSIGNOREPATHS} ${C}"
231	done
232}
233
234# Add to the list of paths within which updates will be performed only if the
235# file on disk has not been modified locally.
236config_UpdateIfUnmodified () {
237	for C in $@; do
238		UPDATEIFUNMODIFIED="${UPDATEIFUNMODIFIED} ${C}"
239	done
240}
241
242# Add to the list of paths within which updates to text files will be merged
243# instead of overwritten.
244config_MergeChanges () {
245	for C in $@; do
246		MERGECHANGES="${MERGECHANGES} ${C}"
247	done
248}
249
250# Work on a FreeBSD installation mounted under $1
251config_BaseDir () {
252	if [ -z ${BASEDIR} ]; then
253		BASEDIR=$1
254	else
255		return 1
256	fi
257}
258
259# When fetching upgrades, should we assume the user wants exactly the
260# components listed in COMPONENTS, rather than trying to guess based on
261# what's currently installed?
262config_StrictComponents () {
263	if [ -z ${STRICTCOMPONENTS} ]; then
264		case $1 in
265		[Yy][Ee][Ss])
266			STRICTCOMPONENTS=yes
267			;;
268		[Nn][Oo])
269			STRICTCOMPONENTS=no
270			;;
271		*)
272			return 1
273			;;
274		esac
275	else
276		return 1
277	fi
278}
279
280# Upgrade to FreeBSD $1
281config_TargetRelease () {
282	if [ -z ${TARGETRELEASE} ]; then
283		TARGETRELEASE=$1
284	else
285		return 1
286	fi
287}
288
289# Define what happens to output of utilities
290config_VerboseLevel () {
291	if [ -z ${VERBOSELEVEL} ]; then
292		case $1 in
293		[Dd][Ee][Bb][Uu][Gg])
294			VERBOSELEVEL=debug
295			;;
296		[Nn][Oo][Ss][Tt][Aa][Tt][Ss])
297			VERBOSELEVEL=nostats
298			;;
299		[Ss][Tt][Aa][Tt][Ss])
300			VERBOSELEVEL=stats
301			;;
302		*)
303			return 1
304			;;
305		esac
306	else
307		return 1
308	fi
309}
310
311# Handle one line of configuration
312configline () {
313	if [ $# -eq 0 ]; then
314		return
315	fi
316
317	OPT=$1
318	shift
319	config_${OPT} $@
320}
321
322#### Parameter handling functions.
323
324# Initialize parameters to null, just in case they're
325# set in the environment.
326init_params () {
327	# Configration settings
328	nullconfig
329
330	# No configuration file set yet
331	CONFFILE=""
332
333	# No commands specified yet
334	COMMANDS=""
335}
336
337# Parse the command line
338parse_cmdline () {
339	while [ $# -gt 0 ]; do
340		case "$1" in
341		# Location of configuration file
342		-f)
343			if [ $# -eq 1 ]; then usage; fi
344			if [ ! -z "${CONFFILE}" ]; then usage; fi
345			shift; CONFFILE="$1"
346			;;
347
348		# Configuration file equivalents
349		-b)
350			if [ $# -eq 1 ]; then usage; fi; shift
351			config_BaseDir $1 || usage
352			;;
353		-d)
354			if [ $# -eq 1 ]; then usage; fi; shift
355			config_WorkDir $1 || usage
356			;;
357		-k)
358			if [ $# -eq 1 ]; then usage; fi; shift
359			config_KeyPrint $1 || usage
360			;;
361		-s)
362			if [ $# -eq 1 ]; then usage; fi; shift
363			config_ServerName $1 || usage
364			;;
365		-r)
366			if [ $# -eq 1 ]; then usage; fi; shift
367			config_TargetRelease $1 || usage
368			;;
369		-t)
370			if [ $# -eq 1 ]; then usage; fi; shift
371			config_MailTo $1 || usage
372			;;
373		-v)
374			if [ $# -eq 1 ]; then usage; fi; shift
375			config_VerboseLevel $1 || usage
376			;;
377
378		# Aliases for "-v debug" and "-v nostats"
379		--debug)
380			config_VerboseLevel debug || usage
381			;;
382		--no-stats)
383			config_VerboseLevel nostats || usage
384			;;
385
386		# Commands
387		cron | fetch | upgrade | install | rollback | IDS)
388			COMMANDS="${COMMANDS} $1"
389			;;
390
391		# Anything else is an error
392		*)
393			usage
394			;;
395		esac
396		shift
397	done
398
399	# Make sure we have at least one command
400	if [ -z "${COMMANDS}" ]; then
401		usage
402	fi
403}
404
405# Parse the configuration file
406parse_conffile () {
407	# If a configuration file was specified on the command line, check
408	# that it exists and is readable.
409	if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then
410		echo -n "File does not exist "
411		echo -n "or is not readable: "
412		echo ${CONFFILE}
413		exit 1
414	fi
415
416	# If a configuration file was not specified on the command line,
417	# use the default configuration file path.  If that default does
418	# not exist, give up looking for any configuration.
419	if [ -z "${CONFFILE}" ]; then
420		CONFFILE="/etc/freebsd-update.conf"
421		if [ ! -r "${CONFFILE}" ]; then
422			return
423		fi
424	fi
425
426	# Save the configuration options specified on the command line, and
427	# clear all the options in preparation for reading the config file.
428	saveconfig
429	nullconfig
430
431	# Read the configuration file.  Anything after the first '#' is
432	# ignored, and any blank lines are ignored.
433	L=0
434	while read LINE; do
435		L=$(($L + 1))
436		LINEX=`echo "${LINE}" | cut -f 1 -d '#'`
437		if ! configline ${LINEX}; then
438			echo "Error processing configuration file, line $L:"
439			echo "==> ${LINE}"
440			exit 1
441		fi
442	done < ${CONFFILE}
443
444	# Merge the settings read from the configuration file with those
445	# provided at the command line.
446	mergeconfig
447}
448
449# Provide some default parameters
450default_params () {
451	# Save any parameters already configured, and clear the slate
452	saveconfig
453	nullconfig
454
455	# Default configurations
456	config_WorkDir /var/db/freebsd-update
457	config_MailTo root
458	config_AllowAdd yes
459	config_AllowDelete yes
460	config_KeepModifiedMetadata yes
461	config_BaseDir /
462	config_VerboseLevel stats
463	config_StrictComponents no
464
465	# Merge these defaults into the earlier-configured settings
466	mergeconfig
467}
468
469# Set utility output filtering options, based on ${VERBOSELEVEL}
470fetch_setup_verboselevel () {
471	case ${VERBOSELEVEL} in
472	debug)
473		QUIETREDIR="/dev/stderr"
474		QUIETFLAG=" "
475		STATSREDIR="/dev/stderr"
476		DDSTATS=".."
477		XARGST="-t"
478		NDEBUG=" "
479		;;
480	nostats)
481		QUIETREDIR=""
482		QUIETFLAG=""
483		STATSREDIR="/dev/null"
484		DDSTATS=".."
485		XARGST=""
486		NDEBUG=""
487		;;
488	stats)
489		QUIETREDIR="/dev/null"
490		QUIETFLAG="-q"
491		STATSREDIR="/dev/stdout"
492		DDSTATS=""
493		XARGST=""
494		NDEBUG="-n"
495		;;
496	esac
497}
498
499# Perform sanity checks and set some final parameters
500# in preparation for fetching files.  Figure out which
501# set of updates should be downloaded: If the user is
502# running *-p[0-9]+, strip off the last part; if the
503# user is running -SECURITY, call it -RELEASE.  Chdir
504# into the working directory.
505fetch_check_params () {
506	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
507
508	_SERVERNAME_z=\
509"SERVERNAME must be given via command line or configuration file."
510	_KEYPRINT_z="Key must be given via -k option or configuration file."
511	_KEYPRINT_bad="Invalid key fingerprint: "
512	_WORKDIR_bad="Directory does not exist or is not writable: "
513
514	if [ -z "${SERVERNAME}" ]; then
515		echo -n "`basename $0`: "
516		echo "${_SERVERNAME_z}"
517		exit 1
518	fi
519	if [ -z "${KEYPRINT}" ]; then
520		echo -n "`basename $0`: "
521		echo "${_KEYPRINT_z}"
522		exit 1
523	fi
524	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
525		echo -n "`basename $0`: "
526		echo -n "${_KEYPRINT_bad}"
527		echo ${KEYPRINT}
528		exit 1
529	fi
530	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
531		echo -n "`basename $0`: "
532		echo -n "${_WORKDIR_bad}"
533		echo ${WORKDIR}
534		exit 1
535	fi
536	cd ${WORKDIR} || exit 1
537
538	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
539	# to provide an upgrade path for FreeBSD Update 1.x users, since
540	# the kernels provided by FreeBSD Update 1.x are always labelled
541	# as X.Y-SECURITY.
542	RELNUM=`uname -r |
543	    sed -E 's,-p[0-9]+,,' |
544	    sed -E 's,-SECURITY,-RELEASE,'`
545	ARCH=`uname -m`
546	FETCHDIR=${RELNUM}/${ARCH}
547	PATCHDIR=${RELNUM}/${ARCH}/bp
548
549	# Figure out what directory contains the running kernel
550	BOOTFILE=`sysctl -n kern.bootfile`
551	KERNELDIR=${BOOTFILE%/kernel}
552	if ! [ -d ${KERNELDIR} ]; then
553		echo "Cannot identify running kernel"
554		exit 1
555	fi
556
557	# Figure out what kernel configuration is running.  We start with
558	# the output of `uname -i`, and then make the following adjustments:
559	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
560	# file says "ident SMP-GENERIC", I don't know...
561	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
562	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
563	# we're running an SMP kernel.  This mis-identification is a bug
564	# which was fixed in 6.2-STABLE.
565	KERNCONF=`uname -i`
566	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
567		KERNCONF=SMP
568	fi
569	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
570		if sysctl kern.version | grep -qE '/SMP$'; then
571			KERNCONF=SMP
572		fi
573	fi
574
575	# Define some paths
576	BSPATCH=/usr/bin/bspatch
577	SHA256=/sbin/sha256
578	PHTTPGET=/usr/libexec/phttpget
579
580	# Set up variables relating to VERBOSELEVEL
581	fetch_setup_verboselevel
582
583	# Construct a unique name from ${BASEDIR}
584	BDHASH=`echo ${BASEDIR} | sha256 -q`
585}
586
587# Perform sanity checks etc. before fetching upgrades.
588upgrade_check_params () {
589	fetch_check_params
590
591	# Unless set otherwise, we're upgrading to the same kernel config.
592	NKERNCONF=${KERNCONF}
593
594	# We need TARGETRELEASE set
595	_TARGETRELEASE_z="Release target must be specified via -r option."
596	if [ -z "${TARGETRELEASE}" ]; then
597		echo -n "`basename $0`: "
598		echo "${_TARGETRELEASE_z}"
599		exit 1
600	fi
601
602	# The target release should be != the current release.
603	if [ "${TARGETRELEASE}" = "${RELNUM}" ]; then
604		echo -n "`basename $0`: "
605		echo "Cannot upgrade from ${RELNUM} to itself"
606		exit 1
607	fi
608
609	# Turning off AllowAdd or AllowDelete is a bad idea for upgrades.
610	if [ "${ALLOWADD}" = "no" ]; then
611		echo -n "`basename $0`: "
612		echo -n "WARNING: \"AllowAdd no\" is a bad idea "
613		echo "when upgrading between releases."
614		echo
615	fi
616	if [ "${ALLOWDELETE}" = "no" ]; then
617		echo -n "`basename $0`: "
618		echo -n "WARNING: \"AllowDelete no\" is a bad idea "
619		echo "when upgrading between releases."
620		echo
621	fi
622
623	# Set EDITOR to /usr/bin/vi if it isn't already set
624	: ${EDITOR:='/usr/bin/vi'}
625}
626
627# Perform sanity checks and set some final parameters in
628# preparation for installing updates.
629install_check_params () {
630	# Check that we are root.  All sorts of things won't work otherwise.
631	if [ `id -u` != 0 ]; then
632		echo "You must be root to run this."
633		exit 1
634	fi
635
636	# Check that securelevel <= 0.  Otherwise we can't update schg files.
637	if [ `sysctl -n kern.securelevel` -gt 0 ]; then
638		echo "Updates cannot be installed when the system securelevel"
639		echo "is greater than zero."
640		exit 1
641	fi
642
643	# Check that we have a working directory
644	_WORKDIR_bad="Directory does not exist or is not writable: "
645	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
646		echo -n "`basename $0`: "
647		echo -n "${_WORKDIR_bad}"
648		echo ${WORKDIR}
649		exit 1
650	fi
651	cd ${WORKDIR} || exit 1
652
653	# Construct a unique name from ${BASEDIR}
654	BDHASH=`echo ${BASEDIR} | sha256 -q`
655
656	# Check that we have updates ready to install
657	if ! [ -L ${BDHASH}-install ]; then
658		echo "No updates are available to install."
659		echo "Run '$0 fetch' first."
660		exit 1
661	fi
662	if ! [ -f ${BDHASH}-install/INDEX-OLD ] ||
663	    ! [ -f ${BDHASH}-install/INDEX-NEW ]; then
664		echo "Update manifest is corrupt -- this should never happen."
665		echo "Re-run '$0 fetch'."
666		exit 1
667	fi
668}
669
670# Perform sanity checks and set some final parameters in
671# preparation for UNinstalling updates.
672rollback_check_params () {
673	# Check that we are root.  All sorts of things won't work otherwise.
674	if [ `id -u` != 0 ]; then
675		echo "You must be root to run this."
676		exit 1
677	fi
678
679	# Check that we have a working directory
680	_WORKDIR_bad="Directory does not exist or is not writable: "
681	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
682		echo -n "`basename $0`: "
683		echo -n "${_WORKDIR_bad}"
684		echo ${WORKDIR}
685		exit 1
686	fi
687	cd ${WORKDIR} || exit 1
688
689	# Construct a unique name from ${BASEDIR}
690	BDHASH=`echo ${BASEDIR} | sha256 -q`
691
692	# Check that we have updates ready to rollback
693	if ! [ -L ${BDHASH}-rollback ]; then
694		echo "No rollback directory found."
695		exit 1
696	fi
697	if ! [ -f ${BDHASH}-rollback/INDEX-OLD ] ||
698	    ! [ -f ${BDHASH}-rollback/INDEX-NEW ]; then
699		echo "Update manifest is corrupt -- this should never happen."
700		exit 1
701	fi
702}
703
704# Perform sanity checks and set some final parameters
705# in preparation for comparing the system against the
706# published index.  Figure out which index we should
707# compare against: If the user is running *-p[0-9]+,
708# strip off the last part; if the user is running
709# -SECURITY, call it -RELEASE.  Chdir into the working
710# directory.
711IDS_check_params () {
712	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
713
714	_SERVERNAME_z=\
715"SERVERNAME must be given via command line or configuration file."
716	_KEYPRINT_z="Key must be given via -k option or configuration file."
717	_KEYPRINT_bad="Invalid key fingerprint: "
718	_WORKDIR_bad="Directory does not exist or is not writable: "
719
720	if [ -z "${SERVERNAME}" ]; then
721		echo -n "`basename $0`: "
722		echo "${_SERVERNAME_z}"
723		exit 1
724	fi
725	if [ -z "${KEYPRINT}" ]; then
726		echo -n "`basename $0`: "
727		echo "${_KEYPRINT_z}"
728		exit 1
729	fi
730	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
731		echo -n "`basename $0`: "
732		echo -n "${_KEYPRINT_bad}"
733		echo ${KEYPRINT}
734		exit 1
735	fi
736	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
737		echo -n "`basename $0`: "
738		echo -n "${_WORKDIR_bad}"
739		echo ${WORKDIR}
740		exit 1
741	fi
742	cd ${WORKDIR} || exit 1
743
744	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
745	# to provide an upgrade path for FreeBSD Update 1.x users, since
746	# the kernels provided by FreeBSD Update 1.x are always labelled
747	# as X.Y-SECURITY.
748	RELNUM=`uname -r |
749	    sed -E 's,-p[0-9]+,,' |
750	    sed -E 's,-SECURITY,-RELEASE,'`
751	ARCH=`uname -m`
752	FETCHDIR=${RELNUM}/${ARCH}
753	PATCHDIR=${RELNUM}/${ARCH}/bp
754
755	# Figure out what directory contains the running kernel
756	BOOTFILE=`sysctl -n kern.bootfile`
757	KERNELDIR=${BOOTFILE%/kernel}
758	if ! [ -d ${KERNELDIR} ]; then
759		echo "Cannot identify running kernel"
760		exit 1
761	fi
762
763	# Figure out what kernel configuration is running.  We start with
764	# the output of `uname -i`, and then make the following adjustments:
765	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
766	# file says "ident SMP-GENERIC", I don't know...
767	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
768	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
769	# we're running an SMP kernel.  This mis-identification is a bug
770	# which was fixed in 6.2-STABLE.
771	KERNCONF=`uname -i`
772	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
773		KERNCONF=SMP
774	fi
775	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
776		if sysctl kern.version | grep -qE '/SMP$'; then
777			KERNCONF=SMP
778		fi
779	fi
780
781	# Define some paths
782	SHA256=/sbin/sha256
783	PHTTPGET=/usr/libexec/phttpget
784
785	# Set up variables relating to VERBOSELEVEL
786	fetch_setup_verboselevel
787}
788
789#### Core functionality -- the actual work gets done here
790
791# Use an SRV query to pick a server.  If the SRV query doesn't provide
792# a useful answer, use the server name specified by the user.
793# Put another way... look up _http._tcp.${SERVERNAME} and pick a server
794# from that; or if no servers are returned, use ${SERVERNAME}.
795# This allows a user to specify "portsnap.freebsd.org" (in which case
796# portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org"
797# (in which case portsnap will use that particular server, since there
798# won't be an SRV entry for that name).
799#
800# We ignore the Port field, since we are always going to use port 80.
801
802# Fetch the mirror list, but do not pick a mirror yet.  Returns 1 if
803# no mirrors are available for any reason.
804fetch_pick_server_init () {
805	: > serverlist_tried
806
807# Check that host(1) exists (i.e., that the system wasn't built with the
808# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist.
809	if ! which -s host; then
810		: > serverlist_full
811		return 1
812	fi
813
814	echo -n "Looking up ${SERVERNAME} mirrors... "
815
816# Issue the SRV query and pull out the Priority, Weight, and Target fields.
817# BIND 9 prints "$name has SRV record ..." while BIND 8 prints
818# "$name server selection ..."; we allow either format.
819	MLIST="_http._tcp.${SERVERNAME}"
820	host -t srv "${MLIST}" |
821	    sed -nE "s/${MLIST} (has SRV record|server selection) //p" |
822	    cut -f 1,2,4 -d ' ' |
823	    sed -e 's/\.$//' |
824	    sort > serverlist_full
825
826# If no records, give up -- we'll just use the server name we were given.
827	if [ `wc -l < serverlist_full` -eq 0 ]; then
828		echo "none found."
829		return 1
830	fi
831
832# Report how many mirrors we found.
833	echo `wc -l < serverlist_full` "mirrors found."
834
835# Generate a random seed for use in picking mirrors.  If HTTP_PROXY
836# is set, this will be used to generate the seed; otherwise, the seed
837# will be random.
838	if [ -n "${HTTP_PROXY}${http_proxy}" ]; then
839		RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" |
840		    tr -d 'a-f' |
841		    cut -c 1-9`
842	else
843		RANDVALUE=`jot -r 1 0 999999999`
844	fi
845}
846
847# Pick a mirror.  Returns 1 if we have run out of mirrors to try.
848fetch_pick_server () {
849# Generate a list of not-yet-tried mirrors
850	sort serverlist_tried |
851	    comm -23 serverlist_full - > serverlist
852
853# Have we run out of mirrors?
854	if [ `wc -l < serverlist` -eq 0 ]; then
855		echo "No mirrors remaining, giving up."
856		return 1
857	fi
858
859# Find the highest priority level (lowest numeric value).
860	SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1`
861
862# Add up the weights of the response lines at that priority level.
863	SRV_WSUM=0;
864	while read X; do
865		case "$X" in
866		${SRV_PRIORITY}\ *)
867			SRV_W=`echo $X | cut -f 2 -d ' '`
868			SRV_WSUM=$(($SRV_WSUM + $SRV_W))
869			;;
870		esac
871	done < serverlist
872
873# If all the weights are 0, pretend that they are all 1 instead.
874	if [ ${SRV_WSUM} -eq 0 ]; then
875		SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l`
876		SRV_W_ADD=1
877	else
878		SRV_W_ADD=0
879	fi
880
881# Pick a value between 0 and the sum of the weights - 1
882	SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}`
883
884# Read through the list of mirrors and set SERVERNAME.  Write the line
885# corresponding to the mirror we selected into serverlist_tried so that
886# we won't try it again.
887	while read X; do
888		case "$X" in
889		${SRV_PRIORITY}\ *)
890			SRV_W=`echo $X | cut -f 2 -d ' '`
891			SRV_W=$(($SRV_W + $SRV_W_ADD))
892			if [ $SRV_RND -lt $SRV_W ]; then
893				SERVERNAME=`echo $X | cut -f 3 -d ' '`
894				echo "$X" >> serverlist_tried
895				break
896			else
897				SRV_RND=$(($SRV_RND - $SRV_W))
898			fi
899			;;
900		esac
901	done < serverlist
902}
903
904# Take a list of ${oldhash}|${newhash} and output a list of needed patches,
905# i.e., those for which we have ${oldhash} and don't have ${newhash}.
906fetch_make_patchlist () {
907	grep -vE "^([0-9a-f]{64})\|\1$" |
908	    tr '|' ' ' |
909		while read X Y; do
910			if [ -f "files/${Y}.gz" ] ||
911			    [ ! -f "files/${X}.gz" ]; then
912				continue
913			fi
914			echo "${X}|${Y}"
915		done | uniq
916}
917
918# Print user-friendly progress statistics
919fetch_progress () {
920	LNC=0
921	while read x; do
922		LNC=$(($LNC + 1))
923		if [ $(($LNC % 10)) = 0 ]; then
924			echo -n $LNC
925		elif [ $(($LNC % 2)) = 0 ]; then
926			echo -n .
927		fi
928	done
929	echo -n " "
930}
931
932# Function for asking the user if everything is ok
933continuep () {
934	while read -p "Does this look reasonable (y/n)? " CONTINUE; do
935		case "${CONTINUE}" in
936		y*)
937			return 0
938			;;
939		n*)
940			return 1
941			;;
942		esac
943	done
944}
945
946# Initialize the working directory
947workdir_init () {
948	mkdir -p files
949	touch tINDEX.present
950}
951
952# Check that we have a public key with an appropriate hash, or
953# fetch the key if it doesn't exist.  Returns 1 if the key has
954# not yet been fetched.
955fetch_key () {
956	if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
957		return 0
958	fi
959
960	echo -n "Fetching public key from ${SERVERNAME}... "
961	rm -f pub.ssl
962	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/pub.ssl \
963	    2>${QUIETREDIR} || true
964	if ! [ -r pub.ssl ]; then
965		echo "failed."
966		return 1
967	fi
968	if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
969		echo "key has incorrect hash."
970		rm -f pub.ssl
971		return 1
972	fi
973	echo "done."
974}
975
976# Fetch metadata signature, aka "tag".
977fetch_tag () {
978	echo -n "Fetching metadata signature "
979	echo ${NDEBUG} "for ${RELNUM} from ${SERVERNAME}... "
980	rm -f latest.ssl
981	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/latest.ssl	\
982	    2>${QUIETREDIR} || true
983	if ! [ -r latest.ssl ]; then
984		echo "failed."
985		return 1
986	fi
987
988	openssl rsautl -pubin -inkey pub.ssl -verify		\
989	    < latest.ssl > tag.new 2>${QUIETREDIR} || true
990	rm latest.ssl
991
992	if ! [ `wc -l < tag.new` = 1 ] ||
993	    ! grep -qE	\
994    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
995		tag.new; then
996		echo "invalid signature."
997		return 1
998	fi
999
1000	echo "done."
1001
1002	RELPATCHNUM=`cut -f 4 -d '|' < tag.new`
1003	TINDEXHASH=`cut -f 5 -d '|' < tag.new`
1004	EOLTIME=`cut -f 6 -d '|' < tag.new`
1005}
1006
1007# Sanity-check the patch number in a tag, to make sure that we're not
1008# going to "update" backwards and to prevent replay attacks.
1009fetch_tagsanity () {
1010	# Check that we're not going to move from -pX to -pY with Y < X.
1011	RELPX=`uname -r | sed -E 's,.*-,,'`
1012	if echo ${RELPX} | grep -qE '^p[0-9]+$'; then
1013		RELPX=`echo ${RELPX} | cut -c 2-`
1014	else
1015		RELPX=0
1016	fi
1017	if [ "${RELPATCHNUM}" -lt "${RELPX}" ]; then
1018		echo
1019		echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1020		echo " appear older than what"
1021		echo "we are currently running (`uname -r`)!"
1022		echo "Cowardly refusing to proceed any further."
1023		return 1
1024	fi
1025
1026	# If "tag" exists and corresponds to ${RELNUM}, make sure that
1027	# it contains a patch number <= RELPATCHNUM, in order to protect
1028	# against rollback (replay) attacks.
1029	if [ -f tag ] &&
1030	    grep -qE	\
1031    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
1032		tag; then
1033		LASTRELPATCHNUM=`cut -f 4 -d '|' < tag`
1034
1035		if [ "${RELPATCHNUM}" -lt "${LASTRELPATCHNUM}" ]; then
1036			echo
1037			echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1038			echo " are older than the"
1039			echo -n "most recently seen updates"
1040			echo " (${RELNUM}-p${LASTRELPATCHNUM})."
1041			echo "Cowardly refusing to proceed any further."
1042			return 1
1043		fi
1044	fi
1045}
1046
1047# Fetch metadata index file
1048fetch_metadata_index () {
1049	echo ${NDEBUG} "Fetching metadata index... "
1050	rm -f ${TINDEXHASH}
1051	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/t/${TINDEXHASH}
1052	    2>${QUIETREDIR}
1053	if ! [ -f ${TINDEXHASH} ]; then
1054		echo "failed."
1055		return 1
1056	fi
1057	if [ `${SHA256} -q ${TINDEXHASH}` != ${TINDEXHASH} ]; then
1058		echo "update metadata index corrupt."
1059		return 1
1060	fi
1061	echo "done."
1062}
1063
1064# Print an error message about signed metadata being bogus.
1065fetch_metadata_bogus () {
1066	echo
1067	echo "The update metadata$1 is correctly signed, but"
1068	echo "failed an integrity check."
1069	echo "Cowardly refusing to proceed any further."
1070	return 1
1071}
1072
1073# Construct tINDEX.new by merging the lines named in $1 from ${TINDEXHASH}
1074# with the lines not named in $@ from tINDEX.present (if that file exists).
1075fetch_metadata_index_merge () {
1076	for METAFILE in $@; do
1077		if [ `grep -E "^${METAFILE}\|" ${TINDEXHASH} | wc -l`	\
1078		    -ne 1 ]; then
1079			fetch_metadata_bogus " index"
1080			return 1
1081		fi
1082
1083		grep -E "${METAFILE}\|" ${TINDEXHASH}
1084	done |
1085	    sort > tINDEX.wanted
1086
1087	if [ -f tINDEX.present ]; then
1088		join -t '|' -v 2 tINDEX.wanted tINDEX.present |
1089		    sort -m - tINDEX.wanted > tINDEX.new
1090		rm tINDEX.wanted
1091	else
1092		mv tINDEX.wanted tINDEX.new
1093	fi
1094}
1095
1096# Sanity check all the lines of tINDEX.new.  Even if more metadata lines
1097# are added by future versions of the server, this won't cause problems,
1098# since the only lines which appear in tINDEX.new are the ones which we
1099# specifically grepped out of ${TINDEXHASH}.
1100fetch_metadata_index_sanity () {
1101	if grep -qvE '^[0-9A-Z.-]+\|[0-9a-f]{64}$' tINDEX.new; then
1102		fetch_metadata_bogus " index"
1103		return 1
1104	fi
1105}
1106
1107# Sanity check the metadata file $1.
1108fetch_metadata_sanity () {
1109	# Some aliases to save space later: ${P} is a character which can
1110	# appear in a path; ${M} is the four numeric metadata fields; and
1111	# ${H} is a sha256 hash.
1112	P="[-+./:=_[[:alnum:]]"
1113	M="[0-9]+\|[0-9]+\|[0-9]+\|[0-9]+"
1114	H="[0-9a-f]{64}"
1115
1116	# Check that the first four fields make sense.
1117	if gunzip -c < files/$1.gz |
1118	    grep -qvE "^[a-z]+\|[0-9a-z]+\|${P}+\|[fdL-]\|"; then
1119		fetch_metadata_bogus ""
1120		return 1
1121	fi
1122
1123	# Remove the first three fields.
1124	gunzip -c < files/$1.gz |
1125	    cut -f 4- -d '|' > sanitycheck.tmp
1126
1127	# Sanity check entries with type 'f'
1128	if grep -E '^f' sanitycheck.tmp |
1129	    grep -qvE "^f\|${M}\|${H}\|${P}*\$"; then
1130		fetch_metadata_bogus ""
1131		return 1
1132	fi
1133
1134	# Sanity check entries with type 'd'
1135	if grep -E '^d' sanitycheck.tmp |
1136	    grep -qvE "^d\|${M}\|\|\$"; then
1137		fetch_metadata_bogus ""
1138		return 1
1139	fi
1140
1141	# Sanity check entries with type 'L'
1142	if grep -E '^L' sanitycheck.tmp |
1143	    grep -qvE "^L\|${M}\|${P}*\|\$"; then
1144		fetch_metadata_bogus ""
1145		return 1
1146	fi
1147
1148	# Sanity check entries with type '-'
1149	if grep -E '^-' sanitycheck.tmp |
1150	    grep -qvE "^-\|\|\|\|\|\|"; then
1151		fetch_metadata_bogus ""
1152		return 1
1153	fi
1154
1155	# Clean up
1156	rm sanitycheck.tmp
1157}
1158
1159# Fetch the metadata index and metadata files listed in $@,
1160# taking advantage of metadata patches where possible.
1161fetch_metadata () {
1162	fetch_metadata_index || return 1
1163	fetch_metadata_index_merge $@ || return 1
1164	fetch_metadata_index_sanity || return 1
1165
1166	# Generate a list of wanted metadata patches
1167	join -t '|' -o 1.2,2.2 tINDEX.present tINDEX.new |
1168	    fetch_make_patchlist > patchlist
1169
1170	if [ -s patchlist ]; then
1171		# Attempt to fetch metadata patches
1172		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
1173		echo ${NDEBUG} "metadata patches.${DDSTATS}"
1174		tr '|' '-' < patchlist |
1175		    lam -s "${FETCHDIR}/tp/" - -s ".gz" |
1176		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1177			2>${STATSREDIR} | fetch_progress
1178		echo "done."
1179
1180		# Attempt to apply metadata patches
1181		echo -n "Applying metadata patches... "
1182		tr '|' ' ' < patchlist |
1183		    while read X Y; do
1184			if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
1185			gunzip -c < ${X}-${Y}.gz > diff
1186			gunzip -c < files/${X}.gz > diff-OLD
1187
1188			# Figure out which lines are being added and removed
1189			grep -E '^-' diff |
1190			    cut -c 2- |
1191			    while read PREFIX; do
1192				look "${PREFIX}" diff-OLD
1193			    done |
1194			    sort > diff-rm
1195			grep -E '^\+' diff |
1196			    cut -c 2- > diff-add
1197
1198			# Generate the new file
1199			comm -23 diff-OLD diff-rm |
1200			    sort - diff-add > diff-NEW
1201
1202			if [ `${SHA256} -q diff-NEW` = ${Y} ]; then
1203				mv diff-NEW files/${Y}
1204				gzip -n files/${Y}
1205			else
1206				mv diff-NEW ${Y}.bad
1207			fi
1208			rm -f ${X}-${Y}.gz diff
1209			rm -f diff-OLD diff-NEW diff-add diff-rm
1210		done 2>${QUIETREDIR}
1211		echo "done."
1212	fi
1213
1214	# Update metadata without patches
1215	cut -f 2 -d '|' < tINDEX.new |
1216	    while read Y; do
1217		if [ ! -f "files/${Y}.gz" ]; then
1218			echo ${Y};
1219		fi
1220	    done |
1221	    sort -u > filelist
1222
1223	if [ -s filelist ]; then
1224		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
1225		echo ${NDEBUG} "metadata files... "
1226		lam -s "${FETCHDIR}/m/" - -s ".gz" < filelist |
1227		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1228		    2>${QUIETREDIR}
1229
1230		while read Y; do
1231			if ! [ -f ${Y}.gz ]; then
1232				echo "failed."
1233				return 1
1234			fi
1235			if [ `gunzip -c < ${Y}.gz |
1236			    ${SHA256} -q` = ${Y} ]; then
1237				mv ${Y}.gz files/${Y}.gz
1238			else
1239				echo "metadata is corrupt."
1240				return 1
1241			fi
1242		done < filelist
1243		echo "done."
1244	fi
1245
1246# Sanity-check the metadata files.
1247	cut -f 2 -d '|' tINDEX.new > filelist
1248	while read X; do
1249		fetch_metadata_sanity ${X} || return 1
1250	done < filelist
1251
1252# Remove files which are no longer needed
1253	cut -f 2 -d '|' tINDEX.present |
1254	    sort > oldfiles
1255	cut -f 2 -d '|' tINDEX.new |
1256	    sort |
1257	    comm -13 - oldfiles |
1258	    lam -s "files/" - -s ".gz" |
1259	    xargs rm -f
1260	rm patchlist filelist oldfiles
1261	rm ${TINDEXHASH}
1262
1263# We're done!
1264	mv tINDEX.new tINDEX.present
1265	mv tag.new tag
1266
1267	return 0
1268}
1269
1270# Extract a subset of a downloaded metadata file containing only the parts
1271# which are listed in COMPONENTS.
1272fetch_filter_metadata_components () {
1273	METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
1274	gunzip -c < files/${METAHASH}.gz > $1.all
1275
1276	# Fish out the lines belonging to components we care about.
1277	for C in ${COMPONENTS}; do
1278		look "`echo ${C} | tr '/' '|'`|" $1.all
1279	done > $1
1280
1281	# Remove temporary file.
1282	rm $1.all
1283}
1284
1285# Generate a filtered version of the metadata file $1 from the downloaded
1286# file, by fishing out the lines corresponding to components we're trying
1287# to keep updated, and then removing lines corresponding to paths we want
1288# to ignore.
1289fetch_filter_metadata () {
1290	# Fish out the lines belonging to components we care about.
1291	fetch_filter_metadata_components $1
1292
1293	# Canonicalize directory names by removing any trailing / in
1294	# order to avoid listing directories multiple times if they
1295	# belong to multiple components.  Turning "/" into "" doesn't
1296	# matter, since we add a leading "/" when we use paths later.
1297	cut -f 3- -d '|' $1 |
1298	    sed -e 's,/|d|,|d|,' |
1299	    sort -u > $1.tmp
1300
1301	# Figure out which lines to ignore and remove them.
1302	for X in ${IGNOREPATHS}; do
1303		grep -E "^${X}" $1.tmp
1304	done |
1305	    sort -u |
1306	    comm -13 - $1.tmp > $1
1307
1308	# Remove temporary files.
1309	rm $1.tmp
1310}
1311
1312# Filter the metadata file $1 by adding lines with "/boot/$2"
1313# replaced by ${KERNELDIR} (which is `sysctl -n kern.bootfile` minus the
1314# trailing "/kernel"); and if "/boot/$2" does not exist, remove
1315# the original lines which start with that.
1316# Put another way: Deal with the fact that the FOO kernel is sometimes
1317# installed in /boot/FOO/ and is sometimes installed elsewhere.
1318fetch_filter_kernel_names () {
1319	grep ^/boot/$2 $1 |
1320	    sed -e "s,/boot/$2,${KERNELDIR},g" |
1321	    sort - $1 > $1.tmp
1322	mv $1.tmp $1
1323
1324	if ! [ -d /boot/$2 ]; then
1325		grep -v ^/boot/$2 $1 > $1.tmp
1326		mv $1.tmp $1
1327	fi
1328}
1329
1330# For all paths appearing in $1 or $3, inspect the system
1331# and generate $2 describing what is currently installed.
1332fetch_inspect_system () {
1333	# No errors yet...
1334	rm -f .err
1335
1336	# Tell the user why his disk is suddenly making lots of noise
1337	echo -n "Inspecting system... "
1338
1339	# Generate list of files to inspect
1340	cat $1 $3 |
1341	    cut -f 1 -d '|' |
1342	    sort -u > filelist
1343
1344	# Examine each file and output lines of the form
1345	# /path/to/file|type|device-inum|user|group|perm|flags|value
1346	# sorted by device and inode number.
1347	while read F; do
1348		# If the symlink/file/directory does not exist, record this.
1349		if ! [ -e ${BASEDIR}/${F} ]; then
1350			echo "${F}|-||||||"
1351			continue
1352		fi
1353		if ! [ -r ${BASEDIR}/${F} ]; then
1354			echo "Cannot read file: ${BASEDIR}/${F}"	\
1355			    >/dev/stderr
1356			touch .err
1357			return 1
1358		fi
1359
1360		# Otherwise, output an index line.
1361		if [ -L ${BASEDIR}/${F} ]; then
1362			echo -n "${F}|L|"
1363			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1364			readlink ${BASEDIR}/${F};
1365		elif [ -f ${BASEDIR}/${F} ]; then
1366			echo -n "${F}|f|"
1367			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1368			sha256 -q ${BASEDIR}/${F};
1369		elif [ -d ${BASEDIR}/${F} ]; then
1370			echo -n "${F}|d|"
1371			stat -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1372		else
1373			echo "Unknown file type: ${BASEDIR}/${F}"	\
1374			    >/dev/stderr
1375			touch .err
1376			return 1
1377		fi
1378	done < filelist |
1379	    sort -k 3,3 -t '|' > $2.tmp
1380	rm filelist
1381
1382	# Check if an error occured during system inspection
1383	if [ -f .err ]; then
1384		return 1
1385	fi
1386
1387	# Convert to the form
1388	# /path/to/file|type|user|group|perm|flags|value|hlink
1389	# by resolving identical device and inode numbers into hard links.
1390	cut -f 1,3 -d '|' $2.tmp |
1391	    sort -k 1,1 -t '|' |
1392	    sort -s -u -k 2,2 -t '|' |
1393	    join -1 2 -2 3 -t '|' - $2.tmp |
1394	    awk -F \| -v OFS=\|		\
1395		'{
1396		    if (($2 == $3) || ($4 == "-"))
1397			print $3,$4,$5,$6,$7,$8,$9,""
1398		    else
1399			print $3,$4,$5,$6,$7,$8,$9,$2
1400		}' |
1401	    sort > $2
1402	rm $2.tmp
1403
1404	# We're finished looking around
1405	echo "done."
1406}
1407
1408# For any paths matching ${MERGECHANGES}, compare $1 and $2 and find any
1409# files which differ; generate $3 containing these paths and the old hashes.
1410fetch_filter_mergechanges () {
1411	# Pull out the paths and hashes of the files matching ${MERGECHANGES}.
1412	for F in $1 $2; do
1413		for X in ${MERGECHANGES}; do
1414			grep -E "^${X}" ${F}
1415		done |
1416		    cut -f 1,2,7 -d '|' |
1417		    sort > ${F}-values
1418	done
1419
1420	# Any line in $2-values which doesn't appear in $1-values and is a
1421	# file means that we should list the path in $3.
1422	comm -13 $1-values $2-values |
1423	    fgrep '|f|' |
1424	    cut -f 1 -d '|' > $2-paths
1425
1426	# For each path, pull out one (and only one!) entry from $1-values.
1427	# Note that we cannot distinguish which "old" version the user made
1428	# changes to; but hopefully any changes which occur due to security
1429	# updates will exist in both the "new" version and the version which
1430	# the user has installed, so the merging will still work.
1431	while read X; do
1432		look "${X}|" $1-values |
1433		    head -1
1434	done < $2-paths > $3
1435
1436	# Clean up
1437	rm $1-values $2-values $2-paths
1438}
1439
1440# For any paths matching ${UPDATEIFUNMODIFIED}, remove lines from $[123]
1441# which correspond to lines in $2 with hashes not matching $1 or $3, unless
1442# the paths are listed in $4.  For entries in $2 marked "not present"
1443# (aka. type -), remove lines from $[123] unless there is a corresponding
1444# entry in $1.
1445fetch_filter_unmodified_notpresent () {
1446	# Figure out which lines of $1 and $3 correspond to bits which
1447	# should only be updated if they haven't changed, and fish out
1448	# the (path, type, value) tuples.
1449	# NOTE: We don't consider a file to be "modified" if it matches
1450	# the hash from $3.
1451	for X in ${UPDATEIFUNMODIFIED}; do
1452		grep -E "^${X}" $1
1453		grep -E "^${X}" $3
1454	done |
1455	    cut -f 1,2,7 -d '|' |
1456	    sort > $1-values
1457
1458	# Do the same for $2.
1459	for X in ${UPDATEIFUNMODIFIED}; do
1460		grep -E "^${X}" $2
1461	done |
1462	    cut -f 1,2,7 -d '|' |
1463	    sort > $2-values
1464
1465	# Any entry in $2-values which is not in $1-values corresponds to
1466	# a path which we need to remove from $1, $2, and $3, unless it
1467	# that path appears in $4.
1468	comm -13 $1-values $2-values |
1469	    sort -t '|' -k 1,1 > mlines.tmp
1470	cut -f 1 -d '|' $4 |
1471	    sort |
1472	    join -v 2 -t '|' - mlines.tmp |
1473	    sort > mlines
1474	rm $1-values $2-values mlines.tmp
1475
1476	# Any lines in $2 which are not in $1 AND are "not present" lines
1477	# also belong in mlines.
1478	comm -13 $1 $2 |
1479	    cut -f 1,2,7 -d '|' |
1480	    fgrep '|-|' >> mlines
1481
1482	# Remove lines from $1, $2, and $3
1483	for X in $1 $2 $3; do
1484		sort -t '|' -k 1,1 ${X} > ${X}.tmp
1485		cut -f 1 -d '|' < mlines |
1486		    sort |
1487		    join -v 2 -t '|' - ${X}.tmp |
1488		    sort > ${X}
1489		rm ${X}.tmp
1490	done
1491
1492	# Store a list of the modified files, for future reference
1493	fgrep -v '|-|' mlines |
1494	    cut -f 1 -d '|' > modifiedfiles
1495	rm mlines
1496}
1497
1498# For each entry in $1 of type -, remove any corresponding
1499# entry from $2 if ${ALLOWADD} != "yes".  Remove all entries
1500# of type - from $1.
1501fetch_filter_allowadd () {
1502	cut -f 1,2 -d '|' < $1 |
1503	    fgrep '|-' |
1504	    cut -f 1 -d '|' > filesnotpresent
1505
1506	if [ ${ALLOWADD} != "yes" ]; then
1507		sort < $2 |
1508		    join -v 1 -t '|' - filesnotpresent |
1509		    sort > $2.tmp
1510		mv $2.tmp $2
1511	fi
1512
1513	sort < $1 |
1514	    join -v 1 -t '|' - filesnotpresent |
1515	    sort > $1.tmp
1516	mv $1.tmp $1
1517	rm filesnotpresent
1518}
1519
1520# If ${ALLOWDELETE} != "yes", then remove any entries from $1
1521# which don't correspond to entries in $2.
1522fetch_filter_allowdelete () {
1523	# Produce a lists ${PATH}|${TYPE}
1524	for X in $1 $2; do
1525		cut -f 1-2 -d '|' < ${X} |
1526		    sort -u > ${X}.nodes
1527	done
1528
1529	# Figure out which lines need to be removed from $1.
1530	if [ ${ALLOWDELETE} != "yes" ]; then
1531		comm -23 $1.nodes $2.nodes > $1.badnodes
1532	else
1533		: > $1.badnodes
1534	fi
1535
1536	# Remove the relevant lines from $1
1537	while read X; do
1538		look "${X}|" $1
1539	done < $1.badnodes |
1540	    comm -13 - $1 > $1.tmp
1541	mv $1.tmp $1
1542
1543	rm $1.badnodes $1.nodes $2.nodes
1544}
1545
1546# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in $2
1547# with metadata not matching any entry in $1, replace the corresponding
1548# line of $3 with one having the same metadata as the entry in $2.
1549fetch_filter_modified_metadata () {
1550	# Fish out the metadata from $1 and $2
1551	for X in $1 $2; do
1552		cut -f 1-6 -d '|' < ${X} > ${X}.metadata
1553	done
1554
1555	# Find the metadata we need to keep
1556	if [ ${KEEPMODIFIEDMETADATA} = "yes" ]; then
1557		comm -13 $1.metadata $2.metadata > keepmeta
1558	else
1559		: > keepmeta
1560	fi
1561
1562	# Extract the lines which we need to remove from $3, and
1563	# construct the lines which we need to add to $3.
1564	: > $3.remove
1565	: > $3.add
1566	while read LINE; do
1567		NODE=`echo "${LINE}" | cut -f 1-2 -d '|'`
1568		look "${NODE}|" $3 >> $3.remove
1569		look "${NODE}|" $3 |
1570		    cut -f 7- -d '|' |
1571		    lam -s "${LINE}|" - >> $3.add
1572	done < keepmeta
1573
1574	# Remove the specified lines and add the new lines.
1575	sort $3.remove |
1576	    comm -13 - $3 |
1577	    sort -u - $3.add > $3.tmp
1578	mv $3.tmp $3
1579
1580	rm keepmeta $1.metadata $2.metadata $3.add $3.remove
1581}
1582
1583# Remove lines from $1 and $2 which are identical;
1584# no need to update a file if it isn't changing.
1585fetch_filter_uptodate () {
1586	comm -23 $1 $2 > $1.tmp
1587	comm -13 $1 $2 > $2.tmp
1588
1589	mv $1.tmp $1
1590	mv $2.tmp $2
1591}
1592
1593# Fetch any "clean" old versions of files we need for merging changes.
1594fetch_files_premerge () {
1595	# We only need to do anything if $1 is non-empty.
1596	if [ -s $1 ]; then
1597		# Tell the user what we're doing
1598		echo -n "Fetching files from ${OLDRELNUM} for merging... "
1599
1600		# List of files wanted
1601		fgrep '|f|' < $1 |
1602		    cut -f 3 -d '|' |
1603		    sort -u > files.wanted
1604
1605		# Only fetch the files we don't already have
1606		while read Y; do
1607			if [ ! -f "files/${Y}.gz" ]; then
1608				echo ${Y};
1609			fi
1610		done < files.wanted > filelist
1611
1612		# Actually fetch them
1613		lam -s "${OLDFETCHDIR}/f/" - -s ".gz" < filelist |
1614		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1615		    2>${QUIETREDIR}
1616
1617		# Make sure we got them all, and move them into /files/
1618		while read Y; do
1619			if ! [ -f ${Y}.gz ]; then
1620				echo "failed."
1621				return 1
1622			fi
1623			if [ `gunzip -c < ${Y}.gz |
1624			    ${SHA256} -q` = ${Y} ]; then
1625				mv ${Y}.gz files/${Y}.gz
1626			else
1627				echo "${Y} has incorrect hash."
1628				return 1
1629			fi
1630		done < filelist
1631		echo "done."
1632
1633		# Clean up
1634		rm filelist files.wanted
1635	fi
1636}
1637
1638# Prepare to fetch files: Generate a list of the files we need,
1639# copy the unmodified files we have into /files/, and generate
1640# a list of patches to download.
1641fetch_files_prepare () {
1642	# Tell the user why his disk is suddenly making lots of noise
1643	echo -n "Preparing to download files... "
1644
1645	# Reduce indices to ${PATH}|${HASH} pairs
1646	for X in $1 $2 $3; do
1647		cut -f 1,2,7 -d '|' < ${X} |
1648		    fgrep '|f|' |
1649		    cut -f 1,3 -d '|' |
1650		    sort > ${X}.hashes
1651	done
1652
1653	# List of files wanted
1654	cut -f 2 -d '|' < $3.hashes |
1655	    sort -u |
1656	    while read HASH; do
1657		if ! [ -f files/${HASH}.gz ]; then
1658			echo ${HASH}
1659		fi
1660	done > files.wanted
1661
1662	# Generate a list of unmodified files
1663	comm -12 $1.hashes $2.hashes |
1664	    sort -k 1,1 -t '|' > unmodified.files
1665
1666	# Copy all files into /files/.  We only need the unmodified files
1667	# for use in patching; but we'll want all of them if the user asks
1668	# to rollback the updates later.
1669	while read LINE; do
1670		F=`echo "${LINE}" | cut -f 1 -d '|'`
1671		HASH=`echo "${LINE}" | cut -f 2 -d '|'`
1672
1673		# Skip files we already have.
1674		if [ -f files/${HASH}.gz ]; then
1675			continue
1676		fi
1677
1678		# Make sure the file hasn't changed.
1679		cp "${BASEDIR}/${F}" tmpfile
1680		if [ `sha256 -q tmpfile` != ${HASH} ]; then
1681			echo
1682			echo "File changed while FreeBSD Update running: ${F}"
1683			return 1
1684		fi
1685
1686		# Place the file into storage.
1687		gzip -c < tmpfile > files/${HASH}.gz
1688		rm tmpfile
1689	done < $2.hashes
1690
1691	# Produce a list of patches to download
1692	sort -k 1,1 -t '|' $3.hashes |
1693	    join -t '|' -o 2.2,1.2 - unmodified.files |
1694	    fetch_make_patchlist > patchlist
1695
1696	# Garbage collect
1697	rm unmodified.files $1.hashes $2.hashes $3.hashes
1698
1699	# We don't need the list of possible old files any more.
1700	rm $1
1701
1702	# We're finished making noise
1703	echo "done."
1704}
1705
1706# Fetch files.
1707fetch_files () {
1708	# Attempt to fetch patches
1709	if [ -s patchlist ]; then
1710		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
1711		echo ${NDEBUG} "patches.${DDSTATS}"
1712		tr '|' '-' < patchlist |
1713		    lam -s "${PATCHDIR}/" - |
1714		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1715			2>${STATSREDIR} | fetch_progress
1716		echo "done."
1717
1718		# Attempt to apply patches
1719		echo -n "Applying patches... "
1720		tr '|' ' ' < patchlist |
1721		    while read X Y; do
1722			if [ ! -f "${X}-${Y}" ]; then continue; fi
1723			gunzip -c < files/${X}.gz > OLD
1724
1725			bspatch OLD NEW ${X}-${Y}
1726
1727			if [ `${SHA256} -q NEW` = ${Y} ]; then
1728				mv NEW files/${Y}
1729				gzip -n files/${Y}
1730			fi
1731			rm -f diff OLD NEW ${X}-${Y}
1732		done 2>${QUIETREDIR}
1733		echo "done."
1734	fi
1735
1736	# Download files which couldn't be generate via patching
1737	while read Y; do
1738		if [ ! -f "files/${Y}.gz" ]; then
1739			echo ${Y};
1740		fi
1741	done < files.wanted > filelist
1742
1743	if [ -s filelist ]; then
1744		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
1745		echo ${NDEBUG} "files... "
1746		lam -s "${FETCHDIR}/f/" - -s ".gz" < filelist |
1747		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1748		    2>${QUIETREDIR}
1749
1750		while read Y; do
1751			if ! [ -f ${Y}.gz ]; then
1752				echo "failed."
1753				return 1
1754			fi
1755			if [ `gunzip -c < ${Y}.gz |
1756			    ${SHA256} -q` = ${Y} ]; then
1757				mv ${Y}.gz files/${Y}.gz
1758			else
1759				echo "${Y} has incorrect hash."
1760				return 1
1761			fi
1762		done < filelist
1763		echo "done."
1764	fi
1765
1766	# Clean up
1767	rm files.wanted filelist patchlist
1768}
1769
1770# Create and populate install manifest directory; and report what updates
1771# are available.
1772fetch_create_manifest () {
1773	# If we have an existing install manifest, nuke it.
1774	if [ -L "${BDHASH}-install" ]; then
1775		rm -r ${BDHASH}-install/
1776		rm ${BDHASH}-install
1777	fi
1778
1779	# Report to the user if any updates were avoided due to local changes
1780	if [ -s modifiedfiles ]; then
1781		echo
1782		echo -n "The following files are affected by updates, "
1783		echo "but no changes have"
1784		echo -n "been downloaded because the files have been "
1785		echo "modified locally:"
1786		cat modifiedfiles
1787	fi | more
1788	rm modifiedfiles
1789
1790	# If no files will be updated, tell the user and exit
1791	if ! [ -s INDEX-PRESENT ] &&
1792	    ! [ -s INDEX-NEW ]; then
1793		rm INDEX-PRESENT INDEX-NEW
1794		echo
1795		echo -n "No updates needed to update system to "
1796		echo "${RELNUM}-p${RELPATCHNUM}."
1797		return
1798	fi
1799
1800	# Divide files into (a) removed files, (b) added files, and
1801	# (c) updated files.
1802	cut -f 1 -d '|' < INDEX-PRESENT |
1803	    sort > INDEX-PRESENT.flist
1804	cut -f 1 -d '|' < INDEX-NEW |
1805	    sort > INDEX-NEW.flist
1806	comm -23 INDEX-PRESENT.flist INDEX-NEW.flist > files.removed
1807	comm -13 INDEX-PRESENT.flist INDEX-NEW.flist > files.added
1808	comm -12 INDEX-PRESENT.flist INDEX-NEW.flist > files.updated
1809	rm INDEX-PRESENT.flist INDEX-NEW.flist
1810
1811	# Report removed files, if any
1812	if [ -s files.removed ]; then
1813		echo
1814		echo -n "The following files will be removed "
1815		echo "as part of updating to ${RELNUM}-p${RELPATCHNUM}:"
1816		cat files.removed
1817	fi | more
1818	rm files.removed
1819
1820	# Report added files, if any
1821	if [ -s files.added ]; then
1822		echo
1823		echo -n "The following files will be added "
1824		echo "as part of updating to ${RELNUM}-p${RELPATCHNUM}:"
1825		cat files.added
1826	fi | more
1827	rm files.added
1828
1829	# Report updated files, if any
1830	if [ -s files.updated ]; then
1831		echo
1832		echo -n "The following files will be updated "
1833		echo "as part of updating to ${RELNUM}-p${RELPATCHNUM}:"
1834
1835		cat files.updated
1836	fi | more
1837	rm files.updated
1838
1839	# Create a directory for the install manifest.
1840	MDIR=`mktemp -d install.XXXXXX` || return 1
1841
1842	# Populate it
1843	mv INDEX-PRESENT ${MDIR}/INDEX-OLD
1844	mv INDEX-NEW ${MDIR}/INDEX-NEW
1845
1846	# Link it into place
1847	ln -s ${MDIR} ${BDHASH}-install
1848}
1849
1850# Warn about any upcoming EoL
1851fetch_warn_eol () {
1852	# What's the current time?
1853	NOWTIME=`date "+%s"`
1854
1855	# When did we last warn about the EoL date?
1856	if [ -f lasteolwarn ]; then
1857		LASTWARN=`cat lasteolwarn`
1858	else
1859		LASTWARN=`expr ${NOWTIME} - 63072000`
1860	fi
1861
1862	# If the EoL time is past, warn.
1863	if [ ${EOLTIME} -lt ${NOWTIME} ]; then
1864		echo
1865		cat <<-EOF
1866		WARNING: `uname -sr` HAS PASSED ITS END-OF-LIFE DATE.
1867		Any security issues discovered after `date -r ${EOLTIME}`
1868		will not have been corrected.
1869		EOF
1870		return 1
1871	fi
1872
1873	# Figure out how long it has been since we last warned about the
1874	# upcoming EoL, and how much longer we have left.
1875	SINCEWARN=`expr ${NOWTIME} - ${LASTWARN}`
1876	TIMELEFT=`expr ${EOLTIME} - ${NOWTIME}`
1877
1878	# Don't warn if the EoL is more than 3 months away
1879	if [ ${TIMELEFT} -gt 7884000 ]; then
1880		return 0
1881	fi
1882
1883	# Don't warn if the time remaining is more than 3 times the time
1884	# since the last warning.
1885	if [ ${TIMELEFT} -gt `expr ${SINCEWARN} \* 3` ]; then
1886		return 0
1887	fi
1888
1889	# Figure out what time units to use.
1890	if [ ${TIMELEFT} -lt 604800 ]; then
1891		UNIT="day"
1892		SIZE=86400
1893	elif [ ${TIMELEFT} -lt 2678400 ]; then
1894		UNIT="week"
1895		SIZE=604800
1896	else
1897		UNIT="month"
1898		SIZE=2678400
1899	fi
1900
1901	# Compute the right number of units
1902	NUM=`expr ${TIMELEFT} / ${SIZE}`
1903	if [ ${NUM} != 1 ]; then
1904		UNIT="${UNIT}s"
1905	fi
1906
1907	# Print the warning
1908	echo
1909	cat <<-EOF
1910		WARNING: `uname -sr` is approaching its End-of-Life date.
1911		It is strongly recommended that you upgrade to a newer
1912		release within the next ${NUM} ${UNIT}.
1913	EOF
1914
1915	# Update the stored time of last warning
1916	echo ${NOWTIME} > lasteolwarn
1917}
1918
1919# Do the actual work involved in "fetch" / "cron".
1920fetch_run () {
1921	workdir_init || return 1
1922
1923	# Prepare the mirror list.
1924	fetch_pick_server_init && fetch_pick_server
1925
1926	# Try to fetch the public key until we run out of servers.
1927	while ! fetch_key; do
1928		fetch_pick_server || return 1
1929	done
1930
1931	# Try to fetch the metadata index signature ("tag") until we run
1932	# out of available servers; and sanity check the downloaded tag.
1933	while ! fetch_tag; do
1934		fetch_pick_server || return 1
1935	done
1936	fetch_tagsanity || return 1
1937
1938	# Fetch the latest INDEX-NEW and INDEX-OLD files.
1939	fetch_metadata INDEX-NEW INDEX-OLD || return 1
1940
1941	# Generate filtered INDEX-NEW and INDEX-OLD files containing only
1942	# the lines which (a) belong to components we care about, and (b)
1943	# don't correspond to paths we're explicitly ignoring.
1944	fetch_filter_metadata INDEX-NEW || return 1
1945	fetch_filter_metadata INDEX-OLD || return 1
1946
1947	# Translate /boot/${KERNCONF} into ${KERNELDIR}
1948	fetch_filter_kernel_names INDEX-NEW ${KERNCONF}
1949	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
1950
1951	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
1952	# system and generate an INDEX-PRESENT file.
1953	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
1954
1955	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
1956	# correspond to lines in INDEX-PRESENT with hashes not appearing
1957	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
1958	# INDEX-PRESENT has type - and there isn't a corresponding entry in
1959	# INDEX-OLD with type -.
1960	fetch_filter_unmodified_notpresent	\
1961	    INDEX-OLD INDEX-PRESENT INDEX-NEW /dev/null
1962
1963	# For each entry in INDEX-PRESENT of type -, remove any corresponding
1964	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
1965	# of type - from INDEX-PRESENT.
1966	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
1967
1968	# If ${ALLOWDELETE} != "yes", then remove any entries from
1969	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
1970	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
1971
1972	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
1973	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
1974	# replace the corresponding line of INDEX-NEW with one having the
1975	# same metadata as the entry in INDEX-PRESENT.
1976	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
1977
1978	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
1979	# no need to update a file if it isn't changing.
1980	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
1981
1982	# Prepare to fetch files: Generate a list of the files we need,
1983	# copy the unmodified files we have into /files/, and generate
1984	# a list of patches to download.
1985	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
1986
1987	# Fetch files.
1988	fetch_files || return 1
1989
1990	# Create and populate install manifest directory; and report what
1991	# updates are available.
1992	fetch_create_manifest || return 1
1993
1994	# Warn about any upcoming EoL
1995	fetch_warn_eol || return 1
1996}
1997
1998# If StrictComponents is not "yes", generate a new components list
1999# with only the components which appear to be installed.
2000upgrade_guess_components () {
2001	if [ "${STRICTCOMPONENTS}" = "no" ]; then
2002		# Generate filtered INDEX-ALL with only the components listed
2003		# in COMPONENTS.
2004		fetch_filter_metadata_components $1 || return 1
2005
2006		# Tell the user why his disk is suddenly making lots of noise
2007		echo -n "Inspecting system... "
2008
2009		# Look at the files on disk, and assume that a component is
2010		# supposed to be present if it is more than half-present.
2011		cut -f 1-3 -d '|' < INDEX-ALL |
2012		    tr '|' ' ' |
2013		    while read C S F; do
2014			if [ -e ${BASEDIR}/${F} ]; then
2015				echo "+ ${C}|${S}"
2016			fi
2017			echo "= ${C}|${S}"
2018		    done |
2019		    sort |
2020		    uniq -c |
2021		    sed -E 's,^ +,,' > compfreq
2022		grep ' = ' compfreq |
2023		    cut -f 1,3 -d ' ' |
2024		    sort -k 2,2 -t ' ' > compfreq.total
2025		grep ' + ' compfreq |
2026		    cut -f 1,3 -d ' ' |
2027		    sort -k 2,2 -t ' ' > compfreq.present
2028		join -t ' ' -1 2 -2 2 compfreq.present compfreq.total |
2029		    while read S P T; do
2030			if [ ${P} -gt `expr ${T} / 2` ]; then
2031				echo ${S}
2032			fi
2033		    done > comp.present
2034		cut -f 2 -d ' ' < compfreq.total > comp.total
2035		rm INDEX-ALL compfreq compfreq.total compfreq.present
2036
2037		# We're done making noise.
2038		echo "done."
2039
2040		# Sometimes the kernel isn't installed where INDEX-ALL
2041		# thinks that it should be: In particular, it is often in
2042		# /boot/kernel instead of /boot/GENERIC or /boot/SMP.  To
2043		# deal with this, if "kernel|X" is listed in comp.total
2044		# (i.e., is a component which would be upgraded if it is
2045		# found to be present) we will add it to comp.present.
2046		# If "kernel|<anything>" is in comp.total but "kernel|X" is
2047		# not, we print a warning -- the user is running a kernel
2048		# which isn't part of the release.
2049		KCOMP=`echo ${KERNCONF} | tr 'A-Z' 'a-z'`
2050		grep -E "^kernel\|${KCOMP}\$" comp.total >> comp.present
2051
2052		if grep -qE "^kernel\|" comp.total &&
2053		    ! grep -qE "^kernel\|${KCOMP}\$" comp.total; then
2054			cat <<-EOF
2055
2056WARNING: This system is running a "${KCOMP}" kernel, which is not a
2057kernel configuration distributed as part of FreeBSD ${RELNUM}.
2058This kernel will not be updated: you MUST update the kernel manually
2059before running "$0 install".
2060			EOF
2061		fi
2062
2063		# Re-sort the list of installed components and generate
2064		# the list of non-installed components.
2065		sort -u < comp.present > comp.present.tmp
2066		mv comp.present.tmp comp.present
2067		comm -13 comp.present comp.total > comp.absent
2068
2069		# Ask the user to confirm that what we have is correct.  To
2070		# reduce user confusion, translate "X|Y" back to "X/Y" (as
2071		# subcomponents must be listed in the configuration file).
2072		echo
2073		echo -n "The following components of FreeBSD "
2074		echo "seem to be installed:"
2075		tr '|' '/' < comp.present |
2076		    fmt -72
2077		echo
2078		echo -n "The following components of FreeBSD "
2079		echo "do not seem to be installed:"
2080		tr '|' '/' < comp.absent |
2081		    fmt -72
2082		echo
2083		continuep || return 1
2084		echo
2085
2086		# Suck the generated list of components into ${COMPONENTS}.
2087		# Note that comp.present.tmp is used due to issues with
2088		# pipelines and setting variables.
2089		COMPONENTS=""
2090		tr '|' '/' < comp.present > comp.present.tmp
2091		while read C; do
2092			COMPONENTS="${COMPONENTS} ${C}"
2093		done < comp.present.tmp
2094
2095		# Delete temporary files
2096		rm comp.present comp.present.tmp comp.absent comp.total
2097	fi
2098}
2099
2100# If StrictComponents is not "yes", COMPONENTS contains an entry
2101# corresponding to the currently running kernel, and said kernel
2102# does not exist in the new release, add "kernel/generic" to the
2103# list of components.
2104upgrade_guess_new_kernel () {
2105	if [ "${STRICTCOMPONENTS}" = "no" ]; then
2106		# Grab the unfiltered metadata file.
2107		METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
2108		gunzip -c < files/${METAHASH}.gz > $1.all
2109
2110		# If "kernel/${KCOMP}" is in ${COMPONENTS} and that component
2111		# isn't in $1.all, we need to add kernel/generic.
2112		for C in ${COMPONENTS}; do
2113			if [ ${C} = "kernel/${KCOMP}" ] &&
2114			    ! grep -qE "^kernel\|${KCOMP}\|" $1.all; then
2115				COMPONENTS="${COMPONENTS} kernel/generic"
2116				NKERNCONF="GENERIC"
2117				cat <<-EOF
2118
2119WARNING: This system is running a "${KCOMP}" kernel, which is not a
2120kernel configuration distributed as part of FreeBSD ${RELNUM}.
2121As part of upgrading to FreeBSD ${RELNUM}, this kernel will be
2122replaced with a "generic" kernel.
2123				EOF
2124				continuep || return 1
2125			fi
2126		done
2127
2128		# Don't need this any more...
2129		rm $1.all
2130	fi
2131}
2132
2133# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2134# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2135upgrade_oldall_to_oldnew () {
2136	# For each ${F}|... which appears in INDEX-ALL but does not appear
2137	# in INDEX-OLD, add ${F}|-|||||| to INDEX-OLD.
2138	cut -f 1 -d '|' < $1 |
2139	    sort -u > $1.paths
2140	cut -f 1 -d '|' < $2 |
2141	    sort -u |
2142	    comm -13 $1.paths - |
2143	    lam - -s "|-||||||" |
2144	    sort - $1 > $1.tmp
2145	mv $1.tmp $1
2146
2147	# Remove lines from INDEX-OLD which also appear in INDEX-ALL
2148	comm -23 $1 $2 > $1.tmp
2149	mv $1.tmp $1
2150
2151	# Remove lines from INDEX-ALL which have a file name not appearing
2152	# anywhere in INDEX-OLD (since these must be files which haven't
2153	# changed -- if they were new, there would be an entry of type "-").
2154	cut -f 1 -d '|' < $1 |
2155	    sort -u > $1.paths
2156	sort -k 1,1 -t '|' < $2 |
2157	    join -t '|' - $1.paths |
2158	    sort > $2.tmp
2159	rm $1.paths
2160	mv $2.tmp $2
2161
2162	# Rename INDEX-ALL to INDEX-NEW.
2163	mv $2 $3
2164}
2165
2166# From the list of "old" files in $1, merge changes in $2 with those in $3,
2167# and update $3 to reflect the hashes of merged files.
2168upgrade_merge () {
2169	# We only need to do anything if $1 is non-empty.
2170	if [ -s $1 ]; then
2171		cut -f 1 -d '|' $1 |
2172		    sort > $1-paths
2173
2174		# Create staging area for merging files
2175		rm -rf merge/
2176		while read F; do
2177			D=`dirname ${F}`
2178			mkdir -p merge/old/${D}
2179			mkdir -p merge/${OLDRELNUM}/${D}
2180			mkdir -p merge/${RELNUM}/${D}
2181			mkdir -p merge/new/${D}
2182		done < $1-paths
2183
2184		# Copy in files
2185		while read F; do
2186			# Currently installed file
2187			V=`look "${F}|" $2 | cut -f 7 -d '|'`
2188			gunzip < files/${V}.gz > merge/old/${F}
2189
2190			# Old release
2191			if look "${F}|" $1 | fgrep -q "|f|"; then
2192				V=`look "${F}|" $1 | cut -f 3 -d '|'`
2193				gunzip < files/${V}.gz		\
2194				    > merge/${OLDRELNUM}/${F}
2195			fi
2196
2197			# New release
2198			if look "${F}|" $3 | cut -f 1,2,7 -d '|' |
2199			    fgrep -q "|f|"; then
2200				V=`look "${F}|" $3 | cut -f 7 -d '|'`
2201				gunzip < files/${V}.gz		\
2202				    > merge/${RELNUM}/${F}
2203			fi
2204		done < $1-paths
2205
2206		# Attempt to automatically merge changes
2207		echo -n "Attempting to automatically merge "
2208		echo -n "changes in files..."
2209		: > failed.merges
2210		while read F; do
2211			# If the file doesn't exist in the new release,
2212			# the result of "merging changes" is having the file
2213			# not exist.
2214			if ! [ -f merge/${RELNUM}/${F} ]; then
2215				continue
2216			fi
2217
2218			# If the file didn't exist in the old release, we're
2219			# going to throw away the existing file and hope that
2220			# the version from the new release is what we want.
2221			if ! [ -f merge/${OLDRELNUM}/${F} ]; then
2222				cp merge/${RELNUM}/${F} merge/new/${F}
2223				continue
2224			fi
2225
2226			# Some files need special treatment.
2227			case ${F} in
2228			/etc/spwd.db | /etc/pwd.db | /etc/login.conf.db)
2229				# Don't merge these -- we're rebuild them
2230				# after updates are installed.
2231				cp merge/old/${F} merge/new/${F}
2232				;;
2233			*)
2234				if ! merge -p -L "current version"	\
2235				    -L "${OLDRELNUM}" -L "${RELNUM}"	\
2236				    merge/old/${F}			\
2237				    merge/${OLDRELNUM}/${F}		\
2238				    merge/${RELNUM}/${F}		\
2239				    > merge/new/${F} 2>/dev/null; then
2240					echo ${F} >> failed.merges
2241				fi
2242				;;
2243			esac
2244		done < $1-paths
2245		echo " done."
2246
2247		# Ask the user to handle any files which didn't merge.
2248		while read F; do
2249			cat <<-EOF
2250
2251The following file could not be merged automatically: ${F}
2252Press Enter to edit this file in ${EDITOR} and resolve the conflicts
2253manually...
2254			EOF
2255			read dummy </dev/tty
2256			${EDITOR} `pwd`/merge/new/${F} < /dev/tty
2257		done < failed.merges
2258		rm failed.merges
2259
2260		# Ask the user to confirm that he likes how the result
2261		# of merging files.
2262		while read F; do
2263			# Skip files which haven't changed.
2264			if [ -f merge/new/${F} ] &&
2265			    cmp -s merge/old/${F} merge/new/${F}; then
2266				continue
2267			fi
2268
2269			# Warn about files which are ceasing to exist.
2270			if ! [ -f merge/new/${F} ]; then
2271				cat <<-EOF
2272
2273The following file will be removed, as it no longer exists in
2274FreeBSD ${RELNUM}: ${F}
2275				EOF
2276				continuep < /dev/tty || return 1
2277				continue
2278			fi
2279
2280			# Print changes for the user's approval.
2281			cat <<-EOF
2282
2283The following changes, which occurred between FreeBSD ${OLDRELNUM} and
2284FreeBSD ${RELNUM} have been merged into ${F}:
2285EOF
2286			diff -U 5 -L "current version" -L "new version"	\
2287			    merge/old/${F} merge/new/${F} || true
2288			continuep < /dev/tty || return 1
2289		done < $1-paths
2290
2291		# Store merged files.
2292		while read F; do
2293			if [ -f merge/new/${F} ]; then
2294				V=`${SHA256} -q merge/new/${F}`
2295
2296				gzip -c < merge/new/${F} > files/${V}.gz
2297				echo "${F}|${V}"
2298			fi
2299		done < $1-paths > newhashes
2300
2301		# Pull lines out from $3 which need to be updated to
2302		# reflect merged files.
2303		while read F; do
2304			look "${F}|" $3
2305		done < $1-paths > $3-oldlines
2306
2307		# Update lines to reflect merged files
2308		join -t '|' -o 1.1,1.2,1.3,1.4,1.5,1.6,2.2,1.8		\
2309		    $3-oldlines newhashes > $3-newlines
2310
2311		# Remove old lines from $3 and add new lines.
2312		sort $3-oldlines |
2313		    comm -13 - $3 |
2314		    sort - $3-newlines > $3.tmp
2315		mv $3.tmp $3
2316
2317		# Clean up
2318		rm $1-paths newhashes $3-oldlines $3-newlines
2319		rm -rf merge/
2320	fi
2321
2322	# We're done with merging files.
2323	rm $1
2324}
2325
2326# Do the work involved in fetching upgrades to a new release
2327upgrade_run () {
2328	workdir_init || return 1
2329
2330	# Prepare the mirror list.
2331	fetch_pick_server_init && fetch_pick_server
2332
2333	# Try to fetch the public key until we run out of servers.
2334	while ! fetch_key; do
2335		fetch_pick_server || return 1
2336	done
2337
2338	# Try to fetch the metadata index signature ("tag") until we run
2339	# out of available servers; and sanity check the downloaded tag.
2340	while ! fetch_tag; do
2341		fetch_pick_server || return 1
2342	done
2343	fetch_tagsanity || return 1
2344
2345	# Fetch the INDEX-OLD and INDEX-ALL.
2346	fetch_metadata INDEX-OLD INDEX-ALL || return 1
2347
2348	# If StrictComponents is not "yes", generate a new components list
2349	# with only the components which appear to be installed.
2350	upgrade_guess_components INDEX-ALL || return 1
2351
2352	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
2353	# the components we want and without anything marked as "Ignore".
2354	fetch_filter_metadata INDEX-OLD || return 1
2355	fetch_filter_metadata INDEX-ALL || return 1
2356
2357	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-OLD.
2358	sort INDEX-OLD INDEX-ALL > INDEX-OLD.tmp
2359	mv INDEX-OLD.tmp INDEX-OLD
2360	rm INDEX-ALL
2361
2362	# Adjust variables for fetching files from the new release.
2363	OLDRELNUM=${RELNUM}
2364	RELNUM=${TARGETRELEASE}
2365	OLDFETCHDIR=${FETCHDIR}
2366	FETCHDIR=${RELNUM}/${ARCH}
2367
2368	# Try to fetch the NEW metadata index signature ("tag") until we run
2369	# out of available servers; and sanity check the downloaded tag.
2370	while ! fetch_tag; do
2371		fetch_pick_server || return 1
2372	done
2373
2374	# Fetch the new INDEX-ALL.
2375	fetch_metadata INDEX-ALL || return 1
2376
2377	# If StrictComponents is not "yes", COMPONENTS contains an entry
2378	# corresponding to the currently running kernel, and said kernel
2379	# does not exist in the new release, add "kernel/generic" to the
2380	# list of components.
2381	upgrade_guess_new_kernel INDEX-ALL || return 1
2382
2383	# Filter INDEX-ALL to contain only the components we want and without
2384	# anything marked as "Ignore".
2385	fetch_filter_metadata INDEX-ALL || return 1
2386
2387	# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2388	# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2389	upgrade_oldall_to_oldnew INDEX-OLD INDEX-ALL INDEX-NEW
2390
2391	# Translate /boot/${KERNCONF} or /boot/${NKERNCONF} into ${KERNELDIR}
2392	fetch_filter_kernel_names INDEX-NEW ${NKERNCONF}
2393	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
2394
2395	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
2396	# system and generate an INDEX-PRESENT file.
2397	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2398
2399	# Based on ${MERGECHANGES}, generate a file tomerge-old with the
2400	# paths and hashes of old versions of files to merge.
2401	fetch_filter_mergechanges INDEX-OLD INDEX-PRESENT tomerge-old
2402
2403	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
2404	# correspond to lines in INDEX-PRESENT with hashes not appearing
2405	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
2406	# INDEX-PRESENT has type - and there isn't a corresponding entry in
2407	# INDEX-OLD with type -.
2408	fetch_filter_unmodified_notpresent	\
2409	    INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old
2410
2411	# For each entry in INDEX-PRESENT of type -, remove any corresponding
2412	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
2413	# of type - from INDEX-PRESENT.
2414	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
2415
2416	# If ${ALLOWDELETE} != "yes", then remove any entries from
2417	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
2418	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
2419
2420	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
2421	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
2422	# replace the corresponding line of INDEX-NEW with one having the
2423	# same metadata as the entry in INDEX-PRESENT.
2424	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
2425
2426	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
2427	# no need to update a file if it isn't changing.
2428	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
2429
2430	# Fetch "clean" files from the old release for merging changes.
2431	fetch_files_premerge tomerge-old
2432
2433	# Prepare to fetch files: Generate a list of the files we need,
2434	# copy the unmodified files we have into /files/, and generate
2435	# a list of patches to download.
2436	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2437
2438	# Fetch patches from to-${RELNUM}/${ARCH}/bp/
2439	PATCHDIR=to-${RELNUM}/${ARCH}/bp
2440	fetch_files || return 1
2441
2442	# Merge configuration file changes.
2443	upgrade_merge tomerge-old INDEX-PRESENT INDEX-NEW || return 1
2444
2445	# Create and populate install manifest directory; and report what
2446	# updates are available.
2447	fetch_create_manifest || return 1
2448
2449	# Leave a note behind to tell the "install" command that the kernel
2450	# needs to be installed before the world.
2451	touch ${BDHASH}-install/kernelfirst
2452}
2453
2454# Make sure that all the file hashes mentioned in $@ have corresponding
2455# gzipped files stored in /files/.
2456install_verify () {
2457	# Generate a list of hashes
2458	cat $@ |
2459	    cut -f 2,7 -d '|' |
2460	    grep -E '^f' |
2461	    cut -f 2 -d '|' |
2462	    sort -u > filelist
2463
2464	# Make sure all the hashes exist
2465	while read HASH; do
2466		if ! [ -f files/${HASH}.gz ]; then
2467			echo -n "Update files missing -- "
2468			echo "this should never happen."
2469			echo "Re-run '$0 fetch'."
2470			return 1
2471		fi
2472	done < filelist
2473
2474	# Clean up
2475	rm filelist
2476}
2477
2478# Remove the system immutable flag from files
2479install_unschg () {
2480	# Generate file list
2481	cat $@ |
2482	    cut -f 1 -d '|' > filelist
2483
2484	# Remove flags
2485	while read F; do
2486		if ! [ -e ${BASEDIR}/${F} ]; then
2487			continue
2488		fi
2489
2490		chflags noschg ${BASEDIR}/${F} || return 1
2491	done < filelist
2492
2493	# Clean up
2494	rm filelist
2495}
2496
2497# Install new files
2498install_from_index () {
2499	# First pass: Do everything apart from setting file flags.  We
2500	# can't set flags yet, because schg inhibits hard linking.
2501	sort -k 1,1 -t '|' $1 |
2502	    tr '|' ' ' |
2503	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
2504		case ${TYPE} in
2505		d)
2506			# Create a directory
2507			install -d -o ${OWNER} -g ${GROUP}		\
2508			    -m ${PERM} ${BASEDIR}/${FPATH}
2509			;;
2510		f)
2511			if [ -z "${LINK}" ]; then
2512				# Create a file, without setting flags.
2513				gunzip < files/${HASH}.gz > ${HASH}
2514				install -S -o ${OWNER} -g ${GROUP}	\
2515				    -m ${PERM} ${HASH} ${BASEDIR}/${FPATH}
2516				rm ${HASH}
2517			else
2518				# Create a hard link.
2519				ln -f ${BASEDIR}/${LINK} ${BASEDIR}/${FPATH}
2520			fi
2521			;;
2522		L)
2523			# Create a symlink
2524			ln -sfh ${HASH} ${BASEDIR}/${FPATH}
2525			;;
2526		esac
2527	    done
2528
2529	# Perform a second pass, adding file flags.
2530	tr '|' ' ' < $1 |
2531	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
2532		if [ ${TYPE} = "f" ] &&
2533		    ! [ ${FLAGS} = "0" ]; then
2534			chflags ${FLAGS} ${BASEDIR}/${FPATH}
2535		fi
2536	    done
2537}
2538
2539# Remove files which we want to delete
2540install_delete () {
2541	# Generate list of new files
2542	cut -f 1 -d '|' < $2 |
2543	    sort > newfiles
2544
2545	# Generate subindex of old files we want to nuke
2546	sort -k 1,1 -t '|' $1 |
2547	    join -t '|' -v 1 - newfiles |
2548	    sort -r -k 1,1 -t '|' |
2549	    cut -f 1,2 -d '|' |
2550	    tr '|' ' ' > killfiles
2551
2552	# Remove the offending bits
2553	while read FPATH TYPE; do
2554		case ${TYPE} in
2555		d)
2556			rmdir ${BASEDIR}/${FPATH}
2557			;;
2558		f)
2559			rm ${BASEDIR}/${FPATH}
2560			;;
2561		L)
2562			rm ${BASEDIR}/${FPATH}
2563			;;
2564		esac
2565	done < killfiles
2566
2567	# Clean up
2568	rm newfiles killfiles
2569}
2570
2571# Install new files, delete old files, and update linker.hints
2572install_files () {
2573	# If we haven't already dealt with the kernel, deal with it.
2574	if ! [ -f $1/kerneldone ]; then
2575		grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
2576		grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
2577
2578		# Install new files
2579		install_from_index INDEX-NEW || return 1
2580
2581		# Remove files which need to be deleted
2582		install_delete INDEX-OLD INDEX-NEW || return 1
2583
2584		# Update linker.hints if necessary
2585		if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
2586			kldxref -R /boot/ 2>/dev/null
2587		fi
2588
2589		# We've finished updating the kernel.
2590		touch $1/kerneldone
2591
2592		# Do we need to ask for a reboot now?
2593		if [ -f $1/kernelfirst ] &&
2594		    [ -s INDEX-OLD -o -s INDEX-NEW ]; then
2595			cat <<-EOF
2596
2597Kernel updates have been installed.  Please reboot and run
2598"$0 install" again to finish installing updates.
2599			EOF
2600			exit 0
2601		fi
2602	fi
2603
2604	# If we haven't already dealt with the world, deal with it.
2605	if ! [ -f $1/worlddone ]; then
2606		# Install new shared libraries next
2607		grep -vE '^/boot/' $1/INDEX-NEW |
2608		    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
2609		install_from_index INDEX-NEW || return 1
2610
2611		# Deal with everything else
2612		grep -vE '^/boot/' $1/INDEX-OLD |
2613		    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
2614		grep -vE '^/boot/' $1/INDEX-NEW |
2615		    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
2616		install_from_index INDEX-NEW || return 1
2617		install_delete INDEX-OLD INDEX-NEW || return 1
2618
2619		# Rebuild /etc/spwd.db and /etc/pwd.db if necessary.
2620		if [ /etc/master.passwd -nt /etc/spwd.db ] ||
2621		    [ /etc/master.passwd -nt /etc/pwd.db ]; then
2622			pwd_mkdb /etc/master.passwd
2623		fi
2624
2625		# Rebuild /etc/login.conf.db if necessary.
2626		if [ /etc/login.conf -nt /etc/login.conf.db ]; then
2627			cap_mkdb /etc/login.conf
2628		fi
2629
2630		# We've finished installing the world and deleting old files
2631		# which are not shared libraries.
2632		touch $1/worlddone
2633
2634		# Do we need to ask the user to portupgrade now?
2635		grep -vE '^/boot/' $1/INDEX-NEW |
2636		    grep -E '/lib/.*\.so\.[0-9]+\|' |
2637		    cut -f 1 -d '|' |
2638		    sort > newfiles
2639		if grep -vE '^/boot/' $1/INDEX-OLD |
2640		    grep -E '/lib/.*\.so\.[0-9]+\|' |
2641		    cut -f 1 -d '|' |
2642		    sort |
2643		    join -v 1 - newfiles |
2644		    grep -q .; then
2645			cat <<-EOF
2646
2647Completing this upgrade requires removing old shared object files.
2648Please rebuild all installed 3rd party software (e.g., programs
2649installed from the ports tree) and then run "$0 install"
2650again to finish installing updates.
2651			EOF
2652			rm newfiles
2653			exit 0
2654		fi
2655		rm newfiles
2656	fi
2657
2658	# Remove old shared libraries
2659	grep -vE '^/boot/' $1/INDEX-NEW |
2660	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
2661	grep -vE '^/boot/' $1/INDEX-OLD |
2662	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
2663	install_delete INDEX-OLD INDEX-NEW || return 1
2664
2665	# Remove temporary files
2666	rm INDEX-OLD INDEX-NEW
2667}
2668
2669# Rearrange bits to allow the installed updates to be rolled back
2670install_setup_rollback () {
2671	# Remove the "reboot after installing kernel", "kernel updated", and
2672	# "finished installing the world" flags if present -- they are
2673	# irrelevant when rolling back updates.
2674	if [ -f ${BDHASH}-install/kernelfirst ]; then
2675		rm ${BDHASH}-install/kernelfirst
2676		rm ${BDHASH}-install/kerneldone
2677	fi
2678	if [ -f ${BDHASH}-install/worlddone ]; then
2679		rm ${BDHASH}-install/worlddone
2680	fi
2681
2682	if [ -L ${BDHASH}-rollback ]; then
2683		mv ${BDHASH}-rollback ${BDHASH}-install/rollback
2684	fi
2685
2686	mv ${BDHASH}-install ${BDHASH}-rollback
2687}
2688
2689# Actually install updates
2690install_run () {
2691	echo -n "Installing updates..."
2692
2693	# Make sure we have all the files we should have
2694	install_verify ${BDHASH}-install/INDEX-OLD	\
2695	    ${BDHASH}-install/INDEX-NEW || return 1
2696
2697	# Remove system immutable flag from files
2698	install_unschg ${BDHASH}-install/INDEX-OLD	\
2699	    ${BDHASH}-install/INDEX-NEW || return 1
2700
2701	# Install new files, delete old files, and update linker.hints
2702	install_files ${BDHASH}-install || return 1
2703
2704	# Rearrange bits to allow the installed updates to be rolled back
2705	install_setup_rollback
2706
2707	echo " done."
2708}
2709
2710# Rearrange bits to allow the previous set of updates to be rolled back next.
2711rollback_setup_rollback () {
2712	if [ -L ${BDHASH}-rollback/rollback ]; then
2713		mv ${BDHASH}-rollback/rollback rollback-tmp
2714		rm -r ${BDHASH}-rollback/
2715		rm ${BDHASH}-rollback
2716		mv rollback-tmp ${BDHASH}-rollback
2717	else
2718		rm -r ${BDHASH}-rollback/
2719		rm ${BDHASH}-rollback
2720	fi
2721}
2722
2723# Install old files, delete new files, and update linker.hints
2724rollback_files () {
2725	# Install old shared library files which don't have the same path as
2726	# a new shared library file.
2727	grep -vE '^/boot/' $1/INDEX-NEW |
2728	    grep -E '/lib/.*\.so\.[0-9]+\|' |
2729	    cut -f 1 -d '|' |
2730	    sort > INDEX-NEW.libs.flist
2731	grep -vE '^/boot/' $1/INDEX-OLD |
2732	    grep -E '/lib/.*\.so\.[0-9]+\|' |
2733	    sort -k 1,1 -t '|' - |
2734	    join -t '|' -v 1 - INDEX-NEW.libs.flist > INDEX-OLD
2735	install_from_index INDEX-OLD || return 1
2736
2737	# Deal with files which are neither kernel nor shared library
2738	grep -vE '^/boot/' $1/INDEX-OLD |
2739	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
2740	grep -vE '^/boot/' $1/INDEX-NEW |
2741	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
2742	install_from_index INDEX-OLD || return 1
2743	install_delete INDEX-NEW INDEX-OLD || return 1
2744
2745	# Install any old shared library files which we didn't install above.
2746	grep -vE '^/boot/' $1/INDEX-OLD |
2747	    grep -E '/lib/.*\.so\.[0-9]+\|' |
2748	    sort -k 1,1 -t '|' - |
2749	    join -t '|' - INDEX-NEW.libs.flist > INDEX-OLD
2750	install_from_index INDEX-OLD || return 1
2751
2752	# Delete unneeded shared library files
2753	grep -vE '^/boot/' $1/INDEX-OLD |
2754	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
2755	grep -vE '^/boot/' $1/INDEX-NEW |
2756	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
2757	install_delete INDEX-NEW INDEX-OLD || return 1
2758
2759	# Deal with kernel files
2760	grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
2761	grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
2762	install_from_index INDEX-OLD || return 1
2763	install_delete INDEX-NEW INDEX-OLD || return 1
2764	if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
2765		kldxref -R /boot/ 2>/dev/null
2766	fi
2767
2768	# Remove temporary files
2769	rm INDEX-OLD INDEX-NEW INDEX-NEW.libs.flist
2770}
2771
2772# Actually rollback updates
2773rollback_run () {
2774	echo -n "Uninstalling updates..."
2775
2776	# If there are updates waiting to be installed, remove them; we
2777	# want the user to re-run 'fetch' after rolling back updates.
2778	if [ -L ${BDHASH}-install ]; then
2779		rm -r ${BDHASH}-install/
2780		rm ${BDHASH}-install
2781	fi
2782
2783	# Make sure we have all the files we should have
2784	install_verify ${BDHASH}-rollback/INDEX-NEW	\
2785	    ${BDHASH}-rollback/INDEX-OLD || return 1
2786
2787	# Remove system immutable flag from files
2788	install_unschg ${BDHASH}-rollback/INDEX-NEW	\
2789	    ${BDHASH}-rollback/INDEX-OLD || return 1
2790
2791	# Install old files, delete new files, and update linker.hints
2792	rollback_files ${BDHASH}-rollback || return 1
2793
2794	# Remove the rollback directory and the symlink pointing to it; and
2795	# rearrange bits to allow the previous set of updates to be rolled
2796	# back next.
2797	rollback_setup_rollback
2798
2799	echo " done."
2800}
2801
2802# Compare INDEX-ALL and INDEX-PRESENT and print warnings about differences.
2803IDS_compare () {
2804	# Get all the lines which mismatch in something other than file
2805	# flags.  We ignore file flags because sysinstall doesn't seem to
2806	# set them when it installs FreeBSD; warning about these adds a
2807	# very large amount of noise.
2808	cut -f 1-5,7-8 -d '|' $1 > $1.noflags
2809	sort -k 1,1 -t '|' $1.noflags > $1.sorted
2810	cut -f 1-5,7-8 -d '|' $2 |
2811	    comm -13 $1.noflags - |
2812	    fgrep -v '|-|||||' |
2813	    sort -k 1,1 -t '|' |
2814	    join -t '|' $1.sorted - > INDEX-NOTMATCHING
2815
2816	# Ignore files which match IDSIGNOREPATHS.
2817	for X in ${IDSIGNOREPATHS}; do
2818		grep -E "^${X}" INDEX-NOTMATCHING
2819	done |
2820	    sort -u |
2821	    comm -13 - INDEX-NOTMATCHING > INDEX-NOTMATCHING.tmp
2822	mv INDEX-NOTMATCHING.tmp INDEX-NOTMATCHING
2823
2824	# Go through the lines and print warnings.
2825	while read LINE; do
2826		FPATH=`echo "${LINE}" | cut -f 1 -d '|'`
2827		TYPE=`echo "${LINE}" | cut -f 2 -d '|'`
2828		OWNER=`echo "${LINE}" | cut -f 3 -d '|'`
2829		GROUP=`echo "${LINE}" | cut -f 4 -d '|'`
2830		PERM=`echo "${LINE}" | cut -f 5 -d '|'`
2831		HASH=`echo "${LINE}" | cut -f 6 -d '|'`
2832		LINK=`echo "${LINE}" | cut -f 7 -d '|'`
2833		P_TYPE=`echo "${LINE}" | cut -f 8 -d '|'`
2834		P_OWNER=`echo "${LINE}" | cut -f 9 -d '|'`
2835		P_GROUP=`echo "${LINE}" | cut -f 10 -d '|'`
2836		P_PERM=`echo "${LINE}" | cut -f 11 -d '|'`
2837		P_HASH=`echo "${LINE}" | cut -f 12 -d '|'`
2838		P_LINK=`echo "${LINE}" | cut -f 13 -d '|'`
2839
2840		# Warn about different object types.
2841		if ! [ "${TYPE}" = "${P_TYPE}" ]; then
2842			echo -n "${FPATH} is a "
2843			case "${P_TYPE}" in
2844			f)	echo -n "regular file, "
2845				;;
2846			d)	echo -n "directory, "
2847				;;
2848			L)	echo -n "symlink, "
2849				;;
2850			esac
2851			echo -n "but should be a "
2852			case "${TYPE}" in
2853			f)	echo -n "regular file."
2854				;;
2855			d)	echo -n "directory."
2856				;;
2857			L)	echo -n "symlink."
2858				;;
2859			esac
2860			echo
2861
2862			# Skip other tests, since they don't make sense if
2863			# we're comparing different object types.
2864			continue
2865		fi
2866
2867		# Warn about different owners.
2868		if ! [ "${OWNER}" = "${P_OWNER}" ]; then
2869			echo -n "${FPATH} is owned by user id ${P_OWNER}, "
2870			echo "but should be owned by user id ${OWNER}."
2871		fi
2872
2873		# Warn about different groups.
2874		if ! [ "${GROUP}" = "${P_GROUP}" ]; then
2875			echo -n "${FPATH} is owned by group id ${P_GROUP}, "
2876			echo "but should be owned by group id ${GROUP}."
2877		fi
2878
2879		# Warn about different permissions.  We do not warn about
2880		# different permissions on symlinks, since some archivers
2881		# don't extract symlink permissions correctly and they are
2882		# ignored anyway.
2883		if ! [ "${PERM}" = "${P_PERM}" ] &&
2884		    ! [ "${TYPE}" = "L" ]; then
2885			echo -n "${FPATH} has ${P_PERM} permissions, "
2886			echo "but should have ${PERM} permissions."
2887		fi
2888
2889		# Warn about different file hashes / symlink destinations.
2890		if ! [ "${HASH}" = "${P_HASH}" ]; then
2891			if [ "${TYPE}" = "L" ]; then
2892				echo -n "${FPATH} is a symlink to ${P_HASH}, "
2893				echo "but should be a symlink to ${HASH}."
2894			fi
2895			if [ "${TYPE}" = "f" ]; then
2896				echo -n "${FPATH} has SHA256 hash ${P_HASH}, "
2897				echo "but should have SHA256 hash ${HASH}."
2898			fi
2899		fi
2900
2901		# We don't warn about different hard links, since some
2902		# some archivers break hard links, and as long as the
2903		# underlying data is correct they really don't matter.
2904	done < INDEX-NOTMATCHING
2905
2906	# Clean up
2907	rm $1 $1.noflags $1.sorted $2 INDEX-NOTMATCHING
2908}
2909
2910# Do the work involved in comparing the system to a "known good" index
2911IDS_run () {
2912	workdir_init || return 1
2913
2914	# Prepare the mirror list.
2915	fetch_pick_server_init && fetch_pick_server
2916
2917	# Try to fetch the public key until we run out of servers.
2918	while ! fetch_key; do
2919		fetch_pick_server || return 1
2920	done
2921
2922	# Try to fetch the metadata index signature ("tag") until we run
2923	# out of available servers; and sanity check the downloaded tag.
2924	while ! fetch_tag; do
2925		fetch_pick_server || return 1
2926	done
2927	fetch_tagsanity || return 1
2928
2929	# Fetch INDEX-OLD and INDEX-ALL.
2930	fetch_metadata INDEX-OLD INDEX-ALL || return 1
2931
2932	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
2933	# the components we want and without anything marked as "Ignore".
2934	fetch_filter_metadata INDEX-OLD || return 1
2935	fetch_filter_metadata INDEX-ALL || return 1
2936
2937	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-ALL.
2938	sort INDEX-OLD INDEX-ALL > INDEX-ALL.tmp
2939	mv INDEX-ALL.tmp INDEX-ALL
2940	rm INDEX-OLD
2941
2942	# Translate /boot/${KERNCONF} to ${KERNELDIR}
2943	fetch_filter_kernel_names INDEX-ALL ${KERNCONF}
2944
2945	# Inspect the system and generate an INDEX-PRESENT file.
2946	fetch_inspect_system INDEX-ALL INDEX-PRESENT /dev/null || return 1
2947
2948	# Compare INDEX-ALL and INDEX-PRESENT and print warnings about any
2949	# differences.
2950	IDS_compare INDEX-ALL INDEX-PRESENT
2951}
2952
2953#### Main functions -- call parameter-handling and core functions
2954
2955# Using the command line, configuration file, and defaults,
2956# set all the parameters which are needed later.
2957get_params () {
2958	init_params
2959	parse_cmdline $@
2960	parse_conffile
2961	default_params
2962}
2963
2964# Fetch command.  Make sure that we're being called
2965# interactively, then run fetch_check_params and fetch_run
2966cmd_fetch () {
2967	if [ ! -t 0 ]; then
2968		echo -n "`basename $0` fetch should not "
2969		echo "be run non-interactively."
2970		echo "Run `basename $0` cron instead."
2971		exit 1
2972	fi
2973	fetch_check_params
2974	fetch_run || exit 1
2975}
2976
2977# Cron command.  Make sure the parameters are sensible; wait
2978# rand(3600) seconds; then fetch updates.  While fetching updates,
2979# send output to a temporary file; only print that file if the
2980# fetching failed.
2981cmd_cron () {
2982	fetch_check_params
2983	sleep `jot -r 1 0 3600`
2984
2985	TMPFILE=`mktemp /tmp/freebsd-update.XXXXXX` || exit 1
2986	if ! fetch_run >> ${TMPFILE} ||
2987	    ! grep -q "No updates needed" ${TMPFILE} ||
2988	    [ ${VERBOSELEVEL} = "debug" ]; then
2989		mail -s "`hostname` security updates" ${MAILTO} < ${TMPFILE}
2990	fi
2991
2992	rm ${TMPFILE}
2993}
2994
2995# Fetch files for upgrading to a new release.
2996cmd_upgrade () {
2997	upgrade_check_params
2998	upgrade_run || exit 1
2999}
3000
3001# Install downloaded updates.
3002cmd_install () {
3003	install_check_params
3004	install_run || exit 1
3005}
3006
3007# Rollback most recently installed updates.
3008cmd_rollback () {
3009	rollback_check_params
3010	rollback_run || exit 1
3011}
3012
3013# Compare system against a "known good" index.
3014cmd_IDS () {
3015	IDS_check_params
3016	IDS_run || exit 1
3017}
3018
3019#### Entry point
3020
3021# Make sure we find utilities from the base system
3022export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH}
3023
3024# Set LC_ALL in order to avoid problems with character ranges like [A-Z].
3025export LC_ALL=C
3026
3027get_params $@
3028for COMMAND in ${COMMANDS}; do
3029	cmd_${COMMAND}
3030done
3031