xref: /freebsd/usr.sbin/freebsd-update/freebsd-update.sh (revision 1c4ee7dfb8affed302171232b0f612e6bcba3c10)
1#!/bin/sh
2
3#-
4# SPDX-License-Identifier: BSD-2-Clause
5#
6# Copyright 2004-2007 Colin Percival
7# All rights reserved
8#
9# Redistribution and use in source and binary forms, with or without
10# modification, are permitted providing that the following conditions
11# are met:
12# 1. Redistributions of source code must retain the above copyright
13#    notice, this list of conditions and the following disclaimer.
14# 2. Redistributions in binary form must reproduce the above copyright
15#    notice, this list of conditions and the following disclaimer in the
16#    documentation and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
22# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29
30#### Usage function -- called from command-line handling code.
31
32# Usage instructions.  Options not listed:
33# --debug	-- don't filter output from utilities
34# --no-stats	-- don't show progress statistics while fetching files
35usage () {
36	cat <<EOF
37usage: `basename $0` [options] command ...
38
39Options:
40  -b basedir   -- Operate on a system mounted at basedir
41                  (default: /)
42  -d workdir   -- Store working files in workdir
43                  (default: /var/db/freebsd-update/)
44  -f conffile  -- Read configuration options from conffile
45                  (default: /etc/freebsd-update.conf)
46  -F           -- Force a fetch operation to proceed in the
47                  case of an unfinished upgrade
48  -j jail      -- Operate on the given jail specified by jid or name
49  -k KEY       -- Trust an RSA key with SHA256 hash of KEY
50  -r release   -- Target for upgrade (e.g., 13.2-RELEASE)
51  -s server    -- Server from which to fetch updates
52                  (default: update.FreeBSD.org)
53  -t address   -- Mail output of cron command, if any, to address
54                  (default: root)
55  --not-running-from-cron
56               -- Run without a tty, for use by automated tools
57  --currently-running release
58               -- Update as if currently running this release
59Commands:
60  fetch        -- Fetch updates from server
61  cron         -- Sleep rand(3600) seconds, fetch updates, and send an
62                  email if updates were found
63  upgrade      -- Fetch upgrades to FreeBSD version specified via -r option
64  updatesready -- Check if there are fetched updates ready to install
65  install      -- Install downloaded updates or upgrades
66  rollback     -- Uninstall most recently installed updates
67  IDS          -- Compare the system against an index of "known good" files
68  showconfig   -- Show configuration
69EOF
70	exit 0
71}
72
73#### Configuration processing functions
74
75#-
76# Configuration options are set in the following order of priority:
77# 1. Command line options
78# 2. Configuration file options
79# 3. Default options
80# In addition, certain options (e.g., IgnorePaths) can be specified multiple
81# times and (as long as these are all in the same place, e.g., inside the
82# configuration file) they will accumulate.  Finally, because the path to the
83# configuration file can be specified at the command line, the entire command
84# line must be processed before we start reading the configuration file.
85#
86# Sound like a mess?  It is.  Here's how we handle this:
87# 1. Initialize CONFFILE and all the options to "".
88# 2. Process the command line.  Throw an error if a non-accumulating option
89#    is specified twice.
90# 3. If CONFFILE is "", set CONFFILE to /etc/freebsd-update.conf .
91# 4. For all the configuration options X, set X_saved to X.
92# 5. Initialize all the options to "".
93# 6. Read CONFFILE line by line, parsing options.
94# 7. For each configuration option X, set X to X_saved iff X_saved is not "".
95# 8. Repeat steps 4-7, except setting options to their default values at (6).
96
97CONFIGOPTIONS="KEYPRINT WORKDIR SERVERNAME MAILTO ALLOWADD ALLOWDELETE
98    KEEPMODIFIEDMETADATA COMPONENTS IGNOREPATHS UPDATEIFUNMODIFIED
99    BASEDIR VERBOSELEVEL TARGETRELEASE STRICTCOMPONENTS MERGECHANGES
100    IDSIGNOREPATHS BACKUPKERNEL BACKUPKERNELDIR BACKUPKERNELSYMBOLFILES"
101
102# Set all the configuration options to "".
103nullconfig () {
104	for X in ${CONFIGOPTIONS}; do
105		eval ${X}=""
106	done
107}
108
109# For each configuration option X, set X_saved to X.
110saveconfig () {
111	for X in ${CONFIGOPTIONS}; do
112		eval ${X}_saved=\$${X}
113	done
114}
115
116# For each configuration option X, set X to X_saved if X_saved is not "".
117mergeconfig () {
118	for X in ${CONFIGOPTIONS}; do
119		eval _=\$${X}_saved
120		if ! [ -z "${_}" ]; then
121			eval ${X}=\$${X}_saved
122		fi
123	done
124}
125
126# Set the trusted keyprint.
127config_KeyPrint () {
128	if [ -z ${KEYPRINT} ]; then
129		KEYPRINT=$1
130	else
131		return 1
132	fi
133}
134
135# Set the working directory.
136config_WorkDir () {
137	if [ -z ${WORKDIR} ]; then
138		WORKDIR=$1
139	else
140		return 1
141	fi
142}
143
144# Set the name of the server (pool) from which to fetch updates
145config_ServerName () {
146	if [ -z ${SERVERNAME} ]; then
147		SERVERNAME=$1
148	else
149		return 1
150	fi
151}
152
153# Set the address to which 'cron' output will be mailed.
154config_MailTo () {
155	if [ -z ${MAILTO} ]; then
156		MAILTO=$1
157	else
158		return 1
159	fi
160}
161
162# Set whether FreeBSD Update is allowed to add files (or directories, or
163# symlinks) which did not previously exist.
164config_AllowAdd () {
165	if [ -z ${ALLOWADD} ]; then
166		case $1 in
167		[Yy][Ee][Ss])
168			ALLOWADD=yes
169			;;
170		[Nn][Oo])
171			ALLOWADD=no
172			;;
173		*)
174			return 1
175			;;
176		esac
177	else
178		return 1
179	fi
180}
181
182# Set whether FreeBSD Update is allowed to remove files/directories/symlinks.
183config_AllowDelete () {
184	if [ -z ${ALLOWDELETE} ]; then
185		case $1 in
186		[Yy][Ee][Ss])
187			ALLOWDELETE=yes
188			;;
189		[Nn][Oo])
190			ALLOWDELETE=no
191			;;
192		*)
193			return 1
194			;;
195		esac
196	else
197		return 1
198	fi
199}
200
201# Set whether FreeBSD Update should keep existing inode ownership,
202# permissions, and flags, in the event that they have been modified locally
203# after the release.
204config_KeepModifiedMetadata () {
205	if [ -z ${KEEPMODIFIEDMETADATA} ]; then
206		case $1 in
207		[Yy][Ee][Ss])
208			KEEPMODIFIEDMETADATA=yes
209			;;
210		[Nn][Oo])
211			KEEPMODIFIEDMETADATA=no
212			;;
213		*)
214			return 1
215			;;
216		esac
217	else
218		return 1
219	fi
220}
221
222# Add to the list of components which should be kept updated.
223config_Components () {
224	for C in $@; do
225		COMPONENTS="${COMPONENTS} ${C}"
226	done
227}
228
229# Remove src component from list if it isn't installed
230finalize_components_config () {
231	COMPONENTS=""
232	for C in $@; do
233		if [ "$C" = "src" ]; then
234			if [ -e "${BASEDIR}/usr/src/COPYRIGHT" ]; then
235				COMPONENTS="${COMPONENTS} ${C}"
236			else
237				echo "src component not installed, skipped"
238			fi
239		else
240			COMPONENTS="${COMPONENTS} ${C}"
241		fi
242	done
243}
244
245# Add to the list of paths under which updates will be ignored.
246config_IgnorePaths () {
247	for C in $@; do
248		IGNOREPATHS="${IGNOREPATHS} ${C}"
249	done
250}
251
252# Add to the list of paths which IDS should ignore.
253config_IDSIgnorePaths () {
254	for C in $@; do
255		IDSIGNOREPATHS="${IDSIGNOREPATHS} ${C}"
256	done
257}
258
259# Add to the list of paths within which updates will be performed only if the
260# file on disk has not been modified locally.
261config_UpdateIfUnmodified () {
262	for C in $@; do
263		UPDATEIFUNMODIFIED="${UPDATEIFUNMODIFIED} ${C}"
264	done
265}
266
267# Add to the list of paths within which updates to text files will be merged
268# instead of overwritten.
269config_MergeChanges () {
270	for C in $@; do
271		MERGECHANGES="${MERGECHANGES} ${C}"
272	done
273}
274
275# Work on a FreeBSD installation mounted under $1
276config_BaseDir () {
277	if [ -z ${BASEDIR} ]; then
278		BASEDIR=$1
279	else
280		return 1
281	fi
282}
283
284# When fetching upgrades, should we assume the user wants exactly the
285# components listed in COMPONENTS, rather than trying to guess based on
286# what's currently installed?
287config_StrictComponents () {
288	if [ -z ${STRICTCOMPONENTS} ]; then
289		case $1 in
290		[Yy][Ee][Ss])
291			STRICTCOMPONENTS=yes
292			;;
293		[Nn][Oo])
294			STRICTCOMPONENTS=no
295			;;
296		*)
297			return 1
298			;;
299		esac
300	else
301		return 1
302	fi
303}
304
305# Upgrade to FreeBSD $1
306config_TargetRelease () {
307	if [ -z ${TARGETRELEASE} ]; then
308		TARGETRELEASE=$1
309	else
310		return 1
311	fi
312	if echo ${TARGETRELEASE} | grep -qE '^[0-9.]+$'; then
313		TARGETRELEASE="${TARGETRELEASE}-RELEASE"
314	fi
315}
316
317# Pretend current release is FreeBSD $1
318config_SourceRelease () {
319	UNAME_r=$1
320	if echo ${UNAME_r} | grep -qE '^[0-9.]+$'; then
321		UNAME_r="${UNAME_r}-RELEASE"
322	fi
323	export UNAME_r
324}
325
326# Get the Jail's path and the version of its installed userland
327config_TargetJail () {
328	JAIL=$1
329	UNAME_r=$(freebsd-version -j ${JAIL})
330	BASEDIR=$(jls -j ${JAIL} -h path | awk 'NR == 2 {print}')
331	if [ -z ${BASEDIR} ] || [ -z ${UNAME_r} ]; then
332		echo "The specified jail either doesn't exist or" \
333		      "does not have freebsd-version."
334		exit 1
335	fi
336	export UNAME_r
337}
338
339# Define what happens to output of utilities
340config_VerboseLevel () {
341	if [ -z ${VERBOSELEVEL} ]; then
342		case $1 in
343		[Dd][Ee][Bb][Uu][Gg])
344			VERBOSELEVEL=debug
345			;;
346		[Nn][Oo][Ss][Tt][Aa][Tt][Ss])
347			VERBOSELEVEL=nostats
348			;;
349		[Ss][Tt][Aa][Tt][Ss])
350			VERBOSELEVEL=stats
351			;;
352		*)
353			return 1
354			;;
355		esac
356	else
357		return 1
358	fi
359}
360
361config_BackupKernel () {
362	if [ -z ${BACKUPKERNEL} ]; then
363		case $1 in
364		[Yy][Ee][Ss])
365			BACKUPKERNEL=yes
366			;;
367		[Nn][Oo])
368			BACKUPKERNEL=no
369			;;
370		*)
371			return 1
372			;;
373		esac
374	else
375		return 1
376	fi
377}
378
379config_BackupKernelDir () {
380	if [ -z ${BACKUPKERNELDIR} ]; then
381		if [ -z "$1" ]; then
382			echo "BackupKernelDir set to empty dir"
383			return 1
384		fi
385
386		# We check for some paths which would be extremely odd
387		# to use, but which could cause a lot of problems if
388		# used.
389		case $1 in
390		/|/bin|/boot|/etc|/lib|/libexec|/sbin|/usr|/var)
391			echo "BackupKernelDir set to invalid path $1"
392			return 1
393			;;
394		/*)
395			BACKUPKERNELDIR=$1
396			;;
397		*)
398			echo "BackupKernelDir ($1) is not an absolute path"
399			return 1
400			;;
401		esac
402	else
403		return 1
404	fi
405}
406
407config_BackupKernelSymbolFiles () {
408	if [ -z ${BACKUPKERNELSYMBOLFILES} ]; then
409		case $1 in
410		[Yy][Ee][Ss])
411			BACKUPKERNELSYMBOLFILES=yes
412			;;
413		[Nn][Oo])
414			BACKUPKERNELSYMBOLFILES=no
415			;;
416		*)
417			return 1
418			;;
419		esac
420	else
421		return 1
422	fi
423}
424
425config_CreateBootEnv () {
426	if [ -z ${BOOTENV} ]; then
427		case $1 in
428		[Yy][Ee][Ss])
429			BOOTENV=yes
430			;;
431		[Nn][Oo])
432			BOOTENV=no
433			;;
434		*)
435			return 1
436			;;
437		esac
438	else
439		return 1
440	fi
441}
442# Handle one line of configuration
443configline () {
444	if [ $# -eq 0 ]; then
445		return
446	fi
447
448	OPT=$1
449	shift
450	config_${OPT} $@
451}
452
453#### Parameter handling functions.
454
455# Initialize parameters to null, just in case they're
456# set in the environment.
457init_params () {
458	# Configration settings
459	nullconfig
460
461	# No configuration file set yet
462	CONFFILE=""
463
464	# No commands specified yet
465	COMMANDS=""
466
467	# Force fetch to proceed
468	FORCEFETCH=0
469
470	# Run without a TTY
471	NOTTYOK=0
472
473	# Fetched first in a chain of commands
474	ISFETCHED=0
475}
476
477# Parse the command line
478parse_cmdline () {
479	while [ $# -gt 0 ]; do
480		case "$1" in
481		# Location of configuration file
482		-f)
483			if [ $# -eq 1 ]; then usage; fi
484			if [ ! -z "${CONFFILE}" ]; then usage; fi
485			shift; CONFFILE="$1"
486			;;
487		-F)
488			FORCEFETCH=1
489			;;
490		--not-running-from-cron)
491			NOTTYOK=1
492			;;
493		--currently-running)
494			shift
495			config_SourceRelease $1 || usage
496			;;
497
498		# Configuration file equivalents
499		-b)
500			if [ $# -eq 1 ]; then usage; fi; shift
501			config_BaseDir $1 || usage
502			;;
503		-d)
504			if [ $# -eq 1 ]; then usage; fi; shift
505			config_WorkDir $1 || usage
506			;;
507		-j)
508			if [ $# -eq 1 ]; then usage; fi; shift
509			config_TargetJail $1 || usage
510			;;
511		-k)
512			if [ $# -eq 1 ]; then usage; fi; shift
513			config_KeyPrint $1 || usage
514			;;
515		-s)
516			if [ $# -eq 1 ]; then usage; fi; shift
517			config_ServerName $1 || usage
518			;;
519		-r)
520			if [ $# -eq 1 ]; then usage; fi; shift
521			config_TargetRelease $1 || usage
522			;;
523		-t)
524			if [ $# -eq 1 ]; then usage; fi; shift
525			config_MailTo $1 || usage
526			;;
527		-v)
528			if [ $# -eq 1 ]; then usage; fi; shift
529			config_VerboseLevel $1 || usage
530			;;
531
532		# Aliases for "-v debug" and "-v nostats"
533		--debug)
534			config_VerboseLevel debug || usage
535			;;
536		--no-stats)
537			config_VerboseLevel nostats || usage
538			;;
539
540		# Commands
541		cron | fetch | upgrade | updatesready | install | rollback |\
542		IDS | showconfig)
543			COMMANDS="${COMMANDS} $1"
544			;;
545
546		# Anything else is an error
547		*)
548			usage
549			;;
550		esac
551		shift
552	done
553
554	# Make sure we have at least one command
555	if [ -z "${COMMANDS}" ]; then
556		usage
557	fi
558}
559
560# Parse the configuration file
561parse_conffile () {
562	# If a configuration file was specified on the command line, check
563	# that it exists and is readable.
564	if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then
565		echo -n "File does not exist "
566		echo -n "or is not readable: "
567		echo ${CONFFILE}
568		exit 1
569	fi
570
571	# If a configuration file was not specified on the command line,
572	# use the default configuration file path.  If that default does
573	# not exist, give up looking for any configuration.
574	if [ -z "${CONFFILE}" ]; then
575		CONFFILE="/etc/freebsd-update.conf"
576		if [ ! -r "${CONFFILE}" ]; then
577			return
578		fi
579	fi
580
581	# Save the configuration options specified on the command line, and
582	# clear all the options in preparation for reading the config file.
583	saveconfig
584	nullconfig
585
586	# Read the configuration file.  Anything after the first '#' is
587	# ignored, and any blank lines are ignored.
588	L=0
589	while read LINE; do
590		L=$(($L + 1))
591		LINEX=`echo "${LINE}" | cut -f 1 -d '#'`
592		if ! configline ${LINEX}; then
593			echo "Error processing configuration file, line $L:"
594			echo "==> ${LINE}"
595			exit 1
596		fi
597	done < ${CONFFILE}
598
599	# Merge the settings read from the configuration file with those
600	# provided at the command line.
601	mergeconfig
602}
603
604# Provide some default parameters
605default_params () {
606	# Save any parameters already configured, and clear the slate
607	saveconfig
608	nullconfig
609
610	# Default configurations
611	config_WorkDir /var/db/freebsd-update
612	config_MailTo root
613	config_AllowAdd yes
614	config_AllowDelete yes
615	config_KeepModifiedMetadata yes
616	config_BaseDir /
617	config_VerboseLevel stats
618	config_StrictComponents no
619	config_BackupKernel yes
620	config_BackupKernelDir /boot/kernel.old
621	config_BackupKernelSymbolFiles no
622	config_CreateBootEnv yes
623
624	# Merge these defaults into the earlier-configured settings
625	mergeconfig
626}
627
628# Set utility output filtering options, based on ${VERBOSELEVEL}
629fetch_setup_verboselevel () {
630	case ${VERBOSELEVEL} in
631	debug)
632		QUIETREDIR="/dev/stderr"
633		QUIETFLAG=" "
634		STATSREDIR="/dev/stderr"
635		DDSTATS=".."
636		XARGST="-t"
637		NDEBUG=" "
638		;;
639	nostats)
640		QUIETREDIR=""
641		QUIETFLAG=""
642		STATSREDIR="/dev/null"
643		DDSTATS=".."
644		XARGST=""
645		NDEBUG=""
646		;;
647	stats)
648		QUIETREDIR="/dev/null"
649		QUIETFLAG="-q"
650		STATSREDIR="/dev/stdout"
651		DDSTATS=""
652		XARGST=""
653		NDEBUG="-n"
654		;;
655	esac
656}
657
658# Check if there are any kernel modules installed from ports.
659# In that case warn the user that a rebuild from ports (i.e. not from
660# packages) might need necessary for the modules to work in the new release.
661upgrade_check_kmod_ports() {
662	local mod_name
663	local modules
664	local pattern
665	local pkg_name
666	local port_name
667	local report
668	local w
669
670	if ! command -v pkg >/dev/null; then
671		echo "Skipping kernel modules check. pkg(8) not present."
672		return
673	fi
674
675	# Most modules are in /boot/modules but we should actually look
676	# in every module_path passed to the kernel:
677	pattern=$(sysctl -n kern.module_path | tr ";" "|")
678
679	if [ -z "${pattern}" ]; then
680		echo "Empty kern.module_path sysctl. This should not happen."
681		echo "Aborting check of kernel modules installed from ports."
682		return
683	fi
684
685	# Check the pkg database for modules installed in those directories
686	modules=$(pkg query '%Fp' | grep -E "${pattern}")
687
688	if [ -z "${modules}" ]; then
689		return
690	fi
691
692	echo -e "\n"
693	echo "The following modules have been installed from packages."
694	echo "As a consequence they might not work when performing a major or minor upgrade."
695	echo -e "It is advised to rebuild these ports:\n"
696
697
698	report="Module Package Port\n------ ------- ----\n"
699	for module in ${modules}; do
700		w=$(pkg which "${module}")
701		mod_name=$(echo "${w}" | awk '{print $1;}')
702		pkg_name=$(echo "${w}" | awk '{print $6;}')
703		port_name=$(pkg info -o "${pkg_name}" | awk '{print $2;}')
704		report="${report}${mod_name} ${pkg_name} ${port_name}\n"
705	done
706
707	echo -e "${report}" | column -t
708	echo -e "\n"
709}
710
711# Perform sanity checks and set some final parameters
712# in preparation for fetching files.  Figure out which
713# set of updates should be downloaded: If the user is
714# running *-p[0-9]+, strip off the last part; if the
715# user is running -SECURITY, call it -RELEASE.  Chdir
716# into the working directory.
717fetchupgrade_check_params () {
718	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
719
720	_SERVERNAME_z=\
721"SERVERNAME must be given via command line or configuration file."
722	_KEYPRINT_z="Key must be given via -k option or configuration file."
723	_KEYPRINT_bad="Invalid key fingerprint: "
724	_WORKDIR_bad="Directory does not exist or is not writable: "
725	_WORKDIR_bad2="Directory is not on a persistent filesystem: "
726
727	if [ -z "${SERVERNAME}" ]; then
728		echo -n "`basename $0`: "
729		echo "${_SERVERNAME_z}"
730		exit 1
731	fi
732	if [ -z "${KEYPRINT}" ]; then
733		echo -n "`basename $0`: "
734		echo "${_KEYPRINT_z}"
735		exit 1
736	fi
737	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
738		echo -n "`basename $0`: "
739		echo -n "${_KEYPRINT_bad}"
740		echo ${KEYPRINT}
741		exit 1
742	fi
743	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
744		echo -n "`basename $0`: "
745		echo -n "${_WORKDIR_bad}"
746		echo ${WORKDIR}
747		exit 1
748	fi
749	case `df -T ${WORKDIR}` in */dev/md[0-9]* | *tmpfs*)
750		echo -n "`basename $0`: "
751		echo -n "${_WORKDIR_bad2}"
752		echo ${WORKDIR}
753		exit 1
754		;;
755	esac
756	chmod 700 ${WORKDIR}
757	cd ${WORKDIR} || exit 1
758
759	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
760	# to provide an upgrade path for FreeBSD Update 1.x users, since
761	# the kernels provided by FreeBSD Update 1.x are always labelled
762	# as X.Y-SECURITY.
763	RELNUM=`uname -r |
764	    sed -E 's,-p[0-9]+,,' |
765	    sed -E 's,-SECURITY,-RELEASE,'`
766	ARCH=`uname -m`
767	FETCHDIR=${RELNUM}/${ARCH}
768	PATCHDIR=${RELNUM}/${ARCH}/bp
769
770	# Disallow upgrade from a version that is not a release
771	case ${RELNUM} in
772	*-RELEASE | *-ALPHA*  | *-BETA* | *-RC*)
773		;;
774	*)
775		echo -n "`basename $0`: "
776		cat <<- EOF
777			Cannot upgrade from a version that is not a release
778			(including alpha, beta and release candidates)
779			using `basename $0`. Instead, FreeBSD can be directly
780			upgraded by source or upgraded to a RELEASE/RELENG version
781			prior to running `basename $0`.
782			Currently running: ${RELNUM}
783		EOF
784		exit 1
785		;;
786	esac
787
788	# Figure out what directory contains the running kernel
789	BOOTFILE=`sysctl -n kern.bootfile`
790	KERNELDIR=${BOOTFILE%/kernel}
791	if ! [ -d ${KERNELDIR} ]; then
792		echo "Cannot identify running kernel"
793		exit 1
794	fi
795
796	# Figure out what kernel configuration is running.  We start with
797	# the output of `uname -i`, and then make the following adjustments:
798	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
799	# file says "ident SMP-GENERIC", I don't know...
800	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
801	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
802	# we're running an SMP kernel.  This mis-identification is a bug
803	# which was fixed in 6.2-STABLE.
804	KERNCONF=`uname -i`
805	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
806		KERNCONF=SMP
807	fi
808	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
809		if sysctl kern.version | grep -qE '/SMP$'; then
810			KERNCONF=SMP
811		fi
812	fi
813
814	# Define some paths
815	BSPATCH=/usr/bin/bspatch
816	SHA256=/sbin/sha256
817	PHTTPGET=/usr/libexec/phttpget
818
819	# Set up variables relating to VERBOSELEVEL
820	fetch_setup_verboselevel
821
822	# Construct a unique name from ${BASEDIR}
823	BDHASH=`echo ${BASEDIR} | sha256 -q`
824}
825
826# Perform sanity checks etc. before fetching updates.
827fetch_check_params () {
828	fetchupgrade_check_params
829
830	if ! [ -z "${TARGETRELEASE}" ]; then
831		echo -n "`basename $0`: "
832		echo -n "'-r' option is meaningless with 'fetch' command.  "
833		echo "(Did you mean 'upgrade' instead?)"
834		exit 1
835	fi
836
837	# Check that we have updates ready to install
838	if [ -f ${BDHASH}-install/kerneldone -a $FORCEFETCH -eq 0 ]; then
839		echo "You have a partially completed upgrade pending"
840		echo "Run '`basename $0` [options] install' first."
841		echo "Run '`basename $0` [options] fetch -F' to proceed anyway."
842		exit 1
843	fi
844}
845
846# Perform sanity checks etc. before fetching upgrades.
847upgrade_check_params () {
848	fetchupgrade_check_params
849
850	# Unless set otherwise, we're upgrading to the same kernel config.
851	NKERNCONF=${KERNCONF}
852
853	# We need TARGETRELEASE set
854	_TARGETRELEASE_z="Release target must be specified via '-r' option."
855	if [ -z "${TARGETRELEASE}" ]; then
856		echo -n "`basename $0`: "
857		echo "${_TARGETRELEASE_z}"
858		exit 1
859	fi
860
861	# The target release should be != the current release.
862	if [ "${TARGETRELEASE}" = "${RELNUM}" ]; then
863		echo -n "`basename $0`: "
864		echo "Cannot upgrade from ${RELNUM} to itself"
865		exit 1
866	fi
867
868	# Turning off AllowAdd or AllowDelete is a bad idea for upgrades.
869	if [ "${ALLOWADD}" = "no" ]; then
870		echo -n "`basename $0`: "
871		echo -n "WARNING: \"AllowAdd no\" is a bad idea "
872		echo "when upgrading between releases."
873		echo
874	fi
875	if [ "${ALLOWDELETE}" = "no" ]; then
876		echo -n "`basename $0`: "
877		echo -n "WARNING: \"AllowDelete no\" is a bad idea "
878		echo "when upgrading between releases."
879		echo
880	fi
881
882	# Set EDITOR to /usr/bin/vi if it isn't already set
883	: ${EDITOR:='/usr/bin/vi'}
884}
885
886# Perform sanity checks and set some final parameters in
887# preparation for installing updates.
888install_check_params () {
889	# Check that we are root.  All sorts of things won't work otherwise.
890	if [ `id -u` != 0 ]; then
891		echo "You must be root to run this."
892		exit 1
893	fi
894
895	# Check that securelevel <= 0.  Otherwise we can't update schg files.
896	if [ `sysctl -n kern.securelevel` -gt 0 ]; then
897		echo "Updates cannot be installed when the system securelevel"
898		echo "is greater than zero."
899		exit 1
900	fi
901
902	# Check that we have a working directory
903	_WORKDIR_bad="Directory does not exist or is not writable: "
904	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
905		echo -n "`basename $0`: "
906		echo -n "${_WORKDIR_bad}"
907		echo ${WORKDIR}
908		exit 1
909	fi
910	cd ${WORKDIR} || exit 1
911
912	# Construct a unique name from ${BASEDIR}
913	BDHASH=`echo ${BASEDIR} | sha256 -q`
914
915	# Check that we have updates ready to install
916	if ! [ -L ${BDHASH}-install ]; then
917		echo "No updates are available to install."
918		if [ $ISFETCHED -eq 0 ]; then
919			echo "Run '`basename $0` [options] fetch' first."
920			exit 2
921		fi
922		exit 0
923	fi
924	if ! [ -f ${BDHASH}-install/INDEX-OLD ] ||
925	    ! [ -f ${BDHASH}-install/INDEX-NEW ]; then
926		echo "Update manifest is corrupt -- this should never happen."
927		echo "Re-run '`basename $0` [options] fetch'."
928		exit 1
929	fi
930
931	# Figure out what directory contains the running kernel
932	BOOTFILE=`sysctl -n kern.bootfile`
933	KERNELDIR=${BOOTFILE%/kernel}
934	if ! [ -d ${KERNELDIR} ]; then
935		echo "Cannot identify running kernel"
936		exit 1
937	fi
938}
939
940# Creates a new boot environment
941install_create_be () {
942	# Figure out if we're running in a jail and return if we are
943	if [ `sysctl -n security.jail.jailed` = 1 ]; then
944		return 1
945	fi
946	# Operating on roots that aren't located at / will, more often than not,
947	# not touch the boot environment.
948	if [ "$BASEDIR" != "/" ]; then
949		return 1
950	fi
951	# Create a boot environment if enabled
952	if [ ${BOOTENV} = yes ]; then
953		bectl check 2>/dev/null
954		case $? in
955			0)
956				# Boot environment are supported
957				CREATEBE=yes
958				;;
959			255)
960				# Boot environments are not supported
961				CREATEBE=no
962				;;
963			*)
964				# If bectl returns an unexpected exit code, don't create a BE
965				CREATEBE=no
966				;;
967		esac
968		if [ ${CREATEBE} = yes ]; then
969			echo -n "Creating snapshot of existing boot environment... "
970			VERSION=`freebsd-version -ku | sort -V | tail -n 1`
971			TIMESTAMP=`date +"%Y-%m-%d_%H%M%S"`
972			bectl create -r ${VERSION}_${TIMESTAMP}
973			if [ $? -eq 0 ]; then
974				echo "done.";
975			else
976				echo "failed."
977				exit 1
978			fi
979		fi
980	fi
981}
982
983# Perform sanity checks and set some final parameters in
984# preparation for UNinstalling updates.
985rollback_check_params () {
986	# Check that we are root.  All sorts of things won't work otherwise.
987	if [ `id -u` != 0 ]; then
988		echo "You must be root to run this."
989		exit 1
990	fi
991
992	# Check that we have a working directory
993	_WORKDIR_bad="Directory does not exist or is not writable: "
994	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
995		echo -n "`basename $0`: "
996		echo -n "${_WORKDIR_bad}"
997		echo ${WORKDIR}
998		exit 1
999	fi
1000	cd ${WORKDIR} || exit 1
1001
1002	# Construct a unique name from ${BASEDIR}
1003	BDHASH=`echo ${BASEDIR} | sha256 -q`
1004
1005	# Check that we have updates ready to rollback
1006	if ! [ -L ${BDHASH}-rollback ]; then
1007		echo "No rollback directory found."
1008		exit 1
1009	fi
1010	if ! [ -f ${BDHASH}-rollback/INDEX-OLD ] ||
1011	    ! [ -f ${BDHASH}-rollback/INDEX-NEW ]; then
1012		echo "Update manifest is corrupt -- this should never happen."
1013		exit 1
1014	fi
1015}
1016
1017# Perform sanity checks and set some final parameters
1018# in preparation for comparing the system against the
1019# published index.  Figure out which index we should
1020# compare against: If the user is running *-p[0-9]+,
1021# strip off the last part; if the user is running
1022# -SECURITY, call it -RELEASE.  Chdir into the working
1023# directory.
1024IDS_check_params () {
1025	export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
1026
1027	_SERVERNAME_z=\
1028"SERVERNAME must be given via command line or configuration file."
1029	_KEYPRINT_z="Key must be given via '-k' option or configuration file."
1030	_KEYPRINT_bad="Invalid key fingerprint: "
1031	_WORKDIR_bad="Directory does not exist or is not writable: "
1032
1033	if [ -z "${SERVERNAME}" ]; then
1034		echo -n "`basename $0`: "
1035		echo "${_SERVERNAME_z}"
1036		exit 1
1037	fi
1038	if [ -z "${KEYPRINT}" ]; then
1039		echo -n "`basename $0`: "
1040		echo "${_KEYPRINT_z}"
1041		exit 1
1042	fi
1043	if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
1044		echo -n "`basename $0`: "
1045		echo -n "${_KEYPRINT_bad}"
1046		echo ${KEYPRINT}
1047		exit 1
1048	fi
1049	if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
1050		echo -n "`basename $0`: "
1051		echo -n "${_WORKDIR_bad}"
1052		echo ${WORKDIR}
1053		exit 1
1054	fi
1055	cd ${WORKDIR} || exit 1
1056
1057	# Generate release number.  The s/SECURITY/RELEASE/ bit exists
1058	# to provide an upgrade path for FreeBSD Update 1.x users, since
1059	# the kernels provided by FreeBSD Update 1.x are always labelled
1060	# as X.Y-SECURITY.
1061	RELNUM=`uname -r |
1062	    sed -E 's,-p[0-9]+,,' |
1063	    sed -E 's,-SECURITY,-RELEASE,'`
1064	ARCH=`uname -m`
1065	FETCHDIR=${RELNUM}/${ARCH}
1066	PATCHDIR=${RELNUM}/${ARCH}/bp
1067
1068	# Figure out what directory contains the running kernel
1069	BOOTFILE=`sysctl -n kern.bootfile`
1070	KERNELDIR=${BOOTFILE%/kernel}
1071	if ! [ -d ${KERNELDIR} ]; then
1072		echo "Cannot identify running kernel"
1073		exit 1
1074	fi
1075
1076	# Figure out what kernel configuration is running.  We start with
1077	# the output of `uname -i`, and then make the following adjustments:
1078	# 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
1079	# file says "ident SMP-GENERIC", I don't know...
1080	# 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
1081	# _and_ `sysctl kern.version` contains a line which ends "/SMP", then
1082	# we're running an SMP kernel.  This mis-identification is a bug
1083	# which was fixed in 6.2-STABLE.
1084	KERNCONF=`uname -i`
1085	if [ ${KERNCONF} = "SMP-GENERIC" ]; then
1086		KERNCONF=SMP
1087	fi
1088	if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
1089		if sysctl kern.version | grep -qE '/SMP$'; then
1090			KERNCONF=SMP
1091		fi
1092	fi
1093
1094	# Define some paths
1095	SHA256=/sbin/sha256
1096	PHTTPGET=/usr/libexec/phttpget
1097
1098	# Set up variables relating to VERBOSELEVEL
1099	fetch_setup_verboselevel
1100}
1101
1102#### Core functionality -- the actual work gets done here
1103
1104# Use an SRV query to pick a server.  If the SRV query doesn't provide
1105# a useful answer, use the server name specified by the user.
1106# Put another way... look up _http._tcp.${SERVERNAME} and pick a server
1107# from that; or if no servers are returned, use ${SERVERNAME}.
1108# This allows a user to specify "update.FreeBSD.org" (in which case
1109# freebsd-update will select one of the mirrors) or "update1.freebsd.org"
1110# (in which case freebsd-update will use that particular server, since
1111# there won't be an SRV entry for that name).
1112#
1113# We ignore the Port field, since we are always going to use port 80.
1114
1115# Fetch the mirror list, but do not pick a mirror yet.  Returns 1 if
1116# no mirrors are available for any reason.
1117fetch_pick_server_init () {
1118	: > serverlist_tried
1119
1120# Check that host(1) exists (i.e., that the system wasn't built with the
1121# WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist.
1122	if ! which -s host; then
1123		: > serverlist_full
1124		return 1
1125	fi
1126
1127	echo -n "Looking up ${SERVERNAME} mirrors... "
1128
1129# Issue the SRV query and pull out the Priority, Weight, and Target fields.
1130# BIND 9 prints "$name has SRV record ..." while BIND 8 prints
1131# "$name server selection ..."; we allow either format.
1132	MLIST="_http._tcp.${SERVERNAME}"
1133	host -t srv "${MLIST}" |
1134	    sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" |
1135	    cut -f 1,2,4 -d ' ' |
1136	    sed -e 's/\.$//' |
1137	    sort > serverlist_full
1138
1139# If no records, give up -- we'll just use the server name we were given.
1140	if [ `wc -l < serverlist_full` -eq 0 ]; then
1141		echo "none found."
1142		return 1
1143	fi
1144
1145# Report how many mirrors we found.
1146	echo `wc -l < serverlist_full` "mirrors found."
1147
1148# Generate a random seed for use in picking mirrors.  If HTTP_PROXY
1149# is set, this will be used to generate the seed; otherwise, the seed
1150# will be random.
1151	if [ -n "${HTTP_PROXY}${http_proxy}" ]; then
1152		RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" |
1153		    tr -d 'a-f' |
1154		    cut -c 1-9`
1155	else
1156		RANDVALUE=`jot -r 1 0 999999999`
1157	fi
1158}
1159
1160# Pick a mirror.  Returns 1 if we have run out of mirrors to try.
1161fetch_pick_server () {
1162# Generate a list of not-yet-tried mirrors
1163	sort serverlist_tried |
1164	    comm -23 serverlist_full - > serverlist
1165
1166# Have we run out of mirrors?
1167	if [ `wc -l < serverlist` -eq 0 ]; then
1168		cat <<- EOF
1169			No mirrors remaining, giving up.
1170
1171			This may be because upgrading from this platform (${ARCH})
1172			or release (${RELNUM}) is unsupported by `basename $0`. Only
1173			platforms with Tier 1 support can be upgraded by `basename $0`.
1174			See https://www.freebsd.org/platforms/ for more info.
1175
1176			If unsupported, FreeBSD must be upgraded by source.
1177		EOF
1178		return 1
1179	fi
1180
1181# Find the highest priority level (lowest numeric value).
1182	SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1`
1183
1184# Add up the weights of the response lines at that priority level.
1185	SRV_WSUM=0;
1186	while read X; do
1187		case "$X" in
1188		${SRV_PRIORITY}\ *)
1189			SRV_W=`echo $X | cut -f 2 -d ' '`
1190			SRV_WSUM=$(($SRV_WSUM + $SRV_W))
1191			;;
1192		esac
1193	done < serverlist
1194
1195# If all the weights are 0, pretend that they are all 1 instead.
1196	if [ ${SRV_WSUM} -eq 0 ]; then
1197		SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l`
1198		SRV_W_ADD=1
1199	else
1200		SRV_W_ADD=0
1201	fi
1202
1203# Pick a value between 0 and the sum of the weights - 1
1204	SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}`
1205
1206# Read through the list of mirrors and set SERVERNAME.  Write the line
1207# corresponding to the mirror we selected into serverlist_tried so that
1208# we won't try it again.
1209	while read X; do
1210		case "$X" in
1211		${SRV_PRIORITY}\ *)
1212			SRV_W=`echo $X | cut -f 2 -d ' '`
1213			SRV_W=$(($SRV_W + $SRV_W_ADD))
1214			if [ $SRV_RND -lt $SRV_W ]; then
1215				SERVERNAME=`echo $X | cut -f 3 -d ' '`
1216				echo "$X" >> serverlist_tried
1217				break
1218			else
1219				SRV_RND=$(($SRV_RND - $SRV_W))
1220			fi
1221			;;
1222		esac
1223	done < serverlist
1224}
1225
1226# Take a list of ${oldhash}|${newhash} and output a list of needed patches,
1227# i.e., those for which we have ${oldhash} and don't have ${newhash}.
1228fetch_make_patchlist () {
1229	grep -vE "^([0-9a-f]{64})\|\1$" |
1230	    tr '|' ' ' |
1231		while read X Y; do
1232			if [ -f "files/${Y}.gz" ] ||
1233			    [ ! -f "files/${X}.gz" ]; then
1234				continue
1235			fi
1236			echo "${X}|${Y}"
1237		done | sort -u
1238}
1239
1240# Print user-friendly progress statistics
1241fetch_progress () {
1242	LNC=0
1243	while read x; do
1244		LNC=$(($LNC + 1))
1245		if [ $(($LNC % 10)) = 0 ]; then
1246			echo -n $LNC
1247		elif [ $(($LNC % 2)) = 0 ]; then
1248			echo -n .
1249		fi
1250	done
1251	echo -n " "
1252}
1253
1254# Function for asking the user if everything is ok
1255continuep () {
1256	while read -p "Does this look reasonable (y/n)? " CONTINUE; do
1257		case "${CONTINUE}" in
1258		[yY]*)
1259			return 0
1260			;;
1261		[nN]*)
1262			return 1
1263			;;
1264		esac
1265	done
1266}
1267
1268# Initialize the working directory
1269workdir_init () {
1270	mkdir -p files
1271	touch tINDEX.present
1272}
1273
1274# Check that we have a public key with an appropriate hash, or
1275# fetch the key if it doesn't exist.  Returns 1 if the key has
1276# not yet been fetched.
1277fetch_key () {
1278	if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
1279		return 0
1280	fi
1281
1282	echo -n "Fetching public key from ${SERVERNAME}... "
1283	rm -f pub.ssl
1284	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/pub.ssl \
1285	    2>${QUIETREDIR} || true
1286	if ! [ -r pub.ssl ]; then
1287		echo "failed."
1288		return 1
1289	fi
1290	if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
1291		echo "key has incorrect hash."
1292		rm -f pub.ssl
1293		return 1
1294	fi
1295	echo "done."
1296}
1297
1298# Fetch metadata signature, aka "tag".
1299fetch_tag () {
1300	echo -n "Fetching metadata signature "
1301	echo ${NDEBUG} "for ${RELNUM} from ${SERVERNAME}... "
1302	rm -f latest.ssl
1303	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/latest.ssl	\
1304	    2>${QUIETREDIR} || true
1305	if ! [ -r latest.ssl ]; then
1306		echo "failed."
1307		return 1
1308	fi
1309
1310	openssl rsautl -pubin -inkey pub.ssl -verify		\
1311	    < latest.ssl > tag.new 2>${QUIETREDIR} || true
1312	rm latest.ssl
1313
1314	if ! [ `wc -l < tag.new` = 1 ] ||
1315	    ! grep -qE	\
1316    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
1317		tag.new; then
1318		echo "invalid signature."
1319		return 1
1320	fi
1321
1322	echo "done."
1323
1324	RELPATCHNUM=`cut -f 4 -d '|' < tag.new`
1325	TINDEXHASH=`cut -f 5 -d '|' < tag.new`
1326	EOLTIME=`cut -f 6 -d '|' < tag.new`
1327}
1328
1329# Sanity-check the patch number in a tag, to make sure that we're not
1330# going to "update" backwards and to prevent replay attacks.
1331fetch_tagsanity () {
1332	# Check that we're not going to move from -pX to -pY with Y < X.
1333	RELPX=`uname -r | sed -E 's,.*-,,'`
1334	if echo ${RELPX} | grep -qE '^p[0-9]+$'; then
1335		RELPX=`echo ${RELPX} | cut -c 2-`
1336	else
1337		RELPX=0
1338	fi
1339	if [ "${RELPATCHNUM}" -lt "${RELPX}" ]; then
1340		echo
1341		echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1342		echo " appear older than what"
1343		echo "we are currently running (`uname -r`)!"
1344		echo "Cowardly refusing to proceed any further."
1345		return 1
1346	fi
1347
1348	# If "tag" exists and corresponds to ${RELNUM}, make sure that
1349	# it contains a patch number <= RELPATCHNUM, in order to protect
1350	# against rollback (replay) attacks.
1351	if [ -f tag ] &&
1352	    grep -qE	\
1353    "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
1354		tag; then
1355		LASTRELPATCHNUM=`cut -f 4 -d '|' < tag`
1356
1357		if [ "${RELPATCHNUM}" -lt "${LASTRELPATCHNUM}" ]; then
1358			echo
1359			echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1360			echo " are older than the"
1361			echo -n "most recently seen updates"
1362			echo " (${RELNUM}-p${LASTRELPATCHNUM})."
1363			echo "Cowardly refusing to proceed any further."
1364			return 1
1365		fi
1366	fi
1367}
1368
1369# Fetch metadata index file
1370fetch_metadata_index () {
1371	echo ${NDEBUG} "Fetching metadata index... "
1372	rm -f ${TINDEXHASH}
1373	fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/t/${TINDEXHASH}
1374	    2>${QUIETREDIR}
1375	if ! [ -f ${TINDEXHASH} ]; then
1376		echo "failed."
1377		return 1
1378	fi
1379	if [ `${SHA256} -q ${TINDEXHASH}` != ${TINDEXHASH} ]; then
1380		echo "update metadata index corrupt."
1381		return 1
1382	fi
1383	echo "done."
1384}
1385
1386# Print an error message about signed metadata being bogus.
1387fetch_metadata_bogus () {
1388	echo
1389	echo "The update metadata$1 is correctly signed, but"
1390	echo "failed an integrity check."
1391	echo "Cowardly refusing to proceed any further."
1392	return 1
1393}
1394
1395# Construct tINDEX.new by merging the lines named in $1 from ${TINDEXHASH}
1396# with the lines not named in $@ from tINDEX.present (if that file exists).
1397fetch_metadata_index_merge () {
1398	for METAFILE in $@; do
1399		if [ `grep -E "^${METAFILE}\|" ${TINDEXHASH} | wc -l`	\
1400		    -ne 1 ]; then
1401			fetch_metadata_bogus " index"
1402			return 1
1403		fi
1404
1405		grep -E "${METAFILE}\|" ${TINDEXHASH}
1406	done |
1407	    sort > tINDEX.wanted
1408
1409	if [ -f tINDEX.present ]; then
1410		join -t '|' -v 2 tINDEX.wanted tINDEX.present |
1411		    sort -m - tINDEX.wanted > tINDEX.new
1412		rm tINDEX.wanted
1413	else
1414		mv tINDEX.wanted tINDEX.new
1415	fi
1416}
1417
1418# Sanity check all the lines of tINDEX.new.  Even if more metadata lines
1419# are added by future versions of the server, this won't cause problems,
1420# since the only lines which appear in tINDEX.new are the ones which we
1421# specifically grepped out of ${TINDEXHASH}.
1422fetch_metadata_index_sanity () {
1423	if grep -qvE '^[0-9A-Z.-]+\|[0-9a-f]{64}$' tINDEX.new; then
1424		fetch_metadata_bogus " index"
1425		return 1
1426	fi
1427}
1428
1429# Sanity check the metadata file $1.
1430fetch_metadata_sanity () {
1431	# Some aliases to save space later: ${P} is a character which can
1432	# appear in a path; ${M} is the four numeric metadata fields; and
1433	# ${H} is a sha256 hash.
1434	P="[-+./:=,%@_[~[:alnum:]]"
1435	M="[0-9]+\|[0-9]+\|[0-9]+\|[0-9]+"
1436	H="[0-9a-f]{64}"
1437
1438	# Check that the first four fields make sense.
1439	if gunzip -c < files/$1.gz |
1440	    grep -qvE "^[a-z]+\|[0-9a-z-]+\|${P}+\|[fdL-]\|"; then
1441		fetch_metadata_bogus ""
1442		return 1
1443	fi
1444
1445	# Remove the first three fields.
1446	gunzip -c < files/$1.gz |
1447	    cut -f 4- -d '|' > sanitycheck.tmp
1448
1449	# Sanity check entries with type 'f'
1450	if grep -E '^f' sanitycheck.tmp |
1451	    grep -qvE "^f\|${M}\|${H}\|${P}*\$"; then
1452		fetch_metadata_bogus ""
1453		return 1
1454	fi
1455
1456	# Sanity check entries with type 'd'
1457	if grep -E '^d' sanitycheck.tmp |
1458	    grep -qvE "^d\|${M}\|\|\$"; then
1459		fetch_metadata_bogus ""
1460		return 1
1461	fi
1462
1463	# Sanity check entries with type 'L'
1464	if grep -E '^L' sanitycheck.tmp |
1465	    grep -qvE "^L\|${M}\|${P}*\|\$"; then
1466		fetch_metadata_bogus ""
1467		return 1
1468	fi
1469
1470	# Sanity check entries with type '-'
1471	if grep -E '^-' sanitycheck.tmp |
1472	    grep -qvE "^-\|\|\|\|\|\|"; then
1473		fetch_metadata_bogus ""
1474		return 1
1475	fi
1476
1477	# Clean up
1478	rm sanitycheck.tmp
1479}
1480
1481# Fetch the metadata index and metadata files listed in $@,
1482# taking advantage of metadata patches where possible.
1483fetch_metadata () {
1484	fetch_metadata_index || return 1
1485	fetch_metadata_index_merge $@ || return 1
1486	fetch_metadata_index_sanity || return 1
1487
1488	# Generate a list of wanted metadata patches
1489	join -t '|' -o 1.2,2.2 tINDEX.present tINDEX.new |
1490	    fetch_make_patchlist > patchlist
1491
1492	if [ -s patchlist ]; then
1493		# Attempt to fetch metadata patches
1494		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
1495		echo ${NDEBUG} "metadata patches.${DDSTATS}"
1496		tr '|' '-' < patchlist |
1497		    lam -s "${FETCHDIR}/tp/" - -s ".gz" |
1498		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1499			2>${STATSREDIR} | fetch_progress
1500		echo "done."
1501
1502		# Attempt to apply metadata patches
1503		echo -n "Applying metadata patches... "
1504		tr '|' ' ' < patchlist |
1505		    while read X Y; do
1506			if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
1507			gunzip -c < ${X}-${Y}.gz > diff
1508			gunzip -c < files/${X}.gz > diff-OLD
1509
1510			# Figure out which lines are being added and removed
1511			grep -E '^-' diff |
1512			    cut -c 2- |
1513			    while read PREFIX; do
1514				look "${PREFIX}" diff-OLD
1515			    done |
1516			    sort > diff-rm
1517			grep -E '^\+' diff |
1518			    cut -c 2- > diff-add
1519
1520			# Generate the new file
1521			comm -23 diff-OLD diff-rm |
1522			    sort - diff-add > diff-NEW
1523
1524			if [ `${SHA256} -q diff-NEW` = ${Y} ]; then
1525				mv diff-NEW files/${Y}
1526				gzip -n files/${Y}
1527			else
1528				mv diff-NEW ${Y}.bad
1529			fi
1530			rm -f ${X}-${Y}.gz diff
1531			rm -f diff-OLD diff-NEW diff-add diff-rm
1532		done 2>${QUIETREDIR}
1533		echo "done."
1534	fi
1535
1536	# Update metadata without patches
1537	cut -f 2 -d '|' < tINDEX.new |
1538	    while read Y; do
1539		if [ ! -f "files/${Y}.gz" ]; then
1540			echo ${Y};
1541		fi
1542	    done |
1543	    sort -u > filelist
1544
1545	if [ -s filelist ]; then
1546		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
1547		echo ${NDEBUG} "metadata files... "
1548		lam -s "${FETCHDIR}/m/" - -s ".gz" < filelist |
1549		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1550		    2>${QUIETREDIR}
1551
1552		while read Y; do
1553			if ! [ -f ${Y}.gz ]; then
1554				echo "failed."
1555				return 1
1556			fi
1557			if [ `gunzip -c < ${Y}.gz |
1558			    ${SHA256} -q` = ${Y} ]; then
1559				mv ${Y}.gz files/${Y}.gz
1560			else
1561				echo "metadata is corrupt."
1562				return 1
1563			fi
1564		done < filelist
1565		echo "done."
1566	fi
1567
1568# Sanity-check the metadata files.
1569	cut -f 2 -d '|' tINDEX.new > filelist
1570	while read X; do
1571		fetch_metadata_sanity ${X} || return 1
1572	done < filelist
1573
1574# Remove files which are no longer needed
1575	cut -f 2 -d '|' tINDEX.present |
1576	    sort > oldfiles
1577	cut -f 2 -d '|' tINDEX.new |
1578	    sort |
1579	    comm -13 - oldfiles |
1580	    lam -s "files/" - -s ".gz" |
1581	    xargs rm -f
1582	rm patchlist filelist oldfiles
1583	rm ${TINDEXHASH}
1584
1585# We're done!
1586	mv tINDEX.new tINDEX.present
1587	mv tag.new tag
1588
1589	return 0
1590}
1591
1592# Extract a subset of a downloaded metadata file containing only the parts
1593# which are listed in COMPONENTS.
1594fetch_filter_metadata_components () {
1595	METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
1596	gunzip -c < files/${METAHASH}.gz > $1.all
1597
1598	# Fish out the lines belonging to components we care about.
1599	for C in ${COMPONENTS}; do
1600		look "`echo ${C} | tr '/' '|'`|" $1.all
1601	done > $1
1602
1603	# Remove temporary file.
1604	rm $1.all
1605}
1606
1607# Generate a filtered version of the metadata file $1 from the downloaded
1608# file, by fishing out the lines corresponding to components we're trying
1609# to keep updated, and then removing lines corresponding to paths we want
1610# to ignore.
1611fetch_filter_metadata () {
1612	# Fish out the lines belonging to components we care about.
1613	fetch_filter_metadata_components $1
1614
1615	# Canonicalize directory names by removing any trailing / in
1616	# order to avoid listing directories multiple times if they
1617	# belong to multiple components.  Turning "/" into "" doesn't
1618	# matter, since we add a leading "/" when we use paths later.
1619	cut -f 3- -d '|' $1 |
1620	    sed -e 's,/|d|,|d|,' |
1621	    sed -e 's,/|-|,|-|,' |
1622	    sort -u > $1.tmp
1623
1624	# Figure out which lines to ignore and remove them.
1625	for X in ${IGNOREPATHS}; do
1626		grep -E "^${X}" $1.tmp
1627	done |
1628	    sort -u |
1629	    comm -13 - $1.tmp > $1
1630
1631	# Remove temporary files.
1632	rm $1.tmp
1633}
1634
1635# Filter the metadata file $1 by adding lines with "/boot/$2"
1636# replaced by ${KERNELDIR} (which is `sysctl -n kern.bootfile` minus the
1637# trailing "/kernel"); and if "/boot/$2" does not exist, remove
1638# the original lines which start with that.
1639# Put another way: Deal with the fact that the FOO kernel is sometimes
1640# installed in /boot/FOO/ and is sometimes installed elsewhere.
1641fetch_filter_kernel_names () {
1642	grep ^/boot/$2 $1 |
1643	    sed -e "s,/boot/$2,${KERNELDIR},g" |
1644	    sort - $1 > $1.tmp
1645	mv $1.tmp $1
1646
1647	if ! [ -d /boot/$2 ]; then
1648		grep -v ^/boot/$2 $1 > $1.tmp
1649		mv $1.tmp $1
1650	fi
1651}
1652
1653# For all paths appearing in $1 or $3, inspect the system
1654# and generate $2 describing what is currently installed.
1655fetch_inspect_system () {
1656	# No errors yet...
1657	rm -f .err
1658
1659	# Tell the user why his disk is suddenly making lots of noise
1660	echo -n "Inspecting system... "
1661
1662	# Generate list of files to inspect
1663	cat $1 $3 |
1664	    cut -f 1 -d '|' |
1665	    sort -u > filelist
1666
1667	# Examine each file and output lines of the form
1668	# /path/to/file|type|device-inum|user|group|perm|flags|value
1669	# sorted by device and inode number.
1670	while read F; do
1671		# If the symlink/file/directory does not exist, record this.
1672		if ! [ -e ${BASEDIR}/${F} ]; then
1673			echo "${F}|-||||||"
1674			continue
1675		fi
1676		if ! [ -r ${BASEDIR}/${F} ]; then
1677			echo "Cannot read file: ${BASEDIR}/${F}"	\
1678			    >/dev/stderr
1679			touch .err
1680			return 1
1681		fi
1682
1683		# Otherwise, output an index line.
1684		if [ -L ${BASEDIR}/${F} ]; then
1685			echo -n "${F}|L|"
1686			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1687			readlink ${BASEDIR}/${F};
1688		elif [ -f ${BASEDIR}/${F} ]; then
1689			echo -n "${F}|f|"
1690			stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1691			sha256 -q ${BASEDIR}/${F};
1692		elif [ -d ${BASEDIR}/${F} ]; then
1693			echo -n "${F}|d|"
1694			stat -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1695		else
1696			echo "Unknown file type: ${BASEDIR}/${F}"	\
1697			    >/dev/stderr
1698			touch .err
1699			return 1
1700		fi
1701	done < filelist |
1702	    sort -k 3,3 -t '|' > $2.tmp
1703	rm filelist
1704
1705	# Check if an error occurred during system inspection
1706	if [ -f .err ]; then
1707		return 1
1708	fi
1709
1710	# Convert to the form
1711	# /path/to/file|type|user|group|perm|flags|value|hlink
1712	# by resolving identical device and inode numbers into hard links.
1713	cut -f 1,3 -d '|' $2.tmp |
1714	    sort -k 1,1 -t '|' |
1715	    sort -s -u -k 2,2 -t '|' |
1716	    join -1 2 -2 3 -t '|' - $2.tmp |
1717	    awk -F \| -v OFS=\|		\
1718		'{
1719		    if (($2 == $3) || ($4 == "-"))
1720			print $3,$4,$5,$6,$7,$8,$9,""
1721		    else
1722			print $3,$4,$5,$6,$7,$8,$9,$2
1723		}' |
1724	    sort > $2
1725	rm $2.tmp
1726
1727	# We're finished looking around
1728	echo "done."
1729}
1730
1731# For any paths matching ${MERGECHANGES}, compare $2 against $1 and $3 and
1732# find any files with values unique to $2; generate $4 containing these paths
1733# and their corresponding hashes from $1.
1734fetch_filter_mergechanges () {
1735	# Pull out the paths and hashes of the files matching ${MERGECHANGES}.
1736	for F in $1 $2 $3; do
1737		for X in ${MERGECHANGES}; do
1738			grep -E "^${X}" ${F}
1739		done |
1740		    cut -f 1,2,7 -d '|' |
1741		    sort > ${F}-values
1742	done
1743
1744	# Any line in $2-values which doesn't appear in $1-values or $3-values
1745	# and is a file means that we should list the path in $3.
1746	sort $1-values $3-values |
1747	    comm -13 - $2-values |
1748	    fgrep '|f|' |
1749	    cut -f 1 -d '|' > $2-paths
1750
1751	# For each path, pull out one (and only one!) entry from $1-values.
1752	# Note that we cannot distinguish which "old" version the user made
1753	# changes to; but hopefully any changes which occur due to security
1754	# updates will exist in both the "new" version and the version which
1755	# the user has installed, so the merging will still work.
1756	while read X; do
1757		look "${X}|" $1-values |
1758		    head -1
1759	done < $2-paths > $4
1760
1761	# Clean up
1762	rm $1-values $2-values $3-values $2-paths
1763}
1764
1765# For any paths matching ${UPDATEIFUNMODIFIED}, remove lines from $[123]
1766# which correspond to lines in $2 with hashes not matching $1 or $3, unless
1767# the paths are listed in $4.  For entries in $2 marked "not present"
1768# (aka. type -), remove lines from $[123] unless there is a corresponding
1769# entry in $1.
1770fetch_filter_unmodified_notpresent () {
1771	# Figure out which lines of $1 and $3 correspond to bits which
1772	# should only be updated if they haven't changed, and fish out
1773	# the (path, type, value) tuples.
1774	# NOTE: We don't consider a file to be "modified" if it matches
1775	# the hash from $3.
1776	for X in ${UPDATEIFUNMODIFIED}; do
1777		grep -E "^${X}" $1
1778		grep -E "^${X}" $3
1779	done |
1780	    cut -f 1,2,7 -d '|' |
1781	    sort > $1-values
1782
1783	# Do the same for $2.
1784	for X in ${UPDATEIFUNMODIFIED}; do
1785		grep -E "^${X}" $2
1786	done |
1787	    cut -f 1,2,7 -d '|' |
1788	    sort > $2-values
1789
1790	# Any entry in $2-values which is not in $1-values corresponds to
1791	# a path which we need to remove from $1, $2, and $3, unless it
1792	# that path appears in $4.
1793	comm -13 $1-values $2-values |
1794	    sort -t '|' -k 1,1 > mlines.tmp
1795	cut -f 1 -d '|' $4 |
1796	    sort |
1797	    join -v 2 -t '|' - mlines.tmp |
1798	    sort > mlines
1799	rm $1-values $2-values mlines.tmp
1800
1801	# Any lines in $2 which are not in $1 AND are "not present" lines
1802	# also belong in mlines.
1803	comm -13 $1 $2 |
1804	    cut -f 1,2,7 -d '|' |
1805	    fgrep '|-|' >> mlines
1806
1807	# Remove lines from $1, $2, and $3
1808	for X in $1 $2 $3; do
1809		sort -t '|' -k 1,1 ${X} > ${X}.tmp
1810		cut -f 1 -d '|' < mlines |
1811		    sort |
1812		    join -v 2 -t '|' - ${X}.tmp |
1813		    sort > ${X}
1814		rm ${X}.tmp
1815	done
1816
1817	# Store a list of the modified files, for future reference
1818	fgrep -v '|-|' mlines |
1819	    cut -f 1 -d '|' > modifiedfiles
1820	rm mlines
1821}
1822
1823# For each entry in $1 of type -, remove any corresponding
1824# entry from $2 if ${ALLOWADD} != "yes".  Remove all entries
1825# of type - from $1.
1826fetch_filter_allowadd () {
1827	cut -f 1,2 -d '|' < $1 |
1828	    fgrep '|-' |
1829	    cut -f 1 -d '|' > filesnotpresent
1830
1831	if [ ${ALLOWADD} != "yes" ]; then
1832		sort < $2 |
1833		    join -v 1 -t '|' - filesnotpresent |
1834		    sort > $2.tmp
1835		mv $2.tmp $2
1836	fi
1837
1838	sort < $1 |
1839	    join -v 1 -t '|' - filesnotpresent |
1840	    sort > $1.tmp
1841	mv $1.tmp $1
1842	rm filesnotpresent
1843}
1844
1845# If ${ALLOWDELETE} != "yes", then remove any entries from $1
1846# which don't correspond to entries in $2.
1847fetch_filter_allowdelete () {
1848	# Produce a lists ${PATH}|${TYPE}
1849	for X in $1 $2; do
1850		cut -f 1-2 -d '|' < ${X} |
1851		    sort -u > ${X}.nodes
1852	done
1853
1854	# Figure out which lines need to be removed from $1.
1855	if [ ${ALLOWDELETE} != "yes" ]; then
1856		comm -23 $1.nodes $2.nodes > $1.badnodes
1857	else
1858		: > $1.badnodes
1859	fi
1860
1861	# Remove the relevant lines from $1
1862	while read X; do
1863		look "${X}|" $1
1864	done < $1.badnodes |
1865	    comm -13 - $1 > $1.tmp
1866	mv $1.tmp $1
1867
1868	rm $1.badnodes $1.nodes $2.nodes
1869}
1870
1871# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in $2
1872# with metadata not matching any entry in $1, replace the corresponding
1873# line of $3 with one having the same metadata as the entry in $2.
1874fetch_filter_modified_metadata () {
1875	# Fish out the metadata from $1 and $2
1876	for X in $1 $2; do
1877		cut -f 1-6 -d '|' < ${X} > ${X}.metadata
1878	done
1879
1880	# Find the metadata we need to keep
1881	if [ ${KEEPMODIFIEDMETADATA} = "yes" ]; then
1882		comm -13 $1.metadata $2.metadata > keepmeta
1883	else
1884		: > keepmeta
1885	fi
1886
1887	# Extract the lines which we need to remove from $3, and
1888	# construct the lines which we need to add to $3.
1889	: > $3.remove
1890	: > $3.add
1891	while read LINE; do
1892		NODE=`echo "${LINE}" | cut -f 1-2 -d '|'`
1893		look "${NODE}|" $3 >> $3.remove
1894		look "${NODE}|" $3 |
1895		    cut -f 7- -d '|' |
1896		    lam -s "${LINE}|" - >> $3.add
1897	done < keepmeta
1898
1899	# Remove the specified lines and add the new lines.
1900	sort $3.remove |
1901	    comm -13 - $3 |
1902	    sort -u - $3.add > $3.tmp
1903	mv $3.tmp $3
1904
1905	rm keepmeta $1.metadata $2.metadata $3.add $3.remove
1906}
1907
1908# Remove lines from $1 and $2 which are identical;
1909# no need to update a file if it isn't changing.
1910fetch_filter_uptodate () {
1911	comm -23 $1 $2 > $1.tmp
1912	comm -13 $1 $2 > $2.tmp
1913
1914	mv $1.tmp $1
1915	mv $2.tmp $2
1916}
1917
1918# Fetch any "clean" old versions of files we need for merging changes.
1919fetch_files_premerge () {
1920	# We only need to do anything if $1 is non-empty.
1921	if [ -s $1 ]; then
1922		# Tell the user what we're doing
1923		echo -n "Fetching files from ${OLDRELNUM} for merging... "
1924
1925		# List of files wanted
1926		fgrep '|f|' < $1 |
1927		    cut -f 3 -d '|' |
1928		    sort -u > files.wanted
1929
1930		# Only fetch the files we don't already have
1931		while read Y; do
1932			if [ ! -f "files/${Y}.gz" ]; then
1933				echo ${Y};
1934			fi
1935		done < files.wanted > filelist
1936
1937		# Actually fetch them
1938		lam -s "${OLDFETCHDIR}/f/" - -s ".gz" < filelist |
1939		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
1940		    2>${QUIETREDIR}
1941
1942		# Make sure we got them all, and move them into /files/
1943		while read Y; do
1944			if ! [ -f ${Y}.gz ]; then
1945				echo "failed."
1946				return 1
1947			fi
1948			if [ `gunzip -c < ${Y}.gz |
1949			    ${SHA256} -q` = ${Y} ]; then
1950				mv ${Y}.gz files/${Y}.gz
1951			else
1952				echo "${Y} has incorrect hash."
1953				return 1
1954			fi
1955		done < filelist
1956		echo "done."
1957
1958		# Clean up
1959		rm filelist files.wanted
1960	fi
1961}
1962
1963# Prepare to fetch files: Generate a list of the files we need,
1964# copy the unmodified files we have into /files/, and generate
1965# a list of patches to download.
1966fetch_files_prepare () {
1967	# Tell the user why his disk is suddenly making lots of noise
1968	echo -n "Preparing to download files... "
1969
1970	# Reduce indices to ${PATH}|${HASH} pairs
1971	for X in $1 $2 $3; do
1972		cut -f 1,2,7 -d '|' < ${X} |
1973		    fgrep '|f|' |
1974		    cut -f 1,3 -d '|' |
1975		    sort > ${X}.hashes
1976	done
1977
1978	# List of files wanted
1979	cut -f 2 -d '|' < $3.hashes |
1980	    sort -u |
1981	    while read HASH; do
1982		if ! [ -f files/${HASH}.gz ]; then
1983			echo ${HASH}
1984		fi
1985	done > files.wanted
1986
1987	# Generate a list of unmodified files
1988	comm -12 $1.hashes $2.hashes |
1989	    sort -k 1,1 -t '|' > unmodified.files
1990
1991	# Copy all files into /files/.  We only need the unmodified files
1992	# for use in patching; but we'll want all of them if the user asks
1993	# to rollback the updates later.
1994	while read LINE; do
1995		F=`echo "${LINE}" | cut -f 1 -d '|'`
1996		HASH=`echo "${LINE}" | cut -f 2 -d '|'`
1997
1998		# Skip files we already have.
1999		if [ -f files/${HASH}.gz ]; then
2000			continue
2001		fi
2002
2003		# Make sure the file hasn't changed.
2004		cp "${BASEDIR}/${F}" tmpfile
2005		if [ `sha256 -q tmpfile` != ${HASH} ]; then
2006			echo
2007			echo "File changed while FreeBSD Update running: ${F}"
2008			return 1
2009		fi
2010
2011		# Place the file into storage.
2012		gzip -c < tmpfile > files/${HASH}.gz
2013		rm tmpfile
2014	done < $2.hashes
2015
2016	# Produce a list of patches to download
2017	sort -k 1,1 -t '|' $3.hashes |
2018	    join -t '|' -o 2.2,1.2 - unmodified.files |
2019	    fetch_make_patchlist > patchlist
2020
2021	# Garbage collect
2022	rm unmodified.files $1.hashes $2.hashes $3.hashes
2023
2024	# We don't need the list of possible old files any more.
2025	rm $1
2026
2027	# We're finished making noise
2028	echo "done."
2029}
2030
2031# Fetch files.
2032fetch_files () {
2033	# Attempt to fetch patches
2034	if [ -s patchlist ]; then
2035		echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
2036		echo ${NDEBUG} "patches.${DDSTATS}"
2037		tr '|' '-' < patchlist |
2038		    lam -s "${PATCHDIR}/" - |
2039		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
2040			2>${STATSREDIR} | fetch_progress
2041		echo "done."
2042
2043		# Attempt to apply patches
2044		echo -n "Applying patches... "
2045		tr '|' ' ' < patchlist |
2046		    while read X Y; do
2047			if [ ! -f "${X}-${Y}" ]; then continue; fi
2048			gunzip -c < files/${X}.gz > OLD
2049
2050			bspatch OLD NEW ${X}-${Y}
2051
2052			if [ `${SHA256} -q NEW` = ${Y} ]; then
2053				mv NEW files/${Y}
2054				gzip -n files/${Y}
2055			fi
2056			rm -f diff OLD NEW ${X}-${Y}
2057		done 2>${QUIETREDIR}
2058		echo "done."
2059	fi
2060
2061	# Download files which couldn't be generate via patching
2062	while read Y; do
2063		if [ ! -f "files/${Y}.gz" ]; then
2064			echo ${Y};
2065		fi
2066	done < files.wanted > filelist
2067
2068	if [ -s filelist ]; then
2069		echo -n "Fetching `wc -l < filelist | tr -d ' '` "
2070		echo ${NDEBUG} "files... "
2071		lam -s "${FETCHDIR}/f/" - -s ".gz" < filelist |
2072		    xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}	\
2073			2>${STATSREDIR} | fetch_progress
2074
2075		while read Y; do
2076			if ! [ -f ${Y}.gz ]; then
2077				echo "failed."
2078				return 1
2079			fi
2080			if [ `gunzip -c < ${Y}.gz |
2081			    ${SHA256} -q` = ${Y} ]; then
2082				mv ${Y}.gz files/${Y}.gz
2083			else
2084				echo "${Y} has incorrect hash."
2085				return 1
2086			fi
2087		done < filelist
2088		echo "done."
2089	fi
2090
2091	# Clean up
2092	rm files.wanted filelist patchlist
2093}
2094
2095# Create and populate install manifest directory; and report what updates
2096# are available.
2097fetch_create_manifest () {
2098	# If we have an existing install manifest, nuke it.
2099	if [ -L "${BDHASH}-install" ]; then
2100		rm -r ${BDHASH}-install/
2101		rm ${BDHASH}-install
2102	fi
2103
2104	# Report to the user if any updates were avoided due to local changes
2105	if [ -s modifiedfiles ]; then
2106		cat - modifiedfiles <<- EOF | ${PAGER}
2107			The following files are affected by updates. No changes have
2108			been downloaded, however, because the files have been modified
2109			locally:
2110		EOF
2111	fi
2112	rm modifiedfiles
2113
2114	# If no files will be updated, tell the user and exit
2115	if ! [ -s INDEX-PRESENT ] &&
2116	    ! [ -s INDEX-NEW ]; then
2117		rm INDEX-PRESENT INDEX-NEW
2118		echo
2119		echo -n "No updates needed to update system to "
2120		echo "${RELNUM}-p${RELPATCHNUM}."
2121		return
2122	fi
2123
2124	# Divide files into (a) removed files, (b) added files, and
2125	# (c) updated files.
2126	cut -f 1 -d '|' < INDEX-PRESENT |
2127	    sort > INDEX-PRESENT.flist
2128	cut -f 1 -d '|' < INDEX-NEW |
2129	    sort > INDEX-NEW.flist
2130	comm -23 INDEX-PRESENT.flist INDEX-NEW.flist > files.removed
2131	comm -13 INDEX-PRESENT.flist INDEX-NEW.flist > files.added
2132	comm -12 INDEX-PRESENT.flist INDEX-NEW.flist > files.updated
2133	rm INDEX-PRESENT.flist INDEX-NEW.flist
2134
2135	# Report removed files, if any
2136	if [ -s files.removed ]; then
2137		cat - files.removed <<- EOF | ${PAGER}
2138			The following files will be removed as part of updating to
2139			${RELNUM}-p${RELPATCHNUM}:
2140		EOF
2141	fi
2142	rm files.removed
2143
2144	# Report added files, if any
2145	if [ -s files.added ]; then
2146		cat - files.added <<- EOF | ${PAGER}
2147			The following files will be added as part of updating to
2148			${RELNUM}-p${RELPATCHNUM}:
2149		EOF
2150	fi
2151	rm files.added
2152
2153	# Report updated files, if any
2154	if [ -s files.updated ]; then
2155		cat - files.updated <<- EOF | ${PAGER}
2156			The following files will be updated as part of updating to
2157			${RELNUM}-p${RELPATCHNUM}:
2158		EOF
2159	fi
2160	rm files.updated
2161
2162	# Create a directory for the install manifest.
2163	MDIR=`mktemp -d install.XXXXXX` || return 1
2164
2165	# Populate it
2166	mv INDEX-PRESENT ${MDIR}/INDEX-OLD
2167	mv INDEX-NEW ${MDIR}/INDEX-NEW
2168
2169	# Link it into place
2170	ln -s ${MDIR} ${BDHASH}-install
2171}
2172
2173# Warn about any upcoming EoL
2174fetch_warn_eol () {
2175	# What's the current time?
2176	NOWTIME=`date "+%s"`
2177
2178	# When did we last warn about the EoL date?
2179	if [ -f lasteolwarn ]; then
2180		LASTWARN=`cat lasteolwarn`
2181	else
2182		LASTWARN=`expr ${NOWTIME} - 63072000`
2183	fi
2184
2185	# If the EoL time is past, warn.
2186	if [ ${EOLTIME} -lt ${NOWTIME} ]; then
2187		echo
2188		cat <<-EOF
2189		WARNING: `uname -sr` HAS PASSED ITS END-OF-LIFE DATE.
2190		Any security issues discovered after `date -r ${EOLTIME}`
2191		will not have been corrected.
2192		EOF
2193		return 1
2194	fi
2195
2196	# Figure out how long it has been since we last warned about the
2197	# upcoming EoL, and how much longer we have left.
2198	SINCEWARN=`expr ${NOWTIME} - ${LASTWARN}`
2199	TIMELEFT=`expr ${EOLTIME} - ${NOWTIME}`
2200
2201	# Don't warn if the EoL is more than 3 months away
2202	if [ ${TIMELEFT} -gt 7884000 ]; then
2203		return 0
2204	fi
2205
2206	# Don't warn if the time remaining is more than 3 times the time
2207	# since the last warning.
2208	if [ ${TIMELEFT} -gt `expr ${SINCEWARN} \* 3` ]; then
2209		return 0
2210	fi
2211
2212	# Figure out what time units to use.
2213	if [ ${TIMELEFT} -lt 604800 ]; then
2214		UNIT="day"
2215		SIZE=86400
2216	elif [ ${TIMELEFT} -lt 2678400 ]; then
2217		UNIT="week"
2218		SIZE=604800
2219	else
2220		UNIT="month"
2221		SIZE=2678400
2222	fi
2223
2224	# Compute the right number of units
2225	NUM=`expr ${TIMELEFT} / ${SIZE}`
2226	if [ ${NUM} != 1 ]; then
2227		UNIT="${UNIT}s"
2228	fi
2229
2230	# Print the warning
2231	echo
2232	cat <<-EOF
2233		WARNING: `uname -sr` is approaching its End-of-Life date.
2234		It is strongly recommended that you upgrade to a newer
2235		release within the next ${NUM} ${UNIT}.
2236	EOF
2237
2238	# Update the stored time of last warning
2239	echo ${NOWTIME} > lasteolwarn
2240}
2241
2242# Do the actual work involved in "fetch" / "cron".
2243fetch_run () {
2244	workdir_init || return 1
2245
2246	# Prepare the mirror list.
2247	fetch_pick_server_init && fetch_pick_server
2248
2249	# Try to fetch the public key until we run out of servers.
2250	while ! fetch_key; do
2251		fetch_pick_server || return 1
2252	done
2253
2254	# Try to fetch the metadata index signature ("tag") until we run
2255	# out of available servers; and sanity check the downloaded tag.
2256	while ! fetch_tag; do
2257		fetch_pick_server || return 1
2258	done
2259	fetch_tagsanity || return 1
2260
2261	# Fetch the latest INDEX-NEW and INDEX-OLD files.
2262	fetch_metadata INDEX-NEW INDEX-OLD || return 1
2263
2264	# Generate filtered INDEX-NEW and INDEX-OLD files containing only
2265	# the lines which (a) belong to components we care about, and (b)
2266	# don't correspond to paths we're explicitly ignoring.
2267	fetch_filter_metadata INDEX-NEW || return 1
2268	fetch_filter_metadata INDEX-OLD || return 1
2269
2270	# Translate /boot/${KERNCONF} into ${KERNELDIR}
2271	fetch_filter_kernel_names INDEX-NEW ${KERNCONF}
2272	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
2273
2274	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
2275	# system and generate an INDEX-PRESENT file.
2276	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2277
2278	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
2279	# correspond to lines in INDEX-PRESENT with hashes not appearing
2280	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
2281	# INDEX-PRESENT has type - and there isn't a corresponding entry in
2282	# INDEX-OLD with type -.
2283	fetch_filter_unmodified_notpresent	\
2284	    INDEX-OLD INDEX-PRESENT INDEX-NEW /dev/null
2285
2286	# For each entry in INDEX-PRESENT of type -, remove any corresponding
2287	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
2288	# of type - from INDEX-PRESENT.
2289	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
2290
2291	# If ${ALLOWDELETE} != "yes", then remove any entries from
2292	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
2293	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
2294
2295	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
2296	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
2297	# replace the corresponding line of INDEX-NEW with one having the
2298	# same metadata as the entry in INDEX-PRESENT.
2299	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
2300
2301	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
2302	# no need to update a file if it isn't changing.
2303	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
2304
2305	# Prepare to fetch files: Generate a list of the files we need,
2306	# copy the unmodified files we have into /files/, and generate
2307	# a list of patches to download.
2308	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2309
2310	# Fetch files.
2311	fetch_files || return 1
2312
2313	# Create and populate install manifest directory; and report what
2314	# updates are available.
2315	fetch_create_manifest || return 1
2316
2317	# Warn about any upcoming EoL
2318	fetch_warn_eol || return 1
2319}
2320
2321# If StrictComponents is not "yes", generate a new components list
2322# with only the components which appear to be installed.
2323upgrade_guess_components () {
2324	if [ "${STRICTCOMPONENTS}" = "no" ]; then
2325		# Generate filtered INDEX-ALL with only the components listed
2326		# in COMPONENTS.
2327		fetch_filter_metadata_components $1 || return 1
2328
2329		# Tell the user why his disk is suddenly making lots of noise
2330		echo -n "Inspecting system... "
2331
2332		# Look at the files on disk, and assume that a component is
2333		# supposed to be present if it is more than half-present.
2334		cut -f 1-3 -d '|' < INDEX-ALL |
2335		    tr '|' ' ' |
2336		    while read C S F; do
2337			if [ -e ${BASEDIR}/${F} ]; then
2338				echo "+ ${C}|${S}"
2339			fi
2340			echo "= ${C}|${S}"
2341		    done |
2342		    sort |
2343		    uniq -c |
2344		    sed -E 's,^ +,,' > compfreq
2345		grep ' = ' compfreq |
2346		    cut -f 1,3 -d ' ' |
2347		    sort -k 2,2 -t ' ' > compfreq.total
2348		grep ' + ' compfreq |
2349		    cut -f 1,3 -d ' ' |
2350		    sort -k 2,2 -t ' ' > compfreq.present
2351		join -t ' ' -1 2 -2 2 compfreq.present compfreq.total |
2352		    while read S P T; do
2353			if [ ${T} -ne 0 -a ${P} -gt `expr ${T} / 2` ]; then
2354				echo ${S}
2355			fi
2356		    done > comp.present
2357		cut -f 2 -d ' ' < compfreq.total > comp.total
2358		rm INDEX-ALL compfreq compfreq.total compfreq.present
2359
2360		# We're done making noise.
2361		echo "done."
2362
2363		# Sometimes the kernel isn't installed where INDEX-ALL
2364		# thinks that it should be: In particular, it is often in
2365		# /boot/kernel instead of /boot/GENERIC or /boot/SMP.  To
2366		# deal with this, if "kernel|X" is listed in comp.total
2367		# (i.e., is a component which would be upgraded if it is
2368		# found to be present) we will add it to comp.present.
2369		# If "kernel|<anything>" is in comp.total but "kernel|X" is
2370		# not, we print a warning -- the user is running a kernel
2371		# which isn't part of the release.
2372		KCOMP=`echo ${KERNCONF} | tr 'A-Z' 'a-z'`
2373		grep -E "^kernel\|${KCOMP}\$" comp.total >> comp.present
2374
2375		if grep -qE "^kernel\|" comp.total &&
2376		    ! grep -qE "^kernel\|${KCOMP}\$" comp.total; then
2377			cat <<-EOF
2378
2379WARNING: This system is running a "${KCOMP}" kernel, which is not a
2380kernel configuration distributed as part of FreeBSD ${RELNUM}.
2381This kernel will not be updated: you MUST update the kernel manually
2382before running '`basename $0` [options] install'.
2383			EOF
2384		fi
2385
2386		# Re-sort the list of installed components and generate
2387		# the list of non-installed components.
2388		sort -u < comp.present > comp.present.tmp
2389		mv comp.present.tmp comp.present
2390		comm -13 comp.present comp.total > comp.absent
2391
2392		# Ask the user to confirm that what we have is correct.  To
2393		# reduce user confusion, translate "X|Y" back to "X/Y" (as
2394		# subcomponents must be listed in the configuration file).
2395		echo
2396		echo -n "The following components of FreeBSD "
2397		echo "seem to be installed:"
2398		tr '|' '/' < comp.present |
2399		    fmt -72
2400		echo
2401		echo -n "The following components of FreeBSD "
2402		echo "do not seem to be installed:"
2403		tr '|' '/' < comp.absent |
2404		    fmt -72
2405		echo
2406		continuep || return 1
2407		echo
2408
2409		# Suck the generated list of components into ${COMPONENTS}.
2410		# Note that comp.present.tmp is used due to issues with
2411		# pipelines and setting variables.
2412		COMPONENTS=""
2413		tr '|' '/' < comp.present > comp.present.tmp
2414		while read C; do
2415			COMPONENTS="${COMPONENTS} ${C}"
2416		done < comp.present.tmp
2417
2418		# Delete temporary files
2419		rm comp.present comp.present.tmp comp.absent comp.total
2420	fi
2421}
2422
2423# If StrictComponents is not "yes", COMPONENTS contains an entry
2424# corresponding to the currently running kernel, and said kernel
2425# does not exist in the new release, add "kernel/generic" to the
2426# list of components.
2427upgrade_guess_new_kernel () {
2428	if [ "${STRICTCOMPONENTS}" = "no" ]; then
2429		# Grab the unfiltered metadata file.
2430		METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
2431		gunzip -c < files/${METAHASH}.gz > $1.all
2432
2433		# If "kernel/${KCOMP}" is in ${COMPONENTS} and that component
2434		# isn't in $1.all, we need to add kernel/generic.
2435		for C in ${COMPONENTS}; do
2436			if [ ${C} = "kernel/${KCOMP}" ] &&
2437			    ! grep -qE "^kernel\|${KCOMP}\|" $1.all; then
2438				COMPONENTS="${COMPONENTS} kernel/generic"
2439				NKERNCONF="GENERIC"
2440				cat <<-EOF
2441
2442WARNING: This system is running a "${KCOMP}" kernel, which is not a
2443kernel configuration distributed as part of FreeBSD ${RELNUM}.
2444As part of upgrading to FreeBSD ${RELNUM}, this kernel will be
2445replaced with a "generic" kernel.
2446				EOF
2447				continuep || return 1
2448			fi
2449		done
2450
2451		# Don't need this any more...
2452		rm $1.all
2453	fi
2454}
2455
2456# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2457# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2458upgrade_oldall_to_oldnew () {
2459	# For each ${F}|... which appears in INDEX-ALL but does not appear
2460	# in INDEX-OLD, add ${F}|-|||||| to INDEX-OLD.
2461	cut -f 1 -d '|' < $1 |
2462	    sort -u > $1.paths
2463	cut -f 1 -d '|' < $2 |
2464	    sort -u |
2465	    comm -13 $1.paths - |
2466	    lam - -s "|-||||||" |
2467	    sort - $1 > $1.tmp
2468	mv $1.tmp $1
2469
2470	# Remove lines from INDEX-OLD which also appear in INDEX-ALL
2471	comm -23 $1 $2 > $1.tmp
2472	mv $1.tmp $1
2473
2474	# Remove lines from INDEX-ALL which have a file name not appearing
2475	# anywhere in INDEX-OLD (since these must be files which haven't
2476	# changed -- if they were new, there would be an entry of type "-").
2477	cut -f 1 -d '|' < $1 |
2478	    sort -u > $1.paths
2479	sort -k 1,1 -t '|' < $2 |
2480	    join -t '|' - $1.paths |
2481	    sort > $2.tmp
2482	rm $1.paths
2483	mv $2.tmp $2
2484
2485	# Rename INDEX-ALL to INDEX-NEW.
2486	mv $2 $3
2487}
2488
2489# Helper for upgrade_merge: Return zero true iff the two files differ only
2490# in the contents of their RCS tags.
2491samef () {
2492	X=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $1 | ${SHA256}`
2493	Y=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $2 | ${SHA256}`
2494
2495	if [ $X = $Y ]; then
2496		return 0;
2497	else
2498		return 1;
2499	fi
2500}
2501
2502# From the list of "old" files in $1, merge changes in $2 with those in $3,
2503# and update $3 to reflect the hashes of merged files.
2504upgrade_merge () {
2505	# We only need to do anything if $1 is non-empty.
2506	if [ -s $1 ]; then
2507		cut -f 1 -d '|' $1 |
2508		    sort > $1-paths
2509
2510		# Create staging area for merging files
2511		rm -rf merge/
2512		while read F; do
2513			D=`dirname ${F}`
2514			mkdir -p merge/old/${D}
2515			mkdir -p merge/${OLDRELNUM}/${D}
2516			mkdir -p merge/${RELNUM}/${D}
2517			mkdir -p merge/new/${D}
2518		done < $1-paths
2519
2520		# Copy in files
2521		while read F; do
2522			# Currently installed file
2523			V=`look "${F}|" $2 | cut -f 7 -d '|'`
2524			gunzip < files/${V}.gz > merge/old/${F}
2525
2526			# Old release
2527			if look "${F}|" $1 | fgrep -q "|f|"; then
2528				V=`look "${F}|" $1 | cut -f 3 -d '|'`
2529				gunzip < files/${V}.gz		\
2530				    > merge/${OLDRELNUM}/${F}
2531			fi
2532
2533			# New release
2534			if look "${F}|" $3 | cut -f 1,2,7 -d '|' |
2535			    fgrep -q "|f|"; then
2536				V=`look "${F}|" $3 | cut -f 7 -d '|'`
2537				gunzip < files/${V}.gz		\
2538				    > merge/${RELNUM}/${F}
2539			fi
2540		done < $1-paths
2541
2542		# Attempt to automatically merge changes
2543		echo -n "Attempting to automatically merge "
2544		echo -n "changes in files..."
2545		: > failed.merges
2546		while read F; do
2547			# If the file doesn't exist in the new release,
2548			# the result of "merging changes" is having the file
2549			# not exist.
2550			if ! [ -f merge/${RELNUM}/${F} ]; then
2551				continue
2552			fi
2553
2554			# If the file didn't exist in the old release, we're
2555			# going to throw away the existing file and hope that
2556			# the version from the new release is what we want.
2557			if ! [ -f merge/${OLDRELNUM}/${F} ]; then
2558				cp merge/${RELNUM}/${F} merge/new/${F}
2559				continue
2560			fi
2561
2562			# Some files need special treatment.
2563			case ${F} in
2564			/etc/spwd.db | /etc/pwd.db | /etc/login.conf.db)
2565				# Don't merge these -- we're rebuild them
2566				# after updates are installed.
2567				cp merge/old/${F} merge/new/${F}
2568				;;
2569			*)
2570				if ! diff3 -E -m -L "current version"	\
2571				    -L "${OLDRELNUM}" -L "${RELNUM}"	\
2572				    merge/old/${F}			\
2573				    merge/${OLDRELNUM}/${F}		\
2574				    merge/${RELNUM}/${F}		\
2575				    > merge/new/${F} 2>/dev/null; then
2576					echo ${F} >> failed.merges
2577				fi
2578				;;
2579			esac
2580		done < $1-paths
2581		echo " done."
2582
2583		# Ask the user to handle any files which didn't merge.
2584		while read F; do
2585			# If the installed file differs from the version in
2586			# the old release only due to RCS tag expansion
2587			# then just use the version in the new release.
2588			if samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
2589				cp merge/${RELNUM}/${F} merge/new/${F}
2590				continue
2591			fi
2592
2593			cat <<-EOF
2594
2595The following file could not be merged automatically: ${F}
2596Press Enter to edit this file in ${EDITOR} and resolve the conflicts
2597manually...
2598			EOF
2599			while true; do
2600				read response </dev/tty
2601				if expr "${response}" : '[Aa][Cc][Cc][Ee][Pp][Tt]' > /dev/null; then
2602					echo
2603					break
2604				fi
2605				${EDITOR} `pwd`/merge/new/${F} < /dev/tty
2606
2607				if ! grep -qE '^(<<<<<<<|=======|>>>>>>>)([[:space:]].*)?$' $(pwd)/merge/new/${F} ; then
2608					break
2609				fi
2610				cat <<-EOF
2611
2612Merge conflict markers remain in: ${F}
2613These must be resolved for the system to be functional.
2614
2615Press Enter to return to editing this file, or type "ACCEPT" to carry on with
2616these lines remaining in the file.
2617				EOF
2618			done
2619		done < failed.merges
2620		rm failed.merges
2621
2622		# Ask the user to confirm that he likes how the result
2623		# of merging files.
2624		while read F; do
2625			# Skip files which haven't changed except possibly
2626			# in their RCS tags.
2627			if [ -f merge/old/${F} ] && [ -f merge/new/${F} ] &&
2628			    samef merge/old/${F} merge/new/${F}; then
2629				continue
2630			fi
2631
2632			# Skip files where the installed file differs from
2633			# the old file only due to RCS tags.
2634			if [ -f merge/old/${F} ] &&
2635			    [ -f merge/${OLDRELNUM}/${F} ] &&
2636			    samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
2637				continue
2638			fi
2639
2640			# Warn about files which are ceasing to exist.
2641			if ! [ -f merge/new/${F} ]; then
2642				cat <<-EOF
2643
2644The following file will be removed, as it no longer exists in
2645FreeBSD ${RELNUM}: ${F}
2646				EOF
2647				continuep < /dev/tty || return 1
2648				continue
2649			fi
2650
2651			# Print changes for the user's approval.
2652			cat <<-EOF
2653
2654The following changes, which occurred between FreeBSD ${OLDRELNUM} and
2655FreeBSD ${RELNUM} have been merged into ${F}:
2656EOF
2657			diff -U 5 -L "current version" -L "new version"	\
2658			    merge/old/${F} merge/new/${F} || true
2659			continuep < /dev/tty || return 1
2660		done < $1-paths
2661
2662		# Store merged files.
2663		while read F; do
2664			if [ -f merge/new/${F} ]; then
2665				V=`${SHA256} -q merge/new/${F}`
2666
2667				gzip -c < merge/new/${F} > files/${V}.gz
2668				echo "${F}|${V}"
2669			fi
2670		done < $1-paths > newhashes
2671
2672		# Pull lines out from $3 which need to be updated to
2673		# reflect merged files.
2674		while read F; do
2675			look "${F}|" $3
2676		done < $1-paths > $3-oldlines
2677
2678		# Update lines to reflect merged files
2679		join -t '|' -o 1.1,1.2,1.3,1.4,1.5,1.6,2.2,1.8		\
2680		    $3-oldlines newhashes > $3-newlines
2681
2682		# Remove old lines from $3 and add new lines.
2683		sort $3-oldlines |
2684		    comm -13 - $3 |
2685		    sort - $3-newlines > $3.tmp
2686		mv $3.tmp $3
2687
2688		# Clean up
2689		rm $1-paths newhashes $3-oldlines $3-newlines
2690		rm -rf merge/
2691	fi
2692
2693	# We're done with merging files.
2694	rm $1
2695}
2696
2697# Do the work involved in fetching upgrades to a new release
2698upgrade_run () {
2699	workdir_init || return 1
2700
2701	# Prepare the mirror list.
2702	fetch_pick_server_init && fetch_pick_server
2703
2704	# Try to fetch the public key until we run out of servers.
2705	while ! fetch_key; do
2706		fetch_pick_server || return 1
2707	done
2708
2709	# Try to fetch the metadata index signature ("tag") until we run
2710	# out of available servers; and sanity check the downloaded tag.
2711	while ! fetch_tag; do
2712		fetch_pick_server || return 1
2713	done
2714	fetch_tagsanity || return 1
2715
2716	# Fetch the INDEX-OLD and INDEX-ALL.
2717	fetch_metadata INDEX-OLD INDEX-ALL || return 1
2718
2719	# If StrictComponents is not "yes", generate a new components list
2720	# with only the components which appear to be installed.
2721	upgrade_guess_components INDEX-ALL || return 1
2722
2723	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
2724	# the components we want and without anything marked as "Ignore".
2725	fetch_filter_metadata INDEX-OLD || return 1
2726	fetch_filter_metadata INDEX-ALL || return 1
2727
2728	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-OLD.
2729	sort INDEX-OLD INDEX-ALL > INDEX-OLD.tmp
2730	mv INDEX-OLD.tmp INDEX-OLD
2731	rm INDEX-ALL
2732
2733	# Adjust variables for fetching files from the new release.
2734	OLDRELNUM=${RELNUM}
2735	RELNUM=${TARGETRELEASE}
2736	OLDFETCHDIR=${FETCHDIR}
2737	FETCHDIR=${RELNUM}/${ARCH}
2738
2739	# Try to fetch the NEW metadata index signature ("tag") until we run
2740	# out of available servers; and sanity check the downloaded tag.
2741	while ! fetch_tag; do
2742		fetch_pick_server || return 1
2743	done
2744
2745	# Fetch the new INDEX-ALL.
2746	fetch_metadata INDEX-ALL || return 1
2747
2748	# If StrictComponents is not "yes", COMPONENTS contains an entry
2749	# corresponding to the currently running kernel, and said kernel
2750	# does not exist in the new release, add "kernel/generic" to the
2751	# list of components.
2752	upgrade_guess_new_kernel INDEX-ALL || return 1
2753
2754	# Filter INDEX-ALL to contain only the components we want and without
2755	# anything marked as "Ignore".
2756	fetch_filter_metadata INDEX-ALL || return 1
2757
2758	# Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2759	# INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2760	upgrade_oldall_to_oldnew INDEX-OLD INDEX-ALL INDEX-NEW
2761
2762	# Translate /boot/${KERNCONF} or /boot/${NKERNCONF} into ${KERNELDIR}
2763	fetch_filter_kernel_names INDEX-NEW ${NKERNCONF}
2764	fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
2765
2766	# For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
2767	# system and generate an INDEX-PRESENT file.
2768	fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2769
2770	# Based on ${MERGECHANGES}, generate a file tomerge-old with the
2771	# paths and hashes of old versions of files to merge.
2772	fetch_filter_mergechanges INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old
2773
2774	# Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
2775	# correspond to lines in INDEX-PRESENT with hashes not appearing
2776	# in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
2777	# INDEX-PRESENT has type - and there isn't a corresponding entry in
2778	# INDEX-OLD with type -.
2779	fetch_filter_unmodified_notpresent	\
2780	    INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old
2781
2782	# For each entry in INDEX-PRESENT of type -, remove any corresponding
2783	# entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
2784	# of type - from INDEX-PRESENT.
2785	fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
2786
2787	# If ${ALLOWDELETE} != "yes", then remove any entries from
2788	# INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
2789	fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
2790
2791	# If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
2792	# INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
2793	# replace the corresponding line of INDEX-NEW with one having the
2794	# same metadata as the entry in INDEX-PRESENT.
2795	fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
2796
2797	# Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
2798	# no need to update a file if it isn't changing.
2799	fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
2800
2801	# Fetch "clean" files from the old release for merging changes.
2802	fetch_files_premerge tomerge-old
2803
2804	# Prepare to fetch files: Generate a list of the files we need,
2805	# copy the unmodified files we have into /files/, and generate
2806	# a list of patches to download.
2807	fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2808
2809	# Fetch patches from to-${RELNUM}/${ARCH}/bp/
2810	PATCHDIR=to-${RELNUM}/${ARCH}/bp
2811	fetch_files || return 1
2812
2813	# Merge configuration file changes.
2814	upgrade_merge tomerge-old INDEX-PRESENT INDEX-NEW || return 1
2815
2816	# Create and populate install manifest directory; and report what
2817	# updates are available.
2818	fetch_create_manifest || return 1
2819
2820	# Leave a note behind to tell the "install" command that the kernel
2821	# needs to be installed before the world.
2822	touch ${BDHASH}-install/kernelfirst
2823
2824	# Remind the user that they need to run "freebsd-update install"
2825	# to install the downloaded bits, in case they didn't RTFM.
2826	echo "To install the downloaded upgrades, run '`basename $0` [options] install'."
2827}
2828
2829# Make sure that all the file hashes mentioned in $@ have corresponding
2830# gzipped files stored in /files/.
2831install_verify () {
2832	# Generate a list of hashes
2833	cat $@ |
2834	    cut -f 2,7 -d '|' |
2835	    grep -E '^f' |
2836	    cut -f 2 -d '|' |
2837	    sort -u > filelist
2838
2839	# Make sure all the hashes exist
2840	while read HASH; do
2841		if ! [ -f files/${HASH}.gz ]; then
2842			echo -n "Update files missing -- "
2843			echo "this should never happen."
2844			echo "Re-run '`basename $0` [options] fetch'."
2845			return 1
2846		fi
2847	done < filelist
2848
2849	# Clean up
2850	rm filelist
2851}
2852
2853# Remove the system immutable flag from files
2854install_unschg () {
2855	# Generate file list
2856	cat $@ |
2857	    cut -f 1 -d '|' > filelist
2858
2859	# Remove flags
2860	while read F; do
2861		if ! [ -e ${BASEDIR}/${F} ]; then
2862			continue
2863		else
2864			echo ${BASEDIR}/${F}
2865		fi
2866	done < filelist | xargs chflags noschg || return 1
2867
2868	# Clean up
2869	rm filelist
2870}
2871
2872# Decide which directory name to use for kernel backups.
2873backup_kernel_finddir () {
2874	CNT=0
2875	while true ; do
2876		# Pathname does not exist, so it is OK use that name
2877		# for backup directory.
2878		if [ ! -e $BASEDIR/$BACKUPKERNELDIR ]; then
2879			return 0
2880		fi
2881
2882		# If directory do exist, we only use if it has our
2883		# marker file.
2884		if [ -d $BASEDIR/$BACKUPKERNELDIR -a \
2885			-e $BASEDIR/$BACKUPKERNELDIR/.freebsd-update ]; then
2886			return 0
2887		fi
2888
2889		# We could not use current directory name, so add counter to
2890		# the end and try again.
2891		CNT=$((CNT + 1))
2892		if [ $CNT -gt 9 ]; then
2893			echo "Could not find valid backup dir ($BASEDIR/$BACKUPKERNELDIR)"
2894			exit 1
2895		fi
2896		BACKUPKERNELDIR="`echo $BACKUPKERNELDIR | sed -Ee 's/[0-9]\$//'`"
2897		BACKUPKERNELDIR="${BACKUPKERNELDIR}${CNT}"
2898	done
2899}
2900
2901# Backup the current kernel using hardlinks, if not disabled by user.
2902# Since we delete all files in the directory used for previous backups
2903# we create a marker file called ".freebsd-update" in the directory so
2904# we can determine on the next run that the directory was created by
2905# freebsd-update and we then do not accidentally remove user files in
2906# the unlikely case that the user has created a directory with a
2907# conflicting name.
2908backup_kernel () {
2909	# Only make kernel backup is so configured.
2910	if [ $BACKUPKERNEL != yes ]; then
2911		return 0
2912	fi
2913
2914	# Decide which directory name to use for kernel backups.
2915	backup_kernel_finddir
2916
2917	# Remove old kernel backup files.  If $BACKUPKERNELDIR was
2918	# "not ours", backup_kernel_finddir would have exited, so
2919	# deleting the directory content is as safe as we can make it.
2920	if [ -d $BASEDIR/$BACKUPKERNELDIR ]; then
2921		rm -fr $BASEDIR/$BACKUPKERNELDIR
2922	fi
2923
2924	# Create directories for backup.
2925	mkdir -p $BASEDIR/$BACKUPKERNELDIR
2926	mtree -cdn -p "${BASEDIR}/${KERNELDIR}" | \
2927	    mtree -Ue -p "${BASEDIR}/${BACKUPKERNELDIR}" > /dev/null
2928
2929	# Mark the directory as having been created by freebsd-update.
2930	touch $BASEDIR/$BACKUPKERNELDIR/.freebsd-update
2931	if [ $? -ne 0 ]; then
2932		echo "Could not create kernel backup directory"
2933		exit 1
2934	fi
2935
2936	# Disable pathname expansion to be sure *.symbols is not
2937	# expanded.
2938	set -f
2939
2940	# Use find to ignore symbol files, unless disabled by user.
2941	if [ $BACKUPKERNELSYMBOLFILES = yes ]; then
2942		FINDFILTER=""
2943	else
2944		FINDFILTER="-a ! -name *.debug -a ! -name *.symbols"
2945	fi
2946
2947	# Backup all the kernel files using hardlinks.
2948	(cd ${BASEDIR}/${KERNELDIR} && find . -type f $FINDFILTER -exec \
2949	    cp -pl '{}' ${BASEDIR}/${BACKUPKERNELDIR}/'{}' \;)
2950
2951	# Re-enable pathname expansion.
2952	set +f
2953}
2954
2955# Check for and remove an existing directory that conflicts with the file or
2956# symlink that we are going to install.
2957dir_conflict () {
2958	if [ -d "$1" ]; then
2959		echo "Removing conflicting directory $1"
2960		rm -rf -- "$1"
2961	fi
2962}
2963
2964# Install new files
2965install_from_index () {
2966	# First pass: Do everything apart from setting file flags.  We
2967	# can't set flags yet, because schg inhibits hard linking.
2968	sort -k 1,1 -t '|' $1 |
2969	    tr '|' ' ' |
2970	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
2971		case ${TYPE} in
2972		d)
2973			# Create a directory.  A file may change to a directory
2974			# on upgrade (PR273661).  If that happens, remove the
2975			# file first.
2976			if [ -e "${BASEDIR}/${FPATH}" ] && \
2977			    ! [ -d "${BASEDIR}/${FPATH}" ]; then
2978				rm -f -- "${BASEDIR}/${FPATH}"
2979			fi
2980			install -d -o ${OWNER} -g ${GROUP}		\
2981			    -m ${PERM} ${BASEDIR}/${FPATH}
2982			;;
2983		f)
2984			dir_conflict "${BASEDIR}/${FPATH}"
2985			if [ -z "${LINK}" ]; then
2986				# Create a file, without setting flags.
2987				gunzip < files/${HASH}.gz > ${HASH}
2988				install -S -o ${OWNER} -g ${GROUP}	\
2989				    -m ${PERM} ${HASH} ${BASEDIR}/${FPATH}
2990				rm ${HASH}
2991			else
2992				# Create a hard link.
2993				ln -f ${BASEDIR}/${LINK} ${BASEDIR}/${FPATH}
2994			fi
2995			;;
2996		L)
2997			dir_conflict "${BASEDIR}/${FPATH}"
2998			# Create a symlink
2999			ln -sfh ${HASH} ${BASEDIR}/${FPATH}
3000			;;
3001		esac
3002	    done
3003
3004	# Perform a second pass, adding file flags.
3005	tr '|' ' ' < $1 |
3006	    while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
3007		if [ ${TYPE} = "f" ] &&
3008		    ! [ ${FLAGS} = "0" ]; then
3009			chflags ${FLAGS} ${BASEDIR}/${FPATH}
3010		fi
3011	    done
3012}
3013
3014# Remove files which we want to delete
3015install_delete () {
3016	# Generate list of new files
3017	cut -f 1 -d '|' < $2 |
3018	    sort > newfiles
3019
3020	# Generate subindex of old files we want to nuke
3021	sort -k 1,1 -t '|' $1 |
3022	    join -t '|' -v 1 - newfiles |
3023	    sort -r -k 1,1 -t '|' |
3024	    cut -f 1,2 -d '|' |
3025	    tr '|' ' ' > killfiles
3026
3027	# Remove the offending bits
3028	while read FPATH TYPE; do
3029		case ${TYPE} in
3030		d)
3031			rmdir ${BASEDIR}/${FPATH}
3032			;;
3033		f)
3034			if [ -f "${BASEDIR}/${FPATH}" ]; then
3035				rm "${BASEDIR}/${FPATH}"
3036			fi
3037			;;
3038		L)
3039			if [ -L "${BASEDIR}/${FPATH}" ]; then
3040				rm "${BASEDIR}/${FPATH}"
3041			fi
3042			;;
3043		esac
3044	done < killfiles
3045
3046	# Clean up
3047	rm newfiles killfiles
3048}
3049
3050# Install new files, delete old files, and update generated files
3051install_files () {
3052	# If we haven't already dealt with the kernel, deal with it.
3053	if ! [ -f $1/kerneldone ]; then
3054		grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
3055		grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
3056
3057		# Backup current kernel before installing a new one
3058		backup_kernel || return 1
3059
3060		# Install new files
3061		install_from_index INDEX-NEW || return 1
3062
3063		# Remove files which need to be deleted
3064		install_delete INDEX-OLD INDEX-NEW || return 1
3065
3066		# Update linker.hints if necessary
3067		if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3068			kldxref -R ${BASEDIR}/boot/ 2>/dev/null
3069		fi
3070
3071		# We've finished updating the kernel.
3072		touch $1/kerneldone
3073
3074		# Do we need to ask for a reboot now?
3075		if [ -f $1/kernelfirst ] &&
3076		    [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3077			cat <<-EOF
3078
3079Kernel updates have been installed.  Please reboot and run
3080'`basename $0` [options] install' again to finish installing updates.
3081			EOF
3082			exit 0
3083		fi
3084	fi
3085
3086	# If we haven't already dealt with the world, deal with it.
3087	if ! [ -f $1/worlddone ]; then
3088		# Create any necessary directories first
3089		grep -vE '^/boot/' $1/INDEX-NEW |
3090		    grep -E '^[^|]+\|d\|' > INDEX-NEW
3091		install_from_index INDEX-NEW || return 1
3092
3093		# Install new runtime linker
3094		grep -vE '^/boot/' $1/INDEX-NEW |
3095		    grep -vE '^[^|]+\|d\|' |
3096		    grep -E '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3097		install_from_index INDEX-NEW || return 1
3098
3099		# Install new shared libraries next
3100		grep -vE '^/boot/' $1/INDEX-NEW |
3101		    grep -vE '^[^|]+\|d\|' |
3102		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3103		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3104		install_from_index INDEX-NEW || return 1
3105
3106		# Deal with everything else
3107		grep -vE '^/boot/' $1/INDEX-OLD |
3108		    grep -vE '^[^|]+\|d\|' |
3109		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3110		    grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
3111		grep -vE '^/boot/' $1/INDEX-NEW |
3112		    grep -vE '^[^|]+\|d\|' |
3113		    grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3114		    grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3115		install_from_index INDEX-NEW || return 1
3116		install_delete INDEX-OLD INDEX-NEW || return 1
3117
3118		# Restart host sshd if running (PR263489).  Note that this does
3119		# not affect child sshd processes handling existing sessions.
3120		if [ "$BASEDIR" = / ] && \
3121		    service sshd status >/dev/null 2>/dev/null; then
3122			echo
3123			echo "Restarting sshd after upgrade"
3124			service sshd restart
3125		fi
3126
3127		# Rehash certs if we actually have certctl installed.
3128		if which certctl>/dev/null; then
3129			env DESTDIR=${BASEDIR} certctl rehash
3130		fi
3131
3132		# Rebuild generated pwd files and /etc/login.conf.db.
3133		pwd_mkdb -d ${BASEDIR}/etc -p ${BASEDIR}/etc/master.passwd
3134		cap_mkdb ${BASEDIR}/etc/login.conf
3135
3136		# Rebuild man page databases, if necessary.
3137		for D in /usr/share/man /usr/share/openssl/man; do
3138			if [ ! -d ${BASEDIR}/$D ]; then
3139				continue
3140			fi
3141			if [ -f ${BASEDIR}/$D/mandoc.db ] && \
3142			    [ -z "$(find ${BASEDIR}/$D -type f -newer ${BASEDIR}/$D/mandoc.db)" ]; then
3143				continue;
3144			fi
3145			makewhatis ${BASEDIR}/$D
3146		done
3147
3148		# We've finished installing the world and deleting old files
3149		# which are not shared libraries.
3150		touch $1/worlddone
3151
3152		# Do we need to ask the user to portupgrade now?
3153		grep -vE '^/boot/' $1/INDEX-NEW |
3154		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
3155		    cut -f 1 -d '|' |
3156		    sort > newfiles
3157		if grep -vE '^/boot/' $1/INDEX-OLD |
3158		    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
3159		    cut -f 1 -d '|' |
3160		    sort |
3161		    join -v 1 - newfiles |
3162		    grep -q .; then
3163			cat <<-EOF
3164
3165Completing this upgrade requires removing old shared object files.
3166Please rebuild all installed 3rd party software (e.g., programs
3167installed from the ports tree) and then run
3168'`basename $0` [options] install' again to finish installing updates.
3169			EOF
3170			rm newfiles
3171			exit 0
3172		fi
3173		rm newfiles
3174	fi
3175
3176	# Remove old shared libraries
3177	grep -vE '^/boot/' $1/INDEX-NEW |
3178	    grep -vE '^[^|]+\|d\|' |
3179	    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3180	grep -vE '^/boot/' $1/INDEX-OLD |
3181	    grep -vE '^[^|]+\|d\|' |
3182	    grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
3183	install_delete INDEX-OLD INDEX-NEW || return 1
3184
3185	# Remove old directories
3186	grep -vE '^/boot/' $1/INDEX-NEW |
3187	    grep -E '^[^|]+\|d\|' > INDEX-NEW
3188	grep -vE '^/boot/' $1/INDEX-OLD |
3189	    grep -E '^[^|]+\|d\|' > INDEX-OLD
3190	install_delete INDEX-OLD INDEX-NEW || return 1
3191
3192	# Remove temporary files
3193	rm INDEX-OLD INDEX-NEW
3194}
3195
3196# Rearrange bits to allow the installed updates to be rolled back
3197install_setup_rollback () {
3198	# Remove the "reboot after installing kernel", "kernel updated", and
3199	# "finished installing the world" flags if present -- they are
3200	# irrelevant when rolling back updates.
3201	if [ -f ${BDHASH}-install/kernelfirst ]; then
3202		rm ${BDHASH}-install/kernelfirst
3203		rm ${BDHASH}-install/kerneldone
3204	fi
3205	if [ -f ${BDHASH}-install/worlddone ]; then
3206		rm ${BDHASH}-install/worlddone
3207	fi
3208
3209	if [ -L ${BDHASH}-rollback ]; then
3210		mv ${BDHASH}-rollback ${BDHASH}-install/rollback
3211	fi
3212
3213	mv ${BDHASH}-install ${BDHASH}-rollback
3214}
3215
3216# Actually install updates
3217install_run () {
3218	echo -n "Installing updates..."
3219
3220	# Make sure we have all the files we should have
3221	install_verify ${BDHASH}-install/INDEX-OLD	\
3222	    ${BDHASH}-install/INDEX-NEW || return 1
3223
3224	# Remove system immutable flag from files
3225	install_unschg ${BDHASH}-install/INDEX-OLD	\
3226	    ${BDHASH}-install/INDEX-NEW || return 1
3227
3228	# Install new files, delete old files, and update linker.hints
3229	install_files ${BDHASH}-install || return 1
3230
3231	# Rearrange bits to allow the installed updates to be rolled back
3232	install_setup_rollback
3233
3234	echo " done."
3235}
3236
3237# Rearrange bits to allow the previous set of updates to be rolled back next.
3238rollback_setup_rollback () {
3239	if [ -L ${BDHASH}-rollback/rollback ]; then
3240		mv ${BDHASH}-rollback/rollback rollback-tmp
3241		rm -r ${BDHASH}-rollback/
3242		rm ${BDHASH}-rollback
3243		mv rollback-tmp ${BDHASH}-rollback
3244	else
3245		rm -r ${BDHASH}-rollback/
3246		rm ${BDHASH}-rollback
3247	fi
3248}
3249
3250# Install old files, delete new files, and update linker.hints
3251rollback_files () {
3252	# Create directories first.  They may be needed by files we will
3253	# install in subsequent steps (PR273950).
3254	awk -F \| '{if ($2 == "d") print }' $1/INDEX-OLD > INDEX-OLD
3255	install_from_index INDEX-OLD || return 1
3256
3257	# Install old shared library files which don't have the same path as
3258	# a new shared library file.
3259	grep -vE '^/boot/' $1/INDEX-NEW |
3260	    grep -E '/lib/.*\.so\.[0-9]+\|' |
3261	    cut -f 1 -d '|' |
3262	    sort > INDEX-NEW.libs.flist
3263	grep -vE '^/boot/' $1/INDEX-OLD |
3264	    grep -E '/lib/.*\.so\.[0-9]+\|' |
3265	    sort -k 1,1 -t '|' - |
3266	    join -t '|' -v 1 - INDEX-NEW.libs.flist > INDEX-OLD
3267	install_from_index INDEX-OLD || return 1
3268
3269	# Deal with files which are neither kernel nor shared library
3270	grep -vE '^/boot/' $1/INDEX-OLD |
3271	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
3272	grep -vE '^/boot/' $1/INDEX-NEW |
3273	    grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
3274	install_from_index INDEX-OLD || return 1
3275	install_delete INDEX-NEW INDEX-OLD || return 1
3276
3277	# Install any old shared library files which we didn't install above.
3278	grep -vE '^/boot/' $1/INDEX-OLD |
3279	    grep -E '/lib/.*\.so\.[0-9]+\|' |
3280	    sort -k 1,1 -t '|' - |
3281	    join -t '|' - INDEX-NEW.libs.flist > INDEX-OLD
3282	install_from_index INDEX-OLD || return 1
3283
3284	# Delete unneeded shared library files
3285	grep -vE '^/boot/' $1/INDEX-OLD |
3286	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
3287	grep -vE '^/boot/' $1/INDEX-NEW |
3288	    grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
3289	install_delete INDEX-NEW INDEX-OLD || return 1
3290
3291	# Deal with kernel files
3292	grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
3293	grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
3294	install_from_index INDEX-OLD || return 1
3295	install_delete INDEX-NEW INDEX-OLD || return 1
3296	if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3297		kldxref -R /boot/ 2>/dev/null
3298	fi
3299
3300	# Remove temporary files
3301	rm INDEX-OLD INDEX-NEW INDEX-NEW.libs.flist
3302}
3303
3304# Actually rollback updates
3305rollback_run () {
3306	echo -n "Uninstalling updates..."
3307
3308	# If there are updates waiting to be installed, remove them; we
3309	# want the user to re-run 'fetch' after rolling back updates.
3310	if [ -L ${BDHASH}-install ]; then
3311		rm -r ${BDHASH}-install/
3312		rm ${BDHASH}-install
3313	fi
3314
3315	# Make sure we have all the files we should have
3316	install_verify ${BDHASH}-rollback/INDEX-NEW	\
3317	    ${BDHASH}-rollback/INDEX-OLD || return 1
3318
3319	# Remove system immutable flag from files
3320	install_unschg ${BDHASH}-rollback/INDEX-NEW	\
3321	    ${BDHASH}-rollback/INDEX-OLD || return 1
3322
3323	# Install old files, delete new files, and update linker.hints
3324	rollback_files ${BDHASH}-rollback || return 1
3325
3326	# Remove the rollback directory and the symlink pointing to it; and
3327	# rearrange bits to allow the previous set of updates to be rolled
3328	# back next.
3329	rollback_setup_rollback
3330
3331	echo " done."
3332}
3333
3334# Compare INDEX-ALL and INDEX-PRESENT and print warnings about differences.
3335IDS_compare () {
3336	# Get all the lines which mismatch in something other than file
3337	# flags.  We ignore file flags because sysinstall doesn't seem to
3338	# set them when it installs FreeBSD; warning about these adds a
3339	# very large amount of noise.
3340	cut -f 1-5,7-8 -d '|' $1 > $1.noflags
3341	sort -k 1,1 -t '|' $1.noflags > $1.sorted
3342	cut -f 1-5,7-8 -d '|' $2 |
3343	    comm -13 $1.noflags - |
3344	    fgrep -v '|-|||||' |
3345	    sort -k 1,1 -t '|' |
3346	    join -t '|' $1.sorted - > INDEX-NOTMATCHING
3347
3348	# Ignore files which match IDSIGNOREPATHS.
3349	for X in ${IDSIGNOREPATHS}; do
3350		grep -E "^${X}" INDEX-NOTMATCHING
3351	done |
3352	    sort -u |
3353	    comm -13 - INDEX-NOTMATCHING > INDEX-NOTMATCHING.tmp
3354	mv INDEX-NOTMATCHING.tmp INDEX-NOTMATCHING
3355
3356	# Go through the lines and print warnings.
3357	local IFS='|'
3358	while read FPATH TYPE OWNER GROUP PERM HASH LINK P_TYPE P_OWNER P_GROUP P_PERM P_HASH P_LINK; do
3359		# Warn about different object types.
3360		if ! [ "${TYPE}" = "${P_TYPE}" ]; then
3361			echo -n "${FPATH} is a "
3362			case "${P_TYPE}" in
3363			f)	echo -n "regular file, "
3364				;;
3365			d)	echo -n "directory, "
3366				;;
3367			L)	echo -n "symlink, "
3368				;;
3369			esac
3370			echo -n "but should be a "
3371			case "${TYPE}" in
3372			f)	echo -n "regular file."
3373				;;
3374			d)	echo -n "directory."
3375				;;
3376			L)	echo -n "symlink."
3377				;;
3378			esac
3379			echo
3380
3381			# Skip other tests, since they don't make sense if
3382			# we're comparing different object types.
3383			continue
3384		fi
3385
3386		# Warn about different owners.
3387		if ! [ "${OWNER}" = "${P_OWNER}" ]; then
3388			echo -n "${FPATH} is owned by user id ${P_OWNER}, "
3389			echo "but should be owned by user id ${OWNER}."
3390		fi
3391
3392		# Warn about different groups.
3393		if ! [ "${GROUP}" = "${P_GROUP}" ]; then
3394			echo -n "${FPATH} is owned by group id ${P_GROUP}, "
3395			echo "but should be owned by group id ${GROUP}."
3396		fi
3397
3398		# Warn about different permissions.  We do not warn about
3399		# different permissions on symlinks, since some archivers
3400		# don't extract symlink permissions correctly and they are
3401		# ignored anyway.
3402		if ! [ "${PERM}" = "${P_PERM}" ] &&
3403		    ! [ "${TYPE}" = "L" ]; then
3404			echo -n "${FPATH} has ${P_PERM} permissions, "
3405			echo "but should have ${PERM} permissions."
3406		fi
3407
3408		# Warn about different file hashes / symlink destinations.
3409		if ! [ "${HASH}" = "${P_HASH}" ]; then
3410			if [ "${TYPE}" = "L" ]; then
3411				echo -n "${FPATH} is a symlink to ${P_HASH}, "
3412				echo "but should be a symlink to ${HASH}."
3413			fi
3414			if [ "${TYPE}" = "f" ]; then
3415				echo -n "${FPATH} has SHA256 hash ${P_HASH}, "
3416				echo "but should have SHA256 hash ${HASH}."
3417			fi
3418		fi
3419
3420		# We don't warn about different hard links, since some
3421		# some archivers break hard links, and as long as the
3422		# underlying data is correct they really don't matter.
3423	done < INDEX-NOTMATCHING
3424
3425	# Clean up
3426	rm $1 $1.noflags $1.sorted $2 INDEX-NOTMATCHING
3427}
3428
3429# Do the work involved in comparing the system to a "known good" index
3430IDS_run () {
3431	workdir_init || return 1
3432
3433	# Prepare the mirror list.
3434	fetch_pick_server_init && fetch_pick_server
3435
3436	# Try to fetch the public key until we run out of servers.
3437	while ! fetch_key; do
3438		fetch_pick_server || return 1
3439	done
3440
3441	# Try to fetch the metadata index signature ("tag") until we run
3442	# out of available servers; and sanity check the downloaded tag.
3443	while ! fetch_tag; do
3444		fetch_pick_server || return 1
3445	done
3446	fetch_tagsanity || return 1
3447
3448	# Fetch INDEX-OLD and INDEX-ALL.
3449	fetch_metadata INDEX-OLD INDEX-ALL || return 1
3450
3451	# Generate filtered INDEX-OLD and INDEX-ALL files containing only
3452	# the components we want and without anything marked as "Ignore".
3453	fetch_filter_metadata INDEX-OLD || return 1
3454	fetch_filter_metadata INDEX-ALL || return 1
3455
3456	# Merge the INDEX-OLD and INDEX-ALL files into INDEX-ALL.
3457	sort INDEX-OLD INDEX-ALL > INDEX-ALL.tmp
3458	mv INDEX-ALL.tmp INDEX-ALL
3459	rm INDEX-OLD
3460
3461	# Translate /boot/${KERNCONF} to ${KERNELDIR}
3462	fetch_filter_kernel_names INDEX-ALL ${KERNCONF}
3463
3464	# Inspect the system and generate an INDEX-PRESENT file.
3465	fetch_inspect_system INDEX-ALL INDEX-PRESENT /dev/null || return 1
3466
3467	# Compare INDEX-ALL and INDEX-PRESENT and print warnings about any
3468	# differences.
3469	IDS_compare INDEX-ALL INDEX-PRESENT
3470}
3471
3472#### Main functions -- call parameter-handling and core functions
3473
3474# Using the command line, configuration file, and defaults,
3475# set all the parameters which are needed later.
3476get_params () {
3477	init_params
3478	parse_cmdline $@
3479	parse_conffile
3480	default_params
3481}
3482
3483# Fetch command.  Make sure that we're being called
3484# interactively, then run fetch_check_params and fetch_run
3485cmd_fetch () {
3486	finalize_components_config ${COMPONENTS}
3487	if [ ! -t 0 -a $NOTTYOK -eq 0 ]; then
3488		echo -n "`basename $0` fetch should not "
3489		echo "be run non-interactively."
3490		echo "Run `basename $0` cron instead."
3491		exit 1
3492	fi
3493	fetch_check_params
3494	fetch_run || exit 1
3495	ISFETCHED=1
3496}
3497
3498# Cron command.  Make sure the parameters are sensible; wait
3499# rand(3600) seconds; then fetch updates.  While fetching updates,
3500# send output to a temporary file; only print that file if the
3501# fetching failed.
3502cmd_cron () {
3503	fetch_check_params
3504	sleep `jot -r 1 0 3600`
3505
3506	TMPFILE=`mktemp /tmp/freebsd-update.XXXXXX` || exit 1
3507	finalize_components_config ${COMPONENTS} >> ${TMPFILE}
3508	if ! fetch_run >> ${TMPFILE} ||
3509	    ! grep -q "No updates needed" ${TMPFILE} ||
3510	    [ ${VERBOSELEVEL} = "debug" ]; then
3511		mail -s "`hostname` security updates" ${MAILTO} < ${TMPFILE}
3512	fi
3513	ISFETCHED=1
3514
3515	rm ${TMPFILE}
3516}
3517
3518# Fetch files for upgrading to a new release.
3519cmd_upgrade () {
3520	finalize_components_config ${COMPONENTS}
3521	upgrade_check_params
3522	upgrade_check_kmod_ports
3523	upgrade_run || exit 1
3524}
3525
3526# Check if there are fetched updates ready to install.
3527# Chdir into the working directory.
3528cmd_updatesready () {
3529	finalize_components_config ${COMPONENTS}
3530	# Check if working directory exists (if not, no updates pending)
3531	if ! [ -e "${WORKDIR}" ]; then
3532		echo "No updates are available to install."
3533		exit 2
3534	fi
3535
3536	# Change into working directory (fail if no permission/directory etc.)
3537	cd ${WORKDIR} || exit 1
3538
3539	# Construct a unique name from ${BASEDIR}
3540	BDHASH=`echo ${BASEDIR} | sha256 -q`
3541
3542	# Check that we have updates ready to install
3543	if ! [ -L ${BDHASH}-install ]; then
3544		echo "No updates are available to install."
3545		exit 2
3546	fi
3547
3548	echo "There are updates available to install."
3549	echo "Run '`basename $0` [options] install' to proceed."
3550}
3551
3552# Install downloaded updates.
3553cmd_install () {
3554	finalize_components_config ${COMPONENTS}
3555	install_check_params
3556	install_create_be
3557	install_run || exit 1
3558}
3559
3560# Rollback most recently installed updates.
3561cmd_rollback () {
3562	finalize_components_config ${COMPONENTS}
3563	rollback_check_params
3564	rollback_run || exit 1
3565}
3566
3567# Compare system against a "known good" index.
3568cmd_IDS () {
3569	finalize_components_config ${COMPONENTS}
3570	IDS_check_params
3571	IDS_run || exit 1
3572}
3573
3574# Output configuration.
3575cmd_showconfig () {
3576	finalize_components_config ${COMPONENTS}
3577	for X in ${CONFIGOPTIONS}; do
3578		echo $X=$(eval echo \$${X})
3579	done
3580}
3581
3582#### Entry point
3583
3584# Make sure we find utilities from the base system
3585export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH}
3586
3587# Set a pager if the user doesn't
3588if [ -z "$PAGER" ]; then
3589	PAGER=/usr/bin/less
3590fi
3591
3592# Set LC_ALL in order to avoid problems with character ranges like [A-Z].
3593export LC_ALL=C
3594
3595# Clear environment variables that may affect operation of tools that we use.
3596unset GREP_OPTIONS
3597
3598get_params $@
3599for COMMAND in ${COMMANDS}; do
3600	cmd_${COMMAND}
3601done
3602